Skip to content

Commit

Permalink
Added system test prestage jars
Browse files Browse the repository at this point in the history 
- Enables to avoid system test prerequisite zip file being empty.
- System jars are downloaded under system_lib.

related: adoptium/aqa-tests#4912
related: adoptium#560
related: adoptium#561

Signed-off-by: Anna Babu Palathingal <anna.bp@ibm.com>
  • Loading branch information
@annaibm
annaibm committed Jun 5, 2024
1 parent 8cf4d64 commit 7c3ad9b
Showing 1 changed file with 132 additions and 31 deletions.
163 changes: 132 additions & 31 deletions scripts/getDependencies.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,14 +218,85 @@
sha1 => '174bba1574c37352b0eb2c06e02b6403738ad57c'
});

my @dependencies = split(',', $dependencyList);
my %system_jars = (
json_simple => {
url => 'https://repo1.maven.org/maven2/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar',
fname => 'json-simple.jar',
sha1 => 'c9ad4a0850ab676c5c64461a05ca524cdfff59f1',
},
jcstress => {
url => 'https://builds.shipilev.net/jcstress/jcstress-tests-all-20240222.jar',
fname => 'jcstress-tests-all-20240222.jar',
sha1 => '200da75e67689e8a604ec6fe9a6f55b2c000b6ce',
},
ant_launcher => {
url => 'https://repo1.maven.org/maven2/org/apache/ant/ant-launcher/1.8.1/ant-launcher-1.8.1.jar',
dir => 'apache-ant/lib',
fname => 'ant-launcher.jar',
is_system_test => 1
},
asm => {
url => 'https://repository.ow2.org/nexus/content/repositories/releases/org/ow2/asm/asm/9.0/asm-9.0.jar',
dir => 'asm',
fname => 'asm.jar',
is_system_test => 1
},
cvsclient => {
url => 'https://repo1.maven.org/maven2/org/netbeans/lib/cvsclient/20060125/cvsclient-20060125.jar',
dir => 'cvsclient',
fname => 'org-netbeans-lib-cvsclient.jar',
is_system_test => 1
},
hamcrest_core => {
url => 'https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar',
dir => 'junit',
fname => 'hamcrest-core.jar',
is_system_test => 1
},
junit => {
url => 'https://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar',
dir => 'junit',
fname => 'junit.jar',
is_system_test => 1
},
log4j_api => {
url => 'https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.15.0/log4j-api-2.15.0.jar',
dir => 'log4j',
fname => 'log4j-api.jar',
is_system_test => 1
},
log4j_core => {
url => 'https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.15.0/log4j-core-2.15.0.jar',
dir => 'log4j',
fname => 'log4j-core.jar',
is_system_test => 1
},
mauve => {
url => 'https://ci.adoptium.net/job/systemtest.getDependency/lastSuccessfulBuild/artifact/systemtest_prereqs/mauve/mauve.jar',
dir => 'mauve',
fname => 'mauve.jar',
is_system_test => 1
},
tools => {
url => 'https://ci.adoptium.net/job/systemtest.getDependency/lastSuccessfulBuild/artifact/systemtest_prereqs/tools/tools.jar',
dir => 'tools',
fname => 'tools.jar',
is_system_test => 1
});

my %jars_to_use;
if ($path =~ /system_lib/) {
%jars_to_use = (%system_jars);
} else {
%jars_to_use = %base;
}
my @dependencies = split(',', $dependencyList);
# Put all dependent jars hash to array to prepare downloading
my @jars_info;
foreach my $dependency (keys %base) {
foreach my $dependency (keys %jars_to_use) {
foreach my $i (@dependencies) {
if ($i eq "all" || $dependency eq $i) {
push(@jars_info, $base{$dependency});
push(@jars_info, $jars_to_use{$dependency});
}
}
}
Expand All @@ -242,15 +313,33 @@
for my $i (0 .. $#jars_info) {
my $url = $jars_info[$i]{url};
my $fn = $jars_info[$i]{fname};
my $filename = $path . $sep . $fn;
my $sha1 = $jars_info[$i]{sha1};
my $dir = $jars_info[$i]{dir} // "";
my $full_dir_path = File::Spec->catdir($path, $dir);
my $url_custom = $customUrl;

if (!-d $full_dir_path) {
make_path($full_dir_path, {chmod => 0755, verbose => 1}) or die "Failed to create directory: $full_dir_path: $!";
print "Directory created: $full_dir_path\n";
}

my $filename = File::Spec->catfile($full_dir_path, $fn);
my $shaurl = $jars_info[$i]{shaurl};
my $shafn = $jars_info[$i]{shafn};

# if customUrl is provided, use customUrl and reset $url and $shaurl
if ($customUrl ne "") {
$url = "$customUrl/$fn";
# if url_custom is provided, use url_custom and reset $url and $shaurl
if ($url_custom ne "") {
if (defined $jars_info[$i]{is_system_test} && $jars_info[$i]{is_system_test} == 1) {
$url_custom =~ s/test.getDependency/systemtest.getDependency/;
$url_custom .= "systemtest_prereqs/";
$url_custom .= $jars_info[$i]{dir};
$url_custom .= '/' unless $url_custom =~ /\/$/;
$url_custom .= $jars_info[$i]{fname};
$url = "$url_custom";
}

if (defined $shaurl && $shaurl ne '') {
$shaurl = "$customUrl/$shafn";
$shaurl = "$url_custom/$shafn";
}
}

Expand All @@ -268,10 +357,12 @@

my $expectedsha = $jars_info[$i]{sha1};
if (!$expectedsha) {
$shafn = $path . $sep . $shafn;
# if the sha file exists, parse the file and get the expected sha
if (-e $shafn) {
$expectedsha = getShaFromFile($shafn, $fn);
if (defined $shafn && $shafn ne '') {
$shafn = $path . $sep . $shafn;
# if the sha file exists, parse the file and get the expected sha
if (-e $shafn) {
$expectedsha = getShaFromFile($shafn, $fn);
}
}

# if expectedsha is not set above and shaurl is provided, download the sha file
Expand All @@ -287,30 +378,40 @@
next;
}

my $ignoreChecksum = (!defined $sha1 || $sha1 eq '') && (!defined $shaurl || $shaurl eq '');
# download the dependent third party jar
downloadFile($url, $filename);

# if shaurl is provided, re-download the sha file and reset the expectedsha value
# as the dependent third party jar is newly downloadeded
if ($shaurl) {
downloadFile($shaurl, $shafn);
$expectedsha = getShaFromFile($shafn, $fn);
}
if ($ignoreChecksum && -e $filename) {
print "$filename exists, not downloading.\n";
} else {
downloadFile($url, $filename);

if (!$expectedsha) {
die "ERROR: cannot get the expected sha for file $fn.\n";
}
# if shaurl is provided, re-download the sha file and reset the expectedsha value
# as the dependent third party jar is newly downloadeded
if (!$ignoreChecksum) {
if ($shaurl) {
downloadFile($shaurl, $shafn);
$expectedsha = getShaFromFile($shafn, $fn);
}

# validate dependencies sha sum
$sha = Digest::SHA->new($shaalg);
$sha->addfile($filename);
$digest = $sha->hexdigest;
if (!$expectedsha) {
die "ERROR: cannot get the expected sha for file $fn.\n";
}

if ($digest ne $expectedsha) {
print "Expected sha is: $expectedsha,\n";
print "Actual sha is : $digest.\n";
print "Please delete $filename and rerun the program!";
die "ERROR: sha checksum error.\n";
# validate dependencies sha sum
$sha = Digest::SHA->new($shaalg);
$sha->addfile($filename);
$digest = $sha->hexdigest;

if ($digest ne $expectedsha) {
print "Expected sha is: $expectedsha,\n";
print "Actual sha is : $digest.\n";
print "Please delete $filename and rerun the program!";
die "ERROR: sha checksum error.\n";
}
} else {
print "Checksum verification skipped for $filename\n";
}
}
}
print "downloaded dependent third party jars successfully\n";
Expand Down

0 comments on commit 7c3ad9b

Please sign in to comment.