Skip to content

Commit

Permalink
Merge branch '2.8-dls-perf' into 2.9-dls-perf
Browse files Browse the repository at this point in the history
Signed-off-by: Peter Nied <petern@amazon.com>
  • Loading branch information
peternied committed Nov 30, 2023
2 parents 6170fce + 3662949 commit 678077c
Show file tree
Hide file tree
Showing 162 changed files with 13,922 additions and 37 deletions.
68 changes: 68 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ on: [push, pull_request]

env:
GRADLE_OPTS: -Dhttp.keepAlive=false
CI_ENVIRONMENT: normal

jobs:
generate-test-list:
Expand Down Expand Up @@ -106,6 +107,73 @@ jobs:
- run: ./gradlew clean build -Dbuild.snapshot=false -x test
- uses: github/codeql-action/analyze@v1

integration-tests:
name: integration-tests
strategy:
fail-fast: false
matrix:
jdk: [11]
platform: [ubuntu-latest]
runs-on: ${{ matrix.platform }}

steps:
- name: Set up JDK for build and test
uses: actions/setup-java@v3
with:
distribution: temurin # Temurin is a distribution of adoptium
java-version: ${{ matrix.jdk }}

- name: Checkout security
uses: actions/checkout@v4

- name: Build and Test
uses: gradle/gradle-build-action@v2
with:
cache-disabled: true
arguments: |
integrationTest -Dbuild.snapshot=false
- uses: alehechka/upload-tartifact@v2
if: always()
with:
name: integration-${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports
path: |
./build/reports/
resource-tests:
env:
CI_ENVIRONMENT: resource-test
strategy:
fail-fast: false
matrix:
jdk: [11]
platform: [ubuntu-latest]
runs-on: ${{ matrix.platform }}

steps:
- name: Set up JDK for build and test
uses: actions/setup-java@v3
with:
distribution: temurin # Temurin is a distribution of adoptium
java-version: ${{ matrix.jdk }}

- name: Checkout security
uses: actions/checkout@v4

- name: Build and Test
uses: gradle/gradle-build-action@v2
with:
cache-disabled: true
arguments: |
integrationTest -Dbuild.snapshot=false --tests org.opensearch.security.ResourceFocusedTests
- uses: alehechka/upload-tartifact@v2
if: always()
with:
name: resource-test-${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports
path: |
./build/reports/
build-artifact-names:
runs-on: ubuntu-latest
steps:
Expand Down
111 changes: 111 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -424,9 +424,78 @@ configurations.all {
force "com.github.luben:zstd-jni:${versions.zstd}"
force "org.xerial.snappy:snappy-java:1.1.10.1"
force "com.google.guava:guava:${guava_version}"

// For integrationTest
force "org.apache.httpcomponents:httpclient-cache:4.5.13"
force "org.apache.httpcomponents:httpclient:4.5.13"
force "org.apache.httpcomponents:fluent-hc:4.5.13"
force "org.apache.httpcomponents:httpcore:4.4.16"
force "org.apache.httpcomponents:httpcore-nio:4.4.16"
force "org.apache.httpcomponents:httpasyncclient:4.1.5"
}
}

//create source set 'integrationTest'
//add classes from the main source set to the compilation and runtime classpaths of the integrationTest
sourceSets {
integrationTest {
java {
srcDir file ('src/integrationTest/java')
compileClasspath += sourceSets.main.output
runtimeClasspath += sourceSets.main.output
}
resources {
srcDir file('src/integrationTest/resources')
}
processIntegrationTestResources {
duplicatesStrategy(DuplicatesStrategy.INCLUDE)
}
}
}

//add new task that runs integration tests
task integrationTest(type: Test) {
doFirst {
// Only run resources tests on resource-test CI environments or locally
if (System.getenv('CI_ENVIRONMENT') != 'resource-test' && System.getenv('CI_ENVIRONMENT') != null) {
exclude '**/ResourceFocusedTests.class'
}
// Only run with retries while in CI systems
if (System.getenv('CI_ENVIRONMENT') == 'normal') {
retry {
failOnPassedAfterRetry = false
maxRetries = 2
maxFailures = 10
}
}
}
description = 'Run integration tests.'
group = 'verification'
systemProperty "java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"
testClassesDirs = sourceSets.integrationTest.output.classesDirs
classpath = sourceSets.integrationTest.runtimeClasspath
//run the integrationTest task after the test task
shouldRunAfter test
jacoco {
excludes = [
"com.sun.jndi.dns.*",
"com.sun.security.sasl.gsskerb.*",
"java.sql.*",
"javax.script.*",
"org.jcp.xml.dsig.internal.dom.*",
"sun.nio.cs.ext.*",
"sun.security.ec.*",
"sun.security.jgss.*",
"sun.security.pkcs11.*",
"sun.security.smartcardio.*",
"sun.util.resources.provider.*"
]
}
}

//run the integrationTest task before the check task
check.dependsOn integrationTest

dependencies {
implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"
implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}"
Expand Down Expand Up @@ -562,6 +631,48 @@ dependencies {
implementation('com.google.googlejavaformat:google-java-format:1.17.0') {
exclude group: 'com.google.guava'
}
integrationTestCompileOnly "org.opensearch:opensearch:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
integrationTestImplementation "org.opensearch:opensearch-ssl-config:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:lang-mustache-client:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:parent-join-client:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:aggs-matrix-stats-client:${opensearch_version}"
integrationTestImplementation "com.google.guava:guava:32.1.1-jre"
integrationTestImplementation "org.apache.commons:commons-lang3:3.4"
integrationTestImplementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1'
integrationTestImplementation 'io.jsonwebtoken:jjwt-api:0.10.8'
integrationTestImplementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"

integrationTestRuntimeOnly 'org.greenrobot:eventbus:3.2.0'
integrationTestRuntimeOnly 'com.flipkart.zjsonpatch:zjsonpatch:0.4.4'
integrationTestRuntimeOnly 'commons-lang:commons-lang:2.4'
integrationTestRuntimeOnly 'org.ldaptive:ldaptive:1.2.3'
integrationTestRuntimeOnly 'commons-collections:commons-collections:3.2.2'

//integration test framework:
integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.1') {
exclude(group: 'junit', module: 'junit')
}
integrationTestImplementation 'junit:junit:4.13.2'
integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
integrationTestImplementation 'commons-io:commons-io:2.14.0'
integrationTestImplementation "org.apache.logging.log4j:log4j-core:2.17.1"
integrationTestImplementation "org.apache.logging.log4j:log4j-jul:2.17.1"
integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
integrationTestImplementation "org.bouncycastle:bcpkix-jdk15to18:1.75"
integrationTestImplementation "org.bouncycastle:bcutil-jdk15to18:1.75"
integrationTestImplementation('org.awaitility:awaitility:4.2.0') {
exclude(group: 'org.hamcrest', module: 'hamcrest')
}
integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14"
integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14"
integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.13"
integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16"
integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
}

jar {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
*/
package org.opensearch.common.logging;

/**
* Class uses to override OpenSearch NodeAndClusterIdConverter Log4j2 plugin in order to disable plugin and limit number of
* warn messages like "...ApplierService#updateTask][T#1] WARN ClusterApplierService:628 - failed to notify ClusterStateListener..."
* during tests execution.
*
* The class is rather a temporary solution and the real one should be developed in scope of:
* https://github.com/opensearch-project/OpenSearch/pull/4322
*/
import org.apache.logging.log4j.core.LogEvent;

class NodeAndClusterIdConverter {

public NodeAndClusterIdConverter() {}

public static void setNodeIdAndClusterId(String nodeId, String clusterUUID) {}

public void format(LogEvent event, StringBuilder toAppendTo) {}
}
55 changes: 55 additions & 0 deletions src/integrationTest/java/org/opensearch/node/PluginAwareNode.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Copyright 2015-2018 _floragunn_ GmbH
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

package org.opensearch.node;

import java.util.Collection;
import java.util.Collections;

import org.opensearch.common.settings.Settings;
import org.opensearch.plugins.Plugin;

public class PluginAwareNode extends Node {

private final boolean clusterManagerEligible;

public PluginAwareNode(
boolean clusterManagerEligible,
final Settings preparedSettings,
final Collection<Class<? extends Plugin>> plugins
) {
super(
InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(), null, () -> System.getenv("HOSTNAME")),
plugins,
true
);
this.clusterManagerEligible = clusterManagerEligible;
}

public boolean isClusterManagerEligible() {
return clusterManagerEligible;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
*/
package org.opensearch.security;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;

class ConfigurationFiles {

public static void createRoleMappingFile(File destination) {
String resource = "roles_mapping.yml";
copyResourceToFile(resource, destination);
}

public static Path createConfigurationDirectory() {
try {
Path tempDirectory = Files.createTempDirectory("test-security-config");
String[] configurationFiles = {
"config.yml",
"action_groups.yml",
"config.yml",
"internal_users.yml",
"roles.yml",
"roles_mapping.yml",
"security_tenants.yml",
"tenants.yml" };
for (String fileName : configurationFiles) {
Path configFileDestination = tempDirectory.resolve(fileName);
copyResourceToFile(fileName, configFileDestination.toFile());
}
return tempDirectory.toAbsolutePath();
} catch (IOException ex) {
throw new RuntimeException("Cannot create directory with security plugin configuration.", ex);
}
}

private static void copyResourceToFile(String resource, File destination) {
try (InputStream input = ConfigurationFiles.class.getClassLoader().getResourceAsStream(resource)) {
Objects.requireNonNull(input, "Cannot find source resource " + resource);
try (OutputStream output = new FileOutputStream(destination)) {
input.transferTo(output);
}
} catch (IOException e) {
throw new RuntimeException("Cannot create file with security plugin configuration", e);
}
}
}
Loading

0 comments on commit 678077c

Please sign in to comment.