Merge branch 'opensearch-project:main' into main

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @chenqi0805 @engechas @graytaylor0 @dinujoh @kkondaka @asifsmohammed @KarstenSchnitter @dlvenable @oeyh
+*   @chenqi0805 @engechas @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh

.github/workflows/kafka-plugin-integration-tests.yml

@@ -41,7 +41,7 @@ jobs:
         run: |
           echo 'KAFKA_VERSION=${{ matrix.kafka }}' > data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/.env
           docker compose --project-directory data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/zookeeper --env-file  data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/.env up -d
-          sleep 10
+          sleep 2
       - name: Wait for Kafka
         run: |
           ./gradlew data-prepper-plugins:kafka-plugins:integrationTest -Dtests.kafka.bootstrap_servers=localhost:9092 -Dtests.kafka.authconfig.username=admin -Dtests.kafka.authconfig.password=admin --tests KafkaStartIT

.gitignore

@@ -13,3 +13,6 @@ gradle/tools
 # Ignore config file generated by test
 data-prepper-main/src/test/resources/logstash-filter.yaml
 data-prepper-main/src/test/resources/logstash-conf/logstash-filter.yaml
+
+# Python virtual environments
+.venv

.whitesource

@@ -5,11 +5,29 @@
     "projectToken": "",
     "baseBranches": []
   },
+  "scanSettingsSAST": {
+    "enableScan": false,
+    "scanPullRequests": false,
+    "incrementalScan": true,
+    "baseBranches": [],
+    "snippetSize": 10
+  },
   "checkRunSettings": {
     "vulnerableCheckRunConclusionLevel": "failure",
-    "displayMode": "diff"
+    "displayMode": "diff",
+    "useMendCheckNames": true
+  },
+  "checkRunSettingsSAST": {
+    "checkRunConclusionLevel": "failure",
+    "severityThreshold": "high"
   },
   "issueSettings": {
-    "minSeverityLevel": "LOW"
+    "minSeverityLevel": "LOW",
+    "issueType": "DEPENDENCY"
+  },
+  "remediateSettings": {
+    "workflowRules": {
+      "enabled": true
+    }
   }
 }

CODE_OF_CONDUCT.md

@@ -1,5 +1,5 @@
-This code of conduct applies to all spaces provided by the OpenSource project including in code, documentation, issue trackers, mailing lists, chat channels, wikis, blogs, social media and any other communication channels used by the project.
 
+This code of conduct applies to all spaces provided by the OpenSource project including in code, documentation, issue trackers, mailing lists, chat channels, wikis, blogs, social media, events, conferences, meetings, and any other communication channels used by the project.
 
 **Our open source communities endeavor to:**
 
@@ -8,7 +8,6 @@ This code of conduct applies to all spaces provided by the OpenSource project in
 * Be Respectful: We are committed to encouraging differing viewpoints, accepting constructive criticism and work collaboratively towards decisions that help the project grow. Disrespectful and unacceptable behavior will not be tolerated.
 * Be Collaborative: We are committed to supporting what is best for our community and users. When we build anything for the benefit of the project, we should document the work we do and communicate to others on how this affects their work.
 
-
 **Our Responsibility. As contributors, members, or bystanders we each individually have the responsibility to behave professionally and respectfully at all times. Disrespectful and unacceptable behaviors include, but are not limited to:**
 
 * The use of violent threats, abusive, discriminatory, or derogatory language;
@@ -19,6 +18,7 @@ This code of conduct applies to all spaces provided by the OpenSource project in
 * Publishing private information, such as physical or electronic address, without permission;
 * Other conduct which could reasonably be considered inappropriate in a professional setting;
 * Advocating for or encouraging any of the above behaviors.
-* Enforcement and Reporting Code of Conduct Issues:
+
+**Enforcement and Reporting Code of Conduct Issues:**
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be reported. [Contact us](mailto:opensource-codeofconduct@amazon.com). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances.

RELEASING.md

@@ -1,18 +1,122 @@
 # Releasing
 
 This document outlines the process for releasing Data Prepper.
+It is a guide for maintainers of the Data Prepper project to release a new version.
 
-## Release prerequisites
+## Overview
 
-Be sure you have:
+This document has three broad categories of steps to follow:
 
-* Created a release branch.
-* Updated the version in the release branch.
-* Updated the THIRD-PARTY file.
-* Created the release notes file
-* Created the changelog file
+1. [Release setup](#release-setup)
+2. [Performing a release](#performing-a-release)
+3. [Post-release](#post-release)
 
-## Performing a release
+
+## <a name="release-setup">Release setup</a>
+
+### Branch setup
+
+Data Prepper uses a release branch for releasing.
+The [Developer Guide](docs/developer_guide.md#backporting) discusses this in detail.
+
+The repository has a release branch for a major/minor version.
+Patch versions will continue on the same branch.
+For example, Data Prepper `2.6.0` was released from the `2.6` branch.
+Additionally, Data Prepper `2.6.1` and `2.6.2` were also released from the `2.6` branch.
+
+If you are creating a new major/minor release, then you will need to create the release branch.
+Use GitHub to create a new branch.
+
+Steps:
+1. Go to the [branches page here](https://github.com/opensearch-project/data-prepper/branches).
+2. Select `New branch`
+3. Name the branch `{major}.{minor}`. For example, `2.6`.
+4. Select `main` as the source.
+
+Create a backport label for the release branch.
+
+Steps:
+1. Go to the [labels page here](https://github.com/opensearch-project/data-prepper/labels).
+2. Select `New label`
+3. Name the branch `backport {major}.minor`. For example, `backport 2.6`
+
+### Update versions
+
+The Data Prepper version is defined in the [`gradle.properties`](https://github.com/opensearch-project/data-prepper/blob/main/gradle.properties)  file.
+We must update this whenever we create a new release.
+We will need two PRs to update it.
+
+#### Update on release branch
+
+For the current release branch, update the version to the new version.
+You may either need to update by removing the `-SNAPSHOT` or by updating the patch version.
+
+For example, when we released `2.6.2`, the property was set as `version=2.6.2`.
+You can see the [exact commit here](https://github.com/opensearch-project/data-prepper/blob/2.6.2/gradle.properties#L8).
+
+Create a PR that targets the release branch with the change.
+Do not create a PR to `main` for this.
+
+#### Update on the main branch
+
+If you have just created a release branch, you should also create a PR on the `main` branch to bump the version.
+
+For example, if you have started the `2.7` branch, you will need to update the `main` branch from `2.6.0-SNAPSHOT` to `2.7.0-SNAPSHOT`.
+
+#### Update the version of DataPrepperVersion
+
+If you have just created a release branch, you should also create a PR on the `main` branch to bump the version in `DataPrepperVersion`.
+
+Steps:
+1. Modify the `DataPrepperVersion` class to update `CURRENT_VERSION` to the next version.
+2. Create a PR targeting `main`
+
+Note: This step can be automated through [#4877](https://github.com/opensearch-project/data-prepper/issues/4877).
+
+### Update the THIRD-PARTY file
+
+We should update the `THIRD-PARTY` file for every release.
+Data Prepper has a GitHub action that will generate this and create a PR with the updated file.
+
+Steps:
+* Go the [Third Party Generate action](https://github.com/opensearch-project/data-prepper/actions/workflows/third-party-generate.yml)
+* Select `Run workflow`
+* Choose the branch you are releasing. e.g. `2.6`
+* Press `Run workflow`
+* Wait for a new PR to be created
+* Spot check the PR, approve and merge
+
+
+### Prepare release notes
+
+Prepare release notes and check them into the `main` branch in the [`release-notes` directory](https://github.com/opensearch-project/data-prepper/tree/main/release/release-notes).
+The format for the release notes file is `data-prepper.release-notes.{major}.{minor}.{patch}.md`.
+
+You can use a script to help you generate these.
+See the [README](release/script/release-notes/README.md) for the script for instructions.
+
+Once the release notes are ready, create a PR to merge them into `main`.
+Also tag this with the `backport {major}.{minor}` to create a PR that you can merge into your release branch.
+
+### Create changelog
+
+You can create a changelog using [git-release-notes](https://github.com/ariatemplates/git-release-notes).
+
+```
+git fetch upstream
+git switch {major}.{minor}
+git fetch upstream --tags
+git pull
+git-release-notes {previousMajor}.{previousMinor}.{previousPatch}..HEAD markdown > release/release-notes/data-prepper.change-log-{major}.{minor}.{patch}.md
+git switch main
+```
+
+Once the change log ready, create a PR to merge it into `main`.
+Also tag this with the `backport {major}.{minor}` to create a PR that you can merge into your release branch.
+
+
+
+## <a name="performing-a-release">Performing a release</a>
 
 This section outlines how to perform a Data Prepper release using GitHub Actions and the OpenSearch build infrastructure.
 The audience for this section are Data Prepper maintainers.
@@ -27,14 +131,14 @@ Select the "Run workflow" option from the GitHub Actions UI. GitHub will prompt
 #### Use workflow for
 
 Select the release branch which you are releasing for.
-Typically, this will be a branch such as `2.4`.
+Typically, this will be a branch such as `2.6`.
 However, you may select `main` for testing.
 
 #### Whether to create major tag of docker image or not.
 
 This will create a tag such as `2` which points to this version
 
-All releases have a full version tag. For example, `2.4.0`.
+All releases have a full version tag. For example, `2.6.0`.
 The latest release on a major series can also have a tag applied such as `2`.
 Check this option if you are releasing the latest version withing that series of major versions.
 This value can be true for old major series as well such as the 1.x series.
@@ -69,3 +173,28 @@ You can also deny a release by using _deny_ or _denied_ in the comment.
 ### Further details
 
 For more details on the release build, or to setup your own GitHub repository, see [release/README.md](release/README.md).
+
+
+
+## <a name="post-release">Post release</a>
+
+After the release, there are a few other steps to clean up the repository.
+
+### Update the release notes
+
+The release process will have created a draft release for the new version.
+The next step is to update the draft release with the release notes created before the release.
+
+Steps:
+* Go to the [releases page](https://github.com/opensearch-project/data-prepper/releases)
+* Find the new draft release. It should be at the top.
+* Replace the auto-generated release notes with the release notes created previous to the release.
+* Release it
+
+### Close the GitHub milestone
+
+Steps:
+* Go to the [milestones](https://github.com/opensearch-project/data-prepper/milestones) page.
+* Find the milestone for the release.
+* Make sure there are no issues. If there are any triage them by closing, or changing the milestone.
+* Click the "Close" button

TRIAGING.md

@@ -0,0 +1,73 @@
+<img src="https://raw.githubusercontent.com/opensearch-project/data-prepper/main/docs/images/DataPrepper_auto.svg" height="64px" alt="Data Prepper">
+
+The Data Prepper maintainers seek to promote an inclusive and engaged community of contributors.
+In order to facilitate this, weekly triage meetings are open to all and attendance is encouraged for anyone who hopes to contribute, discuss an issue, or learn more about the project.
+To learn more about contributing to the Data Prepper project visit the [Contributing](./CONTRIBUTING.md) documentation.
+
+### Do I need to attend for my issue to be addressed/triaged?
+
+Attendance is not required for your issue to be triaged or addressed.
+All new issues are triaged weekly.
+
+### What happens if my issue does not get covered this time?
+
+Each meeting we seek to address all new issues.
+However, should we run out of time before your issue is discussed, you are always welcome to attend the next meeting or to follow up on the issue post itself.
+
+### How do I join the triage meeting?
+
+Meetings are hosted regularly Tuesdays at 2:30 PM US Central Time (12:30 PM Pacific Time) and can be joined via the links posted on the [OpenSearch Meetup Group](https://www.meetup.com/opensearch/events/) list of events.
+The event will be titled `Data Prepper Triage Meeting`.
+
+After joining the video meeting, you can enable your video / voice to join the discussion.
+If you do not have a webcam or microphone available, you can still join in via the text chat.
+
+If you have an issue you'd like to bring forth please consider getting a link to the issue so it can be presented to everyone in the meeting.
+
+### Is there an agenda for each week?
+
+Meetings are 30 minutes and structured as follows:
+
+1. Initial Gathering: As we gather, feel free to turn on video and engage in informal and open-to-all conversation. A volunteer Data Prepper maintainer will share the [Data Prepper Tracking Board](https://github.com/orgs/opensearch-project/projects/82/) and proceed.
+2. Announcements: We will make any announcements at the beginning, if necessary.
+3. Untriaged issues: We will review all untriaged [issues](https://github.com/orgs/opensearch-project/projects/82/views/6) for the Data Prepper repository. If you have an item here, you may spend a few minutes to explain your request.
+4. Member Requests: Opportunity for any meeting member to ask for consideration of an issue or pull request.
+5. Release review: If time permits, and we find it necessary, we will review [items for the current release](https://github.com/orgs/opensearch-project/projects/82/views/14).
+6. Follow-up issues: If time permits, we will review the [follow up items](https://github.com/orgs/opensearch-project/projects/82/views/18).
+7. Open Discussion: If time permits, allow for members of the meeting to surface any topics without issues filed or pull request created.
+
+### Do I need to have already contributed to the project to attend a triage meeting?
+
+No, all are welcome and encouraged to attend.
+Attending the triage meetings is a great way for a new contributor to learn about the project as well as explore different avenues of contribution.
+
+### What if I have follow-up questions on an issue?
+
+If you have an existing issue you would like to discuss, you can always comment on the issue itself.
+Alternatively, you are welcome to come to the triage meeting to discuss.
+
+### Is this meeting a good place to get help using Data Prepper?
+
+While we are always happy to help the community, the best resource for usage questions is the [the Data Prepper discussion forum](https://github.com/opensearch-project/data-prepper/discussions) on GitHub.
+
+There you can find answers to many common questions as well as speak with implementation experts and Data Prepper maintainers.
+
+### What are the issue labels associated with triaging?
+
+There are several labels that are particularly important for triaging in Data Prepper:
+
+| Label | When applied | Meaning |
+| ----- | ------------ | ------- |
+| [untriaged](https://github.com/opensearch-project/data-prepper/labels/untriaged) | When issues are created or re-opened. | Issues labeled as `untriaged` require the attention of the repository maintainers and may need to be prioritized for quicker resolution. It's crucial to keep the count of 'untriaged' labels low to ensure all potential security issues are addressed in a timely manner. |
+| [follow up](https://github.com/opensearch-project/data-prepper/labels/follow%20up) | During triage meetings. | Issues labeled as `follow up` have been triaged. However, the maintainers may need to follow up further on it. This tag lets us triage an issue as not critical, but also be able to come back to it.
+| [help wanted](https://github.com/opensearch-project/data-prepper/labels/help%20wanted) | Anytime. | Issues marked as `help wanted` signal that they are actionable and not the current focus of the project maintainers. Community contributions are especially encouraged for these issues. |
+| [good first issue](https://github.com/opensearch-project/data-prepper/labels/good%20first%20issue) | Anytime. | Issues labeled as `good first issue` are small in scope and can be resolved with a single pull request. These are recommended starting points for newcomers looking to make their first contributions. |
+
+
+### Is this where I should bring up potential security vulnerabilities?
+
+Due to the sensitive nature of security vulnerabilities, please report all potential vulnerabilities directly by following the steps outlined in the [Security Issue Response Process](https://github.com/opensearch-project/data-prepper/security/policy).
+
+### Who should I contact if I have further questions?
+
+You can always file an [issue](https://github.com/opensearch-project/data-prepper/issues/new/choose) for any question you have about the project.

build-resources.gradle

@@ -14,5 +14,7 @@ ext.coreProjects = [
         project(':data-prepper-plugins'),
         project(':data-prepper-test-common'),
         project(':data-prepper-test-event'),
-        project(':data-prepper-plugin-framework')
-]
+        project(':data-prepper-plugin-framework'),
+        project(':data-prepper-plugin-schema'),
+        project(':data-prepper-plugin-schema-cli')
+]

build.gradle

@@ -69,7 +69,7 @@ subprojects {
         }
     }
     dependencies {
-        implementation platform('com.fasterxml.jackson:jackson-bom:2.16.1')
+        implementation platform('com.fasterxml.jackson:jackson-bom:2.17.2')
         implementation platform('org.eclipse.jetty:jetty-bom:9.4.53.v20231009')
         implementation platform('io.micrometer:micrometer-bom:1.10.5')
         implementation libs.guava.core
@@ -79,12 +79,24 @@ subprojects {
         testImplementation testLibs.hamcrest
         testImplementation testLibs.awaitility
         constraints {
+            implementation('dnsjava:dnsjava') {
+                version {
+                    require '3.6.1'
+                }
+                because 'Fixes CVE-2023-39410.'
+            }
             implementation('org.apache.avro:avro') {
                 version {
                     require '1.11.3'
                 }
                 because 'Fixes CVE-2023-39410.'
             }
+            implementation('org.apache.commons:commons-configuration2') {
+                version {
+                    require '2.11.0'
+                }
+                because 'Fixes CVE-2024-29131 and CVE-2024-29133.'
+            }
             implementation('org.apache.httpcomponents:httpclient') {
                 version {
                     require '4.5.14'
@@ -220,12 +232,19 @@ subprojects {
                 }
             } else if (details.requested.group == 'log4j' && details.requested.name == 'log4j') {
                 details.useTarget group: 'org.apache.logging.log4j', name: 'log4j-1.2-api', version: '2.17.1'
+            } else if (details.requested.group == 'org.bouncycastle' && details.requested.name.endsWith('-jdk15on')) {
+                def java8Name = details.requested.name.replace('-jdk15on', '-jdk18on')
+                details.useTarget group: 'org.bouncycastle', name: java8Name, version: libs.bouncycastle.bcprov.get().version
+                details.because 'Use only the Java 8 artifacts of BouncyCastle'
             }
         }
     }
 
     test {
         useJUnitPlatform()
+        javaLauncher = javaToolchains.launcherFor {
+            languageVersion = JavaLanguageVersion.current()
+        }
         reports {
             junitXml.required
             html.required