DataDog · KSerrania · Aug 18, 2022 · Aug 11, 2022
@@ -118,38 +118,68 @@ deploy_deb_testing-a7_arm64:
     - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID" -m 7 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_7*arm64.deb
     - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID" -m 7 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb
 
-deploy_rpm_testing-a6:
-  rules:
-    !reference [.on_kitchen_tests_a6]
+.deploy_rpm_testing-a6:
   stage: kitchen_deploy
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS
   tags: ["runner:main"]
-  needs: ["agent_rpm-x64-a6", "tests_rpm-x64-py2", "tests_rpm-x64-py3"]
   variables:
     DD_PIPELINE_ID: $CI_PIPELINE_ID-a6
   before_script:
     - ls $OMNIBUS_PACKAGE_DIR
+
+deploy_rpm_testing-a6_x64:
+  rules:
+    !reference [.on_kitchen_tests_a6]
+  extends:
+    - .deploy_rpm_testing-a6
+  needs: ["agent_rpm-x64-a6", "tests_rpm-x64-py2", "tests_rpm-x64-py3"]
   script:
     - *setup_rpm_signing_key
     - set +x
     - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/6/x86_64/" -a "x86_64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-6.*x86_64.rpm
 
-deploy_rpm_testing-a7:
+deploy_rpm_testing-a6_arm64:
   rules:
-    !reference [.on_default_kitchen_tests_a7]
+    !reference [.on_all_kitchen_builds_a6]
+  extends:
+    - .deploy_rpm_testing-a6
+  needs: ["agent_rpm-arm64-a6", "tests_rpm-arm64-py2", "tests_rpm-arm64-py3"]
+  script:
+    - *setup_rpm_signing_key
+    - set +x
+    - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/6/aarch64/" -a "aarch64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-6.*aarch64.rpm
+
+.deploy_rpm_testing-a7:
   stage: kitchen_deploy
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/gitlab_agent_deploy:$DATADOG_AGENT_BUILDERS
   tags: ["runner:main"]
-  needs: ["agent_rpm-x64-a7", "iot_agent_rpm-x64", "dogstatsd_rpm-x64", "tests_rpm-x64-py3"]
   variables:
     DD_PIPELINE_ID: $CI_PIPELINE_ID-a7
   before_script:
     - ls $OMNIBUS_PACKAGE_DIR
+
+deploy_rpm_testing-a7_x64:
+  rules:
+    !reference [.on_default_kitchen_tests_a7]
+  extends:
+    - .deploy_rpm_testing-a7
+  needs: ["agent_rpm-x64-a7", "iot_agent_rpm-x64", "dogstatsd_rpm-x64", "tests_rpm-x64-py3"]
   script:
     - *setup_rpm_signing_key
     - set +x
     - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/7/x86_64/" -a "x86_64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-7.*x86_64.rpm
 
+deploy_rpm_testing-a7_arm64:
+  rules:
+    !reference [.on_all_kitchen_builds_a7]
+  extends:
+    - .deploy_rpm_testing-a7
+  needs: ["agent_rpm-arm64-a7", "tests_rpm-arm64-py3"]
+  script:
+    - *setup_rpm_signing_key
+    - set +x
+    - echo "$RPM_SIGNING_PASSPHRASE" | rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/7/aarch64/" -a "aarch64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-7.*aarch64.rpm
+
 deploy_suse_rpm_testing-a6:
   rules:
     !reference [.on_kitchen_tests_a6]

@@ -3,6 +3,7 @@
 # Contains jobs which run kitchen tests on the Agent packages.
 
 include:
+  - /.gitlab/kitchen_testing/amazonlinux.yml
   - /.gitlab/kitchen_testing/centos.yml
   - /.gitlab/kitchen_testing/debian.yml
   - /.gitlab/kitchen_testing/suse.yml

@@ -0,0 +1,88 @@
+---
+# FIXME: our current Gitlab version doesn't support importing a file more than once
+# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file
+# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987
+# include:
+#   - /.gitlab/kitchen_common/testing.yml
+
+
+# Kitchen: OSes
+# -------------
+
+.kitchen_os_amazonlinux:
+  variables:
+    KITCHEN_PLATFORM: "amazonlinux"
+  before_script:
+    - cd $DD_AGENT_TESTING_DIR
+    - bash -l tasks/kitchen_setup.sh
+
+# Kitchen: scenarios (os * agent * (cloud + arch))
+# -------------------------------
+
+.kitchen_scenario_amazonlinux_a6_x64:
+  variables:
+    KITCHEN_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15"
+    DEFAULT_KITCHEN_OSVERS: "amazonlinux2-5-10"
+  extends:
+    - .kitchen_agent_a6
+    - .kitchen_os_amazonlinux
+    - .kitchen_ec2
+  needs: ["deploy_rpm_testing-a6_x64"]
+
+.kitchen_scenario_amazonlinux_a7_x64:
+  variables:
+    KITCHEN_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15"
+    DEFAULT_KITCHEN_OSVERS: "amazonlinux2-5-10"
+  extends:
+    - .kitchen_agent_a7
+    - .kitchen_os_amazonlinux
+    - .kitchen_ec2
+  needs: ["deploy_rpm_testing-a7_x64"]
+
+.kitchen_scenario_amazonlinux_a6_arm64:
+  variables:
+    KITCHEN_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15"
+    DEFAULT_KITCHEN_OSVERS: "amazonlinux2-5-10"
+  extends:
+    - .kitchen_agent_a6
+    - .kitchen_os_amazonlinux
+    - .kitchen_ec2_arm64
+  needs: ["deploy_rpm_testing-a6_arm64"]
+
+.kitchen_scenario_amazonlinux_a7_arm64:
+  variables:
+    KITCHEN_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15"
+    DEFAULT_KITCHEN_OSVERS: "amazonlinux2-5-10"
+  extends:
+    - .kitchen_agent_a7
+    - .kitchen_os_amazonlinux
+    - .kitchen_ec2_arm64
+  needs: ["deploy_rpm_testing-a7_arm64"]
+
+  # Kitchen: final test matrix (tests * scenarios)
+# ----------------------------------------------
+
+kitchen_amazonlinux_install_script_agent-a6_x64:
+  extends:
+    - .kitchen_scenario_amazonlinux_a6_x64
+    - .kitchen_test_install_script_agent
+
+kitchen_amazonlinux_install_script_agent-a6_arm64:
+  extends:
+    - .kitchen_scenario_amazonlinux_a6_arm64
+    - .kitchen_test_install_script_agent
+
+kitchen_amazonlinux_install_script_agent-a7_x64:
+  # Run install script test on branches, on a reduced number of platforms
+  rules:
+    !reference [.on_default_kitchen_tests_a7]
+  extends:
+    - .kitchen_scenario_amazonlinux_a7_x64
+    - .kitchen_test_install_script_agent
+
+kitchen_amazonlinux_install_script_agent-a7_arm64:
+  rules:
+    !reference [.on_all_kitchen_builds_a7]
+  extends:
+    - .kitchen_scenario_amazonlinux_a7_arm64
+    - .kitchen_test_install_script_agent
@@ -47,42 +47,42 @@
     - .kitchen_agent_a6
     - .kitchen_os_centos_all_non_fips
     - .kitchen_azure_x64
-  needs: ["deploy_rpm_testing-a6"]
+  needs: ["deploy_rpm_testing-a6_x64"]
 
 .kitchen_scenario_centos_all_non_fips_a7:
   extends:
     - .kitchen_agent_a7
     - .kitchen_os_centos_all_non_fips
     - .kitchen_azure_x64
-  needs: ["deploy_rpm_testing-a7"]
+  needs: ["deploy_rpm_testing-a7_x64"]
 
 .kitchen_scenario_centos_6_7_non_fips_a6:
   extends:
     - .kitchen_os_centos_6_7_non_fips
     - .kitchen_agent_a6
     - .kitchen_azure_x64
-  needs: ["deploy_rpm_testing-a6"]
+  needs: ["deploy_rpm_testing-a6_x64"]
 
 .kitchen_scenario_centos_6_7_non_fips_a7:
   extends:
     - .kitchen_agent_a7
     - .kitchen_os_centos_6_7_non_fips
     - .kitchen_azure_x64
-  needs: ["deploy_rpm_testing-a7"]
+  needs: ["deploy_rpm_testing-a7_x64"]
 
 .kitchen_scenario_centos_8_fips_a6:
   extends:
     - .kitchen_agent_a6
     - .kitchen_os_centos_8_fips
     - .kitchen_azure_x64
-  needs: ["deploy_rpm_testing-a6"]
+  needs: ["deploy_rpm_testing-a6_x64"]
 
 .kitchen_scenario_centos_8_fips_a7:
   extends:
     - .kitchen_agent_a7
     - .kitchen_os_centos_8_fips
     - .kitchen_azure_x64
-  needs: ["deploy_rpm_testing-a7"]
+  needs: ["deploy_rpm_testing-a7_x64"]
 
 # Kitchen: final test matrix (tests * scenarios)
 # ----------------------------------------------

@@ -5,6 +5,10 @@ provisioner:
   <% if ENV['KITCHEN_PLATFORM'] == "debian" && ENV['KITCHEN_ARCH'] == "arm64" %>
   download_url: https://packages.chef.io/files/stable/chef/14.15.6/ubuntu/18.04/chef_14.15.6-1_arm64.deb
   product_version: 14.15.6
+  <% elsif ENV['KITCHEN_PLATFORM'] == "amazonlinux" && ENV['KITCHEN_ARCH'] == "arm64" %>
+  # There is no arm64 distribution of Chef 14 for Amazon Linux 2022. Use the CentOS package instead
+  download_url: https://packages.chef.io/files/stable/chef/14.15.6/el/7/chef-14.15.6-1.el7.aarch64.rpm
+  product_version: 14.15.6
   <% else %>
   product_version: <%= ENV['CHEF_VERSION'] ||= '14.12.9' %>
   <% end %>
@@ -18,7 +22,9 @@ provisioner:
 
 driver:
   name: ec2
+  <% if ENV['KITCHEN_EC2_SSH_KEY_ID'] %>
   aws_ssh_key_id: <%= ENV['KITCHEN_EC2_SSH_KEY_ID'] %>
+  <% end %>
   security_group_ids: <%= [ENV['KITCHEN_EC2_SG_IDS']] || ["sg-7fedd80a","sg-46506837"] %>
   region: <%= ENV['KITCHEN_EC2_REGION'] ||= "us-east-1" %>
   instance_type: <%= ENV['KITCHEN_EC2_INSTANCE_TYPE'] ||= 't3.xlarge' %>
@@ -55,6 +61,7 @@ platforms:
 
       windows = platform_name.include?("win")
       sles15 = platform_name.include?("sles-15")
+      al2022 = platform_name.include?("amazonlinux2022")
       windows2008 = windows && platform_name.include?("2008")
 
       if windows
@@ -92,12 +99,39 @@ platforms:
           volume_type: gp2
           volume_size: 40
           delete_on_termination: true
-    <% if allow_rsa_key %>
+    <% if allow_rsa_key || al2022 %>
     user_data: |
       #!/bin/sh
+    <% end %>
+    <% if allow_rsa_key %>
       echo PubkeyAcceptedKeyTypes=+ssh-rsa >> /etc/ssh/sshd_config
       service ssh reload
     <% end %>
+    <% if al2022 %>
+      sudo dnf install -y libxcrypt-compat
+    <% end %>
+  <% if al2022 %>
+  # Add a hook after creating the host, to make sure we wait until the user_data
+  # script has been run.
+  # Snippet taken from the kitchen docs: https://kitchen.ci/docs/reference/lifecycle-hooks/
+  lifecycle:
+    post_create:
+    - local: echo 'Awaiting cloud-init completion'
+    - remote: |
+        declare i=0;
+        declare wait=5;
+        declare timeout=300;
+        while true; do
+          [ -f /var/lib/cloud/instance/boot-finished ] && break;
+          if [ ${i} -ge ${timeout} ]; then
+            echo "Timed out after ${i}s waiting for cloud-init to complete";
+            exit 1;
+          fi;
+          echo "Waited ${i}/${timeout}s for cloud-init to complete, retrying in ${wait} seconds"
+          sleep ${wait};
+          let i+=${wait};
+        done;
+  <% end %>
 
   transport:
     <% if windows %>
@@ -107,8 +141,13 @@ platforms:
     connection_retries: 30
     connection_retry_sleep: 2
     <% end %>
-    <% if sles15 %>
+    <% if sles15 || al2022 %>
+    # The AWS EC2 driver doesn't recognize Amazon Linux 2022 yet,
+    # therefore it doesn't know that it needs to use ec2-user.
     username: ec2-user
     <% end %>
+    <% if ENV['KITCHEN_EC2_SSH_KEY_PATH'] %>
+    ssh_key: <%= ENV['KITCHEN_EC2_SSH_KEY_PATH'] %>
+    <% end %>
 
 <% end %>
@@ -95,11 +95,13 @@
         "ec2": {
             "x86_64": {
                 "amazonlinux2-4-14": "ami-038b3df3312ddf25d",
-                "amazonlinux2-5-10": "ami-033b95fb8079dc481"
+                "amazonlinux2-5-10": "ami-033b95fb8079dc481",
+                "amazonlinux2022-5-15": "ami-0a0cf2b8bc4634fe1"
             },
             "arm64": {
                 "amazonlinux2-4-14": "ami-090230ed0c6b13c74",
-                "amazonlinux2-5-10": "ami-0e449176cecc3e577"
+                "amazonlinux2-5-10": "ami-0e449176cecc3e577",
+                "amazonlinux2022-5-15": "ami-00bca6c8c9d0e6f92"
             }
         }
     },

@@ -14,32 +14,32 @@ if [ -f "$(pwd)/ssh-key.pub" ]; then
   rm ssh-key.pub
 fi
 
-ssh-keygen -f "$(pwd)/ed25519-key" -P "" -a 100 -t ed25519
-KITCHEN_ED25519_SSH_KEY_PATH="$(pwd)/ed25519-key"
-export KITCHEN_ED25519_SSH_KEY_PATH
+# in docker we cannot interact to do this so we must disable it
+mkdir -p ~/.ssh
+[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
 
-# show that the ed25519 ssh key is there
-echo "$(pwd)/ed25519-key"
-echo "$KITCHEN_ED25519_SSH_KEY_PATH"
+if [ "$KITCHEN_PROVIDER" == "azure" ]; then
+  # Generating SSH keys to connect to Azure VMs
 
-ssh-keygen -f "$(pwd)/rsa-key" -P "" -t rsa -b 2048
-KITCHEN_RSA_SSH_KEY_PATH="$(pwd)/rsa-key"
-export KITCHEN_RSA_SSH_KEY_PATH
+  ssh-keygen -f "$(pwd)/ed25519-key" -P "" -a 100 -t ed25519
+  KITCHEN_ED25519_SSH_KEY_PATH="$(pwd)/ed25519-key"
+  export KITCHEN_ED25519_SSH_KEY_PATH
 
-# show that the rsa ssh key is there
-echo "$(pwd)/rsa-key"
-echo "$KITCHEN_RSA_SSH_KEY_PATH"
+  # show that the ed25519 ssh key is there
+  ls "$(pwd)/ed25519-key"
 
-# start the ssh-agent and add the keys
-eval "$(ssh-agent -s)"
-ssh-add "$KITCHEN_RSA_SSH_KEY_PATH"
-ssh-add "$KITCHEN_ED25519_SSH_KEY_PATH"
+  ssh-keygen -f "$(pwd)/rsa-key" -P "" -t rsa -b 2048
+  KITCHEN_RSA_SSH_KEY_PATH="$(pwd)/rsa-key"
+  export KITCHEN_RSA_SSH_KEY_PATH
 
-# in docker we cannot interact to do this so we must disable it
-mkdir -p ~/.ssh
-[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
+  # show that the rsa ssh key is there
+  ls "$(pwd)/rsa-key"
+
+  # start the ssh-agent and add the keys
+  eval "$(ssh-agent -s)"
+  ssh-add "$KITCHEN_RSA_SSH_KEY_PATH"
+  ssh-add "$KITCHEN_ED25519_SSH_KEY_PATH"
 
-if [ "$KITCHEN_PROVIDER" == "azure" ]; then
   # Setup the azure credentials, grabbing them from AWS if they do not exist in the environment already
   # If running locally, they should be imported into the environment
 
@@ -82,6 +82,22 @@ if [ "$KITCHEN_PROVIDER" == "azure" ]; then
 
 elif [ "$KITCHEN_PROVIDER" == "ec2" ]; then
   echo "using ec2 kitchen provider"
+
+  # Setup the AWS credentials: grab the ED25519 ssh key that is needed to connect to Amazon Linux 2022 instances
+  # See: https://github.com/test-kitchen/kitchen-ec2/issues/588
+  # Note: this issue happens even when allowing RSA keys in the ssh service of the remote host (which was the fix we did for Ubuntu 22.04),
+  # therefore using the auto-generated SSH key is not possible at all.
+
+  # These should not be printed out
+  set +x
+  if [ -z ${KITCHEN_EC2_SSH_KEY_ID+x} ]; then
+    KITCHEN_EC2_SSH_KEY_ID="datadog-agent-kitchen"
+    export KITCHEN_EC2_SSH_KEY_ID
+    KITCHEN_EC2_SSH_KEY_PATH="$(pwd)/aws-ssh-key"
+    export KITCHEN_EC2_SSH_KEY_PATH
+    aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.aws_ec2_kitchen_ssh_key --with-decryption --query "Parameter.Value" --out text > $KITCHEN_EC2_SSH_KEY_PATH
+  fi
+  set -x
 fi
 
 # Generate a password to use for the windows servers