Build tests for embedded Chef-Server in Automate

.expeditor/automate_build.sh

@@ -0,0 +1,85 @@
+#!/bin/bash -e
+
+export ORIGIN=chef
+export HAB_ORIGIN=chef
+export HAB_LICENSE=accept-no-persist
+export OCTOKIT_ACCESS_TOKEN
+export CHEF_LICENSE="accept-no-persist"
+export CI=true
+export HAB_ORIGIN_KEYS=chef
+export HAB_STUDIO_SECRET_HAB_FEAT_IGNORE_LOCAL=false
+export HAB_FEAT_IGNORE_LOCAL=false
+export HAB_STUDIO_HOST_ARCH=x86_64-linux
+export HAB_FEAT_OFFLINE_INSTALL=true
+
+curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash
+
+sudo -E hab pkg install core/ruby
+export PATH
+PATH="$(hab pkg path core/ruby)/bin:$PATH"
+sudo -E "$(hab pkg path core/ruby)"/bin/gem install toml
+
+export JOB_TEMP_ROOT
+JOB_TEMP_ROOT=$(mktemp -d /tmp/job-root-XXXXXX)
+export HAB_CACHE_KEY_PATH
+HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys"
+
+echo "--- :key: Generating fake origin key"
+hab license accept
+hab origin key generate
+
+for pkg_name in `echo "bookshelf chef-server-ctl oc-id oc_bifrost oc_erchef openresty-noroot"`
+do
+  echo "generating package for $pkg_name"
+  hab pkg build "src/$pkg_name"
+done
+
+./.expeditor/replace.sh "nginx" "src"
+
+echo "generating package for nginx"
+openresty_hart=$(ls -1t results/chef-openresty*.hart | head -1)
+HAB_FEAT_OFFLINE_INSTALL=true HAB_FEAT_IGNORE_LOCAL=false HAB_ORIGIN=chef HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" DO_CHECK=true HAB_BLDR_CHANNEL=dev hab studio run -D "set -e; hab pkg install $openresty_hart; hab pkg build src/nginx"
+
+git clone https://github.com/chef/automate.git
+cd automate
+git checkout kalroy/cs_plan_changes
+
+RESOLVED_RESULTS_DIR=$(realpath results/)
+export DO_CHECK=true
+
+cp ../results/*.hart $HAB_CACHE_KEY_PATH/* results
+
+#cp ../results/*.hart ../results/chef*.pub ../results/chef*.key results
+../.expeditor/replace.sh
+bookshelf_hart=$(ls -1t results/chef-bookshelf*.hart | head -1)
+chef_server_ctl_hart=$(ls -1t results/chef-chef-server-ctl*.hart | head -1)
+nginx=$(ls -1t results/chef-chef-server-nginx*.hart | head -1)
+oc_id=$(ls -1t results/chef-oc_id*.hart | head -1)
+bifrost_hart=$(ls -1t results/chef-oc_bifrost*.hart | head -1)
+erchef_hart=$(ls -1t results/chef-oc_erchef*.hart | head -1)
+
+
+HAB_FEAT_OFFLINE_INSTALL=true HAB_FEAT_IGNORE_LOCAL=false HAB_ORIGIN=chef HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" DO_CHECK=true HAB_BLDR_CHANNEL=dev hab studio run -D "set -e; hab pkg install $bookshelf_hart; hab pkg build components/automate-cs-bookshelf"
+
+HAB_FEAT_OFFLINE_INSTALL=true HAB_FEAT_IGNORE_LOCAL=false HAB_ORIGIN=chef HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" DO_CHECK=true HAB_BLDR_CHANNEL=dev hab studio run -D "set -e; hab pkg install $bifrost_hart; hab pkg build components/automate-cs-oc-bifrost"
+
+HAB_FEAT_OFFLINE_INSTALL=true HAB_FEAT_IGNORE_LOCAL=false HAB_ORIGIN=chef HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" DO_CHECK=true HAB_BLDR_CHANNEL=dev hab studio run -D "set -e; hab pkg install $erchef_hart; hab pkg build components/automate-cs-oc-erchef"
+
+HAB_FEAT_OFFLINE_INSTALL=true HAB_FEAT_IGNORE_LOCAL=false HAB_ORIGIN=chef HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" DO_CHECK=true HAB_BLDR_CHANNEL=dev hab studio run -D "set -e; hab pkg install $oc_id; hab pkg build components/automate-cs-ocid"
+
+HAB_FEAT_OFFLINE_INSTALL=true HAB_FEAT_IGNORE_LOCAL=false HAB_ORIGIN=chef HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" DO_CHECK=true HAB_BLDR_CHANNEL=dev hab studio run -D "set -e; hab pkg install $nginx; hab pkg build components/automate-cs-nginx"
+
+.expeditor/create-manifest.rb
+mv manifest.json results/build.json
+
+HAB_PKG_CHANNEL=unstable NO_PIN_HAB=true .expeditor/create-manifest.rb
+mv manifest.json results/build-habdev.json
+
+echo "after build" `ls -l results`
+
+# we require chef-server hart files also for for next steps.
+# ls results/*.hart | grep -v automate | xargs rm
+
+tar -cvf results.tar results
+gzip results.tar
+buildkite-agent artifact upload results.tar.gz

.expeditor/chef_server.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#===============================================================================
+#Downloading the automate repo
+#===============================================================================
+
+git clone https://github.com/chef/automate.git
+cd automate
+git checkout kalroy/cs_plan_changes
+
+#===============================================================================
+chmod +x ./integration/tests/chef_server.sh
+#running the chef_server.sh script from the automate repo
+integration/run_test integration/tests/chef_server.sh

.expeditor/chef_server_only.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#===============================================================================
+#Downloading the automate repo
+#===============================================================================
+
+git clone https://github.com/chef/automate.git
+cd automate
+git checkout kalroy/cs_plan_changes
+
+#===============================================================================
+chmod +x ./integration/tests/chef_server_only.sh
+#running the chef_server_only.sh script from the automate repo
+integration/run_test integration/tests/chef_server_only.sh

.expeditor/ha_chef_server.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+#===============================================================================
+#Downloading the automate repo
+#===============================================================================
+
+git clone https://github.com/chef/automate.git
+cd automate
+git checkout kalroy/cs_plan_changes
+
+#===============================================================================
+
+#running the ha_chef_server.sh script from the automate repo
+chmod +x ./integration/tests/ha_chef_server.sh
+integration/run_test integration/tests/ha_chef_server.sh

.expeditor/habitat.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -e
+
+# this script builds all the essential habitat packages for running Chef Server
+# additionaly, it exports them as a local docker image
+
+export CHEF_SERVER_SRC='/workdir/src'
+export ORIGIN=chef
+export HAB_LICENSE=accept-no-persist
+# export PACKAGE_NAME=$1
+
+
+curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash
+
+export JOB_TEMP_ROOT
+JOB_TEMP_ROOT=$(mktemp -d /tmp/job-root-XXXXXX)
+export HAB_CACHE_KEY_PATH
+HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys"
+
+echo "--- :key: Generating fake origin key"
+hab license accept
+hab origin key generate
+
+# cd /workdir/src/bookshelf
+echo "generating package for $PACKAGE_NAME"
+hab pkg build "src/$PACKAGE_NAME"
+echo "which pushd " $(which pushd)
+
+pushd results
+pkg_name=$(ls -1t *.hart | head -1)
+popd
+echo pkg_name is $pkg_name
+buildkite-agent artifact upload results/$pkg_name

.expeditor/replace.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+DEFAULT_COMPONENTS="automate-cs-bookshelf automate-cs-nginx automate-cs-oc-bifrost automate-cs-oc-erchef automate-cs-ocid"
+DEFAULT_COMPONENTS_DIR="components"
+tmp_file_name=""
+
+if [[ $# -ge 1 ]]
+then
+  components=$1
+else
+  components=${DEFAULT_COMPONENTS}
+fi
+
+if [[ $# -ge 2 ]]
+then
+  components_dir="$2"
+else
+  components_dir="${DEFAULT_COMPONENTS_DIR}"
+fi
+
+check_hart() {
+    formated=` echo $1 | awk -F'-' -v b=2 -v e=7 '{printf "%s %s ", $(NF-2), $(NF-3); for (i=b;i<= (NF - 4);i++) printf "%s%s", $i, (i< (NF -4) ? "-" : "\n")}'`
+    IFS=' ' read -r timestamp version component <<< "${formated}"
+    echo "timestamp $timestamp, version $version, component $component"
+    component1=`echo $component | tr -d "_\-\""`
+    export ${component1}="$component/$version/$timestamp"
+}
+
+escape_char() {
+  echo $1 | sed 's/\([-]\)/\\\1/g'
+}
+
+make_tmp() {
+  count=0
+  while [ -f "tmp_$count" ]
+  do
+    count=`expr $count + 1`
+  done
+  tmp_file_name="tmp_$count"
+}
+
+for hart in `ls results/*.hart`
+do
+  check_hart $hart
+done
+
+for component in `echo "$components"`
+do
+  plan_file="${components_dir}/$component/habitat/plan.sh"
+  echo "Processing $plan_file"
+  make_tmp
+  tmp_file=${tmp_file_name}
+  cp $plan_file "$tmp_file"
+  sed -n '/pkg_deps/,/)/p' < "$plan_file" | grep "/" |
+  while read dep 
+  do
+    echo "processing $dep in $plan_file"
+    dep_component=`echo $dep | awk -F'/' '{print $2}' | tr -d "_\-\""`
+    echo "dep_component = ${dep_component}, value = ${!dep_component}"
+    if [[ -n "${!dep_component}" ]]
+    then
+      IFS='/' read -r CScomponent version timestamp <<< "${!dep_component}"
+      echo "timestamp $timestamp, version $version, component $CScomponent"
+      CScomponent1=`escape_char $CScomponent`
+      echo "sed \"/\/$CScomponent1/ s|\/$CScomponent1.*|$CScomponent1/$version/$timestamp|\""
+      sed -i "/pkg_deps/,/)/ s|\/$CScomponent1.*|/$CScomponent1/$version/$timestamp\"|" "${tmp_file}"
+    fi
+  done
+  cp $tmp_file $plan_file
+  rm $tmp_file
+  echo "=================================="
+  echo "file content after replace"
+  cat $plan_file
+  echo "=================================="
+done
+
+exit 0

.expeditor/verify.pipeline.yml

@@ -4,7 +4,7 @@ expeditor:
     - vendor
   defaults:
     buildkite:
-      timeout_in_minutes: 30
+      timeout_in_minutes: 90
 
 steps:
 
@@ -131,44 +131,77 @@ steps:
           - LUALIB=~/.luarocks/lib/lua/5.2
           - COMPONENT=src/bookshelf
 
-# #########################################################################
-#   # Pedant Tests
-# #########################################################################
+- label: automate_build
+  command:
+      - .expeditor/automate_build.sh
+  env:
+      ALLOW_LOCAL_PACKAGES: true
+      HAB_STUDIO_SUP: false
+      HAB_NONINTERACTIVE: true
+  expeditor:
+    secrets:
+        HAB_STUDIO_SECRET_GITHUB_TOKEN:
+          account: github/chef
+          field: token
+        OPENSEARCH_ROOT_CA_PEM:
+          path: secret/a2/a2ha/opensearch
+          field: root-ca.pem
+        OPENSEARCH_ADMIN_PEM:
+          path: secret/a2/a2ha/opensearch
+          field: admin.pem
+        OPENSEARCH_ADMIN_KEY_PEM:
+          path: secret/a2/a2ha/opensearch
+          field: admin-key.pem
+        OPENSEARCH_NODE1_PEM:
+          path: secret/a2/a2ha/opensearch
+          field: node1.pem
+        OPENSEARCH_NODE1_KEY_PEM:
+          path: secret/a2/a2ha/opensearch
+          field: node1-key.pem
+    executor:
+        linux:
+          privileged: true
+
+# waiting for the automate build to finish before running the chef server tests
+- wait
+
 
-- label: With chef-zero
+- label: "chef server"
   command:
-    - /workdir/scripts/bk_tests/bk_install.sh
-    - gem uninstall bundler -v '>= 2.0.1' -a
-    - cp scripts/bk_tests/chef_zero-Gemfile oc-chef-pedant/Gemfile
-    - bundle install --jobs=3 --retry=3 --path=/workdir/vendor/bundle
-    - cd /workdir/oc-chef-pedant
-    - bundle exec rake chef_zero_spec
+    - .expeditor/chef_server.sh
+  timeout_in_minutes: 30 # longer timeout for chef-server
   expeditor:
     executor:
-      docker:
-        image: "chefes/a1-buildkite"
-        environment:
-          - LUALIB=~/.luarocks/lib/lua/5.2
-          - USE_OMNIBUS_FILES=0
-          - PEDANT_OPTS="--skip-oc_id"
-          - BUNDLE_GEMFILE=/workdir/oc-chef-pedant/Gemfile
+      linux:
+        privileged: true
+    secrets:
+      A2_LICENSE:
+        path: secret/a2/license
+        field: license
 
-- label: With ChefFS=1
+- label: "chef server only"
   command:
-    - /workdir/scripts/bk_tests/bk_install.sh
-    - gem install bundler -v 2.2.4
-    - gem uninstall bundler -v '> 2.2.4' -a
-    - cp scripts/bk_tests/chef_zero-Gemfile oc-chef-pedant/Gemfile
-    - bundle install --jobs=3 --retry=3 --path=/workdir/vendor/bundle
-    - cd /workdir/oc-chef-pedant
-    - bundle exec rake chef_zero_spec
+    - .expeditor/chef_server_only.sh
+  timeout_in_minutes: 20
   expeditor:
     executor:
-      docker:
-        image: "chefes/a1-buildkite"
-        environment:
-          - LUALIB=~/.luarocks/lib/lua/5.2
-          - USE_OMNIBUS_FILES=0
-          - PEDANT_OPTS="--skip-oc_id"
-          - CHEF_FS=1
-          - BUNDLE_GEMFILE=/workdir/oc-chef-pedant/Gemfile
+      linux:
+        privileged: true
+    secrets:
+      A2_LICENSE:
+        path: secret/a2/license
+        field: license
+
+- label: "ha chef server"
+  command:
+    - .expeditor/ha_chef_server.sh
+  timeout_in_minutes: 35
+  expeditor:
+    executor:
+      linux:
+        single-use: true
+        privileged: true
+    secrets:
+      A2_LICENSE:
+        path: secret/a2/license
+        field: license

src/nginx/habitat/plan.sh

@@ -5,7 +5,7 @@ pkg_license=('Apache-2.0')
 pkg_deps=(
   core/curl
   core/libossp-uuid
-  ${HAB_ORIGIN:-chef}/openresty-noroot
+  "${HAB_ORIGIN:-chef}/openresty-noroot"
 )
 pkg_build_deps=()
 pkg_exposes=(port ssl-port)