[CI] Validate CDN assets in typescript (#181733)

## Summary
Replaces bash/curl validation with Typescript, to see if it's any faster
or more reliable

This was a successful run:
https://buildkite.com/elastic/kibana-artifacts-container-image/builds/11689#018f1a82-bf0f-4cd4-ac34-4f822d73a17b
- the validation seems to have taken place in 5 seconds.

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -99,16 +99,9 @@ gsutil -m cp -r "$CDN_ASSETS_FOLDER/*" "gs://$GCS_SA_CDN_BUCKET/$GIT_ABBREV_COMM
 gcloud auth revoke "$GCS_SA_CDN_EMAIL"
 
 echo "--- Validate CDN assets"
-(
-  shopt -s globstar
-  cd $CDN_ASSETS_FOLDER
-  for CDN_ASSET in **/*; do
-    if [[ -f "$CDN_ASSET" ]]; then
-      echo -n "Testing $CDN_ASSET..."
-      curl --retry 10 --retry-max-time 600 --connect-timeout 120 --max-time 120 -I --write-out '%{http_code}\n' --fail --silent --output /dev/null "$GCS_SA_CDN_URL/$CDN_ASSET"
-    fi
-  done
-)
+ts-node "$(git rev-parse --show-toplevel)/.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts" \
+  "$GCS_SA_CDN_URL" \
+  "$CDN_ASSETS_FOLDER"
 
 echo "--- Upload archives"
 buildkite-agent artifact upload "kibana-$BASE_VERSION-linux-x86_64.tar.gz"

.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as glob from 'glob';
+import axios from 'axios';
+
+const CDN_URL_PREFIX = process.argv[2];
+const CDN_ASSETS_FOLDER = process.argv[3];
+
+async function main() {
+  const allAssets = glob.sync('**/*', { cwd: CDN_ASSETS_FOLDER, nodir: true });
+  const totalAssetCount = allAssets.length;
+  let assetsProcessed = 0;
+  let assetsFound = 0;
+
+  const batchSize = 50;
+
+  console.log(`Starting CDN asset validation for ${totalAssetCount} assets...`);
+
+  while (assetsProcessed < totalAssetCount) {
+    const batch = allAssets.slice(assetsProcessed, assetsProcessed + batchSize);
+    const results = await Promise.all(batch.map((url) => headAssetUrlWithRetry(url)));
+    assetsProcessed += results.length;
+    results.forEach((result) => {
+      if (result.status === 200) {
+        console.log(`Testing ${result.assetPath}...${result.status}`);
+        assetsFound++;
+      } else {
+        console.error(`Testing ${result.assetPath}...${result.status} (${result.testUrl})`);
+      }
+    });
+  }
+
+  return {
+    totalAssetCount,
+    assetsProcessed,
+    assetsFound,
+    assetsNotFound: totalAssetCount - assetsFound,
+  };
+}
+
+async function headAssetUrl(assetPath: string) {
+  const testUrl = `${CDN_URL_PREFIX}/${assetPath}`;
+  const response = await axios.head(testUrl, {
+    timeout: 1000,
+  });
+  return {
+    status: response.status,
+    testUrl,
+    assetPath,
+  };
+}
+
+async function headAssetUrlWithRetry(
+  assetPath: string,
+  retries = 5
+): Promise<{
+  status: number;
+  testUrl: string;
+  assetPath: string;
+}> {
+  const result = await headAssetUrl(assetPath);
+  if (result.status === 200) {
+    return result;
+  } else if (retries > 0) {
+    console.log(`Retrying ${assetPath}...(retries left: ${retries})`);
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    return headAssetUrlWithRetry(assetPath, retries - 1);
+  } else {
+    return {
+      status: result.status || 0,
+      testUrl: result.testUrl,
+      assetPath,
+    };
+  }
+}
+
+main()
+  .then(({ totalAssetCount, assetsNotFound }) => {
+    if (assetsNotFound) {
+      console.error(`Couldn't find ${assetsNotFound} assets on CDN.`);
+      process.exit(1);
+    } else {
+      console.log(`All ${totalAssetCount} found on CDN.`);
+      process.exit(0);
+    }
+  })
+  .catch((error) => {
+    console.error(error.message);
+    process.exit(1);
+  });