Manually backports Windows changes from #2601 (#2647)

* [Windows] Replaces `rm -rf` with `remove.js`
* [dev/build] Facilitates using zipped archives of node releases
* [div/build] Introduces Windows as a platform
* [dev/build] Corrects cleaning of platform specific build artifacts
* [dev/build] Enhances the cleanup of downloaded node binaries
* [opensearch-dashboards-plugin] Removes prohibition on installing plugins on Windows
* [@osd/utils] Adds a method to standardize path references across platforms
* [dev/build] Standardize paths in tests
* [@osd/telemetry-tools] Normalizes the collection paths
* [plugins/url-forwarding] Fixes the usage of `normalizePath` across node and browser
* [@osd/pm] Allows symlink created for tests without elevated privileges on Windows
* [@osd/opensearch] Allows usage of Windows snapshots in integration tests
* [@osd/opensearch] Employs absolute paths in tests
* [@osd/apm-config-loader] Employs absolute paths in tests
* [core/server] Employs absolute and posix references to paths
* [@osd/optimizer] Standardize paths in tests
* [@osd/tests] Employs absolute paths in tests
* [@osd/pm] Standardize paths in project trees
* [plugins/telemetry] Accommodates the inability of Windows to create unreadable files for testing
* [@osd/config-schema] Normalize paths in tests
* [@osd/plugin-helpers] Standardize paths in tests
* [@osd/plugin-generator] Standardize paths in tests
* [Windows] Update changelog

backport PR: #2601

Signed-off-by: Miki <miki@amazon.com>

CHANGELOG.md

@@ -23,6 +23,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 * [Multi DataSource] Add data source config to opensearch-dashboards-docker ([#2557](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2557))
 * [Multi DataSource] Make text content dynamically translated & update unit tests ([#2570](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2570))
 * [Vis Builder] Change classname prefix wiz to vb ([#2581](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2581/files))
+* [Windows] Facilitate building and running OSD and plugins on Windows platforms ([#2601](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2601))
 
 ### 🐛 Bug Fixes
 * [Vis Builder] Fixes auto bounds for timeseries bar chart visualization ([2401](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2401))

package.json

@@ -65,7 +65,7 @@
     "makelogs": "node scripts/makelogs",
     "uiFramework:compileCss": "cd packages/osd-ui-framework && yarn compileCss",
     "osd:watch": "node scripts/opensearch_dashboards --dev --logging.json=false",
-    "build:types": "rm -rf ./target/types && tsc --p tsconfig.types.json",
+    "build:types": "node scripts/remove.js ./target/types && tsc --p tsconfig.types.json",
     "docs:acceptApiChanges": "node --max-old-space-size=6144 scripts/check_published_api_changes.js --accept",
     "osd:bootstrap": "node scripts/build_ts_refs && node scripts/register_git_hook",
     "spec_to_console": "node scripts/spec_to_console",
@@ -410,6 +410,7 @@
     "mutation-observer": "^1.0.3",
     "ngreact": "^0.5.1",
     "nock": "12.0.3",
+    "node-stream-zip": "^1.15.0",
     "normalize-path": "^3.0.0",
     "nyc": "^15.1.0",
     "pixelmatch": "^5.1.0",
@@ -459,4 +460,4 @@
     "node": "14.20.0",
     "yarn": "^1.21.1"
   }
-}
+}

packages/osd-apm-config-loader/src/utils/get_config_file_paths.test.ts

@@ -40,7 +40,7 @@ describe('getConfigurationFilePaths', () => {
 
     expect(getConfigurationFilePaths(argv)).toEqual([
       resolve(cwd, join('.', 'relative-path')),
-      '/absolute-path',
+      resolve('/absolute-path'),
     ]);
   });
 

packages/osd-config-schema/src/errors/schema_error.test.ts

@@ -28,7 +28,7 @@
  * under the License.
  */
 
-import { relative } from 'path';
+import { relative, sep } from 'path';
 import { SchemaError } from '.';
 
 /**
@@ -37,7 +37,7 @@ import { SchemaError } from '.';
 export const cleanStack = (stack: string) =>
   stack
     .split('\n')
-    .filter((line) => !line.includes('node_modules/') && !line.includes('internal/'))
+    .filter((line) => !line.includes('node_modules' + sep) && !line.includes('internal/'))
     .map((line) => {
       const parts = /.*\((.*)\).?/.exec(line) || [];
 
@@ -46,7 +46,11 @@ export const cleanStack = (stack: string) =>
       }
 
       const path = parts[1];
-      return line.replace(path, relative(process.cwd(), path));
+      // Cannot use `standardize` from `@osd/utils
+      let relativePath = relative(process.cwd(), path);
+      if (process.platform === 'win32') relativePath = relativePath.replace(/\\/g, '/');
+
+      return line.replace(path, relativePath);
     })
     .join('\n');
 

packages/osd-opensearch-archiver/package.json

@@ -7,8 +7,8 @@
     "devOnly": true
   },
   "scripts": {
-    "osd:bootstrap": "rm -rf target && tsc",
-    "osd:watch": "rm -rf target && tsc --watch"
+    "osd:bootstrap": "node ../../scripts/remove.js target && tsc",
+    "osd:watch": "node ../../scripts/remove.js target && tsc --watch"
   },
   "dependencies": {
     "@osd/dev-utils": "1.0.0",
@@ -17,4 +17,4 @@
   "devDependencies": {
     "@types/elasticsearch": "^5.0.33"
   }
-}
+}

packages/osd-opensearch/src/artifact.js

@@ -186,12 +186,14 @@ async function getArtifactSpecForSnapshotFromUrl(urlVersion, log) {
   // issue: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/475
   const platform = process.platform === 'win32' ? 'windows' : process.platform;
   const arch = process.arch === 'arm64' ? 'arm64' : 'x64';
-  if (platform !== 'linux') {
-    throw createCliError(`Snapshots are only available for Linux`);
+  const extension = process.platform === 'win32' ? 'zip' : 'tar.gz';
+
+  if (platform !== 'linux' && platform !== 'windows') {
+    throw createCliError(`Snapshots are only available for Linux and Windows`);
   }
 
   const latestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}-SNAPSHOT`;
-  const latestFile = `opensearch-min-${desiredVersion}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`;
+  const latestFile = `opensearch-min-${desiredVersion}-SNAPSHOT-${platform}-${arch}-latest.${extension}`;
   const completeLatestUrl = `${latestUrl}/${latestFile}`;
 
   let { abc, resp } = await verifySnapshotUrl(completeLatestUrl, log);

packages/osd-opensearch/src/artifact.test.js

@@ -163,10 +163,10 @@ describe('Artifact', () => {
         });
       });
 
-      it('should throw when on a non-Linux platform', async () => {
+      it('should throw when on a non-Linux or non-Windows platform', async () => {
         Object.defineProperties(process, {
           platform: {
-            value: 'win32',
+            value: 'darwin',
           },
           arch: {
             value: ORIGINAL_ARCHITECTURE,

packages/osd-opensearch/src/utils/decompress.test.js

@@ -52,8 +52,8 @@ beforeEach(() => {
   fs.copyFileSync(path.resolve(fixturesFolder, 'snapshot.tar.gz'), tarGzSnapshot);
 });
 
-afterEach(() => {
-  del.sync(tmpFolder, { force: true });
+afterEach(async () => {
+  await del(tmpFolder, { force: true });
 });
 
 test('zip strips root directory', async () => {

packages/osd-opensearch/src/utils/extract_config_files.test.js

@@ -36,6 +36,7 @@ jest.mock('fs', () => ({
 
 const { extractConfigFiles } = require('./extract_config_files');
 const fs = require('fs');
+const path = require('path');
 
 afterEach(() => {
   jest.clearAllMocks();
@@ -55,7 +56,7 @@ test('copies file', () => {
   extractConfigFiles(['path=/data/foo.yml'], '/opensearch');
 
   expect(fs.readFileSync.mock.calls[0][0]).toEqual('/data/foo.yml');
-  expect(fs.writeFileSync.mock.calls[0][0]).toEqual('/opensearch/config/foo.yml');
+  expect(fs.writeFileSync.mock.calls[0][0]).toEqual(path.resolve('/opensearch/config/foo.yml'));
 });
 
 test('ignores file which does not exist', () => {

packages/osd-opensearch/src/utils/find_most_recently_changed.test.js

@@ -27,6 +27,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+const path = require('path');
 
 jest.mock('fs', () => ({
   statSync: jest.fn().mockImplementation((path) => {
@@ -57,7 +58,7 @@ const { findMostRecentlyChanged } = require('./find_most_recently_changed');
 
 test('returns newest file', () => {
   const file = findMostRecentlyChanged('/data/*.yml');
-  expect(file).toEqual('/data/newest.yml');
+  expect(file).toEqual(path.resolve('/data/newest.yml'));
 });
 
 afterAll(() => {

packages/osd-optimizer/src/common/bundle.test.ts

@@ -28,6 +28,7 @@
  * under the License.
  */
 
+import { resolve } from 'path';
 import { Bundle, BundleSpec, parseBundles } from './bundle';
 
 jest.mock('fs');
@@ -88,13 +89,16 @@ it('provides the module count from the cache', () => {
 
 it('parses bundles from JSON specs', () => {
   const bundles = parseBundles(JSON.stringify([SPEC]));
+  let expectedCachePath = resolve('/foo/bar/target/.osd-optimizer-cache');
+  // Cannot use `standardize` from `@osd/util` due to mocking of fs
+  if (process?.platform === 'win32') expectedCachePath = expectedCachePath.replace(/\\/g, '\\\\');
 
   expect(bundles).toMatchInlineSnapshot(`
     Array [
       Bundle {
         "banner": undefined,
         "cache": BundleCache {
-          "path": "/foo/bar/target/.osd-optimizer-cache",
+          "path": "${expectedCachePath}",
           "state": undefined,
         },
         "contextDir": "/foo/bar",

packages/osd-optimizer/src/optimizer/get_changes.test.ts

@@ -28,9 +28,12 @@
  * under the License.
  */
 
+import path from 'path';
+
 jest.mock('execa');
 
 import { getChanges } from './get_changes';
+import { standardize } from '@osd/dev-utils';
 
 const execa: jest.Mock = jest.requireMock('execa');
 
@@ -56,12 +59,16 @@ it('parses git ls-files output', async () => {
     };
   });
 
+  const rootPath = path.resolve('/foo/bar/x/osd-optimizer') + path.sep;
+  const srcPath = path.join(rootPath, 'src') + path.sep;
+  const commonPath = path.join(srcPath, 'common') + path.sep;
+
   await expect(getChanges('/foo/bar/x')).resolves.toMatchInlineSnapshot(`
     Map {
-      "/foo/bar/x/osd-optimizer/package.json" => "modified",
-      "/foo/bar/x/osd-optimizer/src/common/bundle.ts" => "modified",
-      "/foo/bar/x/osd-optimizer/src/common/bundles.ts" => "deleted",
-      "/foo/bar/x/osd-optimizer/src/get_bundle_definitions.test.ts" => "deleted",
+      "${standardize(rootPath, false, true)}package.json" => "modified",
+      "${standardize(commonPath, false, true)}bundle.ts" => "modified",
+      "${standardize(commonPath, false, true)}bundles.ts" => "deleted",
+      "${standardize(srcPath, false, true)}get_bundle_definitions.test.ts" => "deleted",
     }
   `);
 });

packages/osd-optimizer/src/optimizer/get_plugin_bundles.test.ts

@@ -31,10 +31,15 @@
 import { createAbsolutePathSerializer } from '@osd/dev-utils';
 
 import { getPluginBundles } from './get_plugin_bundles';
+import path from 'path';
 
 expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo', '<repoRoot>'));
 expect.addSnapshotSerializer(createAbsolutePathSerializer('/output', '<outputRoot>'));
+expect.addSnapshotSerializer(createAbsolutePathSerializer(path.resolve('/output'), '<outputRoot>'));
 expect.addSnapshotSerializer(createAbsolutePathSerializer('/outside/of/repo', '<outsideOfRepo>'));
+expect.addSnapshotSerializer(
+  createAbsolutePathSerializer(path.resolve('/outside/of/repo'), '<outsideOfRepo>')
+);
 
 it('returns a bundle for core and each plugin', () => {
   expect(

packages/osd-plugin-generator/src/integration_tests/generate_plugin.test.ts

@@ -32,13 +32,17 @@ import Path from 'path';
 
 import del from 'del';
 import execa from 'execa';
-import { REPO_ROOT } from '@osd/utils';
-import { createAbsolutePathSerializer } from '@osd/dev-utils';
+import { REPO_ROOT, standardize, createAbsolutePathSerializer } from '@osd/dev-utils';
 import globby from 'globby';
 
-const GENERATED_DIR = Path.resolve(REPO_ROOT, `plugins`);
+// Has to be a posix reference because it is used to generate glob patterns
+const GENERATED_DIR = standardize(Path.resolve(REPO_ROOT, `plugins`), true);
 
-expect.addSnapshotSerializer(createAbsolutePathSerializer());
+expect.addSnapshotSerializer(
+  createAbsolutePathSerializer(
+    process?.platform === 'win32' ? standardize(REPO_ROOT, true) : REPO_ROOT
+  )
+);
 
 beforeEach(async () => {
   await del([`${GENERATED_DIR}/**`, `!${GENERATED_DIR}`, `!${GENERATED_DIR}/.gitignore`], {

packages/osd-plugin-helpers/package.json

@@ -12,7 +12,7 @@
     "plugin-helpers": "bin/plugin-helpers.js"
   },
   "scripts": {
-    "osd:bootstrap": "rm -rf target && tsc",
+    "osd:bootstrap": "node ../../scripts/remove.js && tsc",
     "osd:watch": "tsc --watch"
   },
   "dependencies": {

packages/osd-plugin-helpers/src/integration_tests/build.test.ts

@@ -32,8 +32,12 @@ import Path from 'path';
 import Fs from 'fs';
 
 import execa from 'execa';
-import { REPO_ROOT } from '@osd/utils';
-import { createStripAnsiSerializer, createReplaceSerializer } from '@osd/dev-utils';
+import {
+  REPO_ROOT,
+  standardize,
+  createStripAnsiSerializer,
+  createReplaceSerializer,
+} from '@osd/dev-utils';
 import extract from 'extract-zip';
 import del from 'del';
 import globby from 'globby';
@@ -78,7 +82,7 @@ it('builds a generated plugin into a viable archive', async () => {
   expect(generateProc.all).toMatchInlineSnapshot(`
     " succ 🎉
 
-          Your plugin has been created in plugins/foo_test_plugin
+          Your plugin has been created in ${standardize('plugins/foo_test_plugin', false, true)}
     "
   `);
 
@@ -165,7 +169,7 @@ it('builds a non-semver generated plugin into a viable archive', async () => {
   expect(generateProc.all).toMatchInlineSnapshot(`
     " succ 🎉
 
-          Your plugin has been created in plugins/foo_test_plugin
+          Your plugin has been created in ${standardize('plugins/foo_test_plugin', false, true)}
     "
   `);
 

packages/osd-pm/src/utils/projects.test.ts

@@ -53,7 +53,8 @@ describe('#getProjects', () => {
 
     await promisify(symlink)(
       join(__dirname, '__fixtures__/symlinked-plugins/corge'),
-      join(rootPlugins, 'corge')
+      join(rootPlugins, 'corge'),
+      'junction' // This parameter would only be used on Windows
     );
   });
 

packages/osd-pm/src/utils/projects_tree.ts

@@ -31,6 +31,7 @@
 import chalk from 'chalk';
 import path from 'path';
 
+import { standardize } from '@osd/utils';
 import { Project } from './project';
 
 const projectKey = Symbol('__project');
@@ -117,7 +118,7 @@ function createTreeStructure(tree: IProjectsTree): ITree {
     // `foo/bar/baz` instead.
     if (subtree.children && subtree.children.length === 1) {
       const child = subtree.children[0];
-      const newName = chalk.dim(path.join(dir.toString(), child.name!));
+      const newName = chalk.dim(standardize(path.join(dir.toString(), child.name!), true));
 
       children.push({
         children: child.children,

packages/osd-telemetry-tools/src/tools/ts_parser.ts

@@ -31,7 +31,7 @@
 import * as ts from 'typescript';
 import { createFailError } from '@osd/dev-utils';
 import * as path from 'path';
-import { getProperty, getPropertyValue } from './utils';
+import { getProperty, getPropertyValue, normalizePath } from './utils';
 import { getDescriptor, Descriptor } from './serializer';
 
 export function* traverseNodes(maybeNodes: ts.Node | ts.Node[]): Generator<ts.Node> {
@@ -205,7 +205,7 @@ export function* parseUsageCollection(
   sourceFile: ts.SourceFile,
   program: ts.Program
 ): Generator<ParsedUsageCollection> {
-  const relativePath = path.relative(process.cwd(), sourceFile.fileName);
+  const relativePath = normalizePath(path.relative(process.cwd(), sourceFile.fileName), false);
   if (sourceHasUsageCollector(sourceFile)) {
     for (const node of traverseNodes(sourceFile)) {
       if (isMakeUsageCollectorFunction(node, sourceFile)) {

packages/osd-telemetry-tools/src/tools/utils.ts

@@ -311,6 +311,7 @@ export function difference(actual: any, expected: any) {
   return changes(actual, expected);
 }
 
-export function normalizePath(inputPath: string) {
-  return normalize(path.relative('.', inputPath));
+export function normalizePath(inputPath: string, relativeToRoot: boolean = true) {
+  if (relativeToRoot) return normalize(path.relative('.', inputPath));
+  return normalize(inputPath);
 }

packages/osd-test/src/functional_test_runner/lib/suite_tracker.test.ts

@@ -40,7 +40,7 @@ import { REPO_ROOT } from '@osd/dev-utils';
 import { Lifecycle } from './lifecycle';
 import { SuiteTracker } from './suite_tracker';
 
-const DEFAULT_TEST_METADATA_PATH = join(REPO_ROOT, 'target', 'test_metadata.json');
+const DEFAULT_TEST_METADATA_PATH = resolve(REPO_ROOT, 'target', 'test_metadata.json');
 const MOCK_CONFIG_PATH = join('test', 'config.js');
 const MOCK_TEST_PATH = join('test', 'apps', 'test.js');
 const ENVS_TO_RESET = ['TEST_METADATA_PATH'];