fix: use fast-glob for resolving projects (#1494)

It takes about 30 seconds for the `glob` stuff to run any command in my project. This PR fixes that
ipfs · Apr 25, 2024 · c45c922 · c45c922
1 parent 8974445
commit c45c922
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 155 deletions.
diff --git a/package.json b/package.json
@@ -283,7 +283,7 @@
     "micromark-extension-gfm-strikethrough": "^2.0.0",
     "micromark-extension-gfm-table": "^2.0.0",
     "micromark-extension-gfm-task-list-item": "^2.0.1",
-    "minimatch": "^9.0.0",
+    "fast-glob": "^3.3.2",
     "mocha": "^10.0.0",
     "npm-package-json-lint": "^7.0.0",
     "nyc": "^15.1.0",

diff --git a/src/check-project/index.js b/src/check-project/index.js
@@ -10,8 +10,8 @@ import prompt from 'prompt'
 import semver from 'semver'
 import yargsParser from 'yargs-parser'
 import {
+  getSubprojectDirectories,
   isMonorepoProject,
-  glob,
   usesReleasePlease
 } from '../utils.js'
 import { checkBuildFiles } from './check-build-files.js'
@@ -114,32 +114,27 @@ async function processMonorepo (projectDir, manifest, branchName, repoUrl, ciFil
 
   const projectDirs = []
 
-  for (const workspace of workspaces) {
-    for await (const subProjectDir of glob('.', workspace, {
-      cwd: projectDir,
-      absolute: true
-    })) {
-      const stat = await fs.stat(subProjectDir)
+  for (const subProjectDir of await getSubprojectDirectories(projectDir, workspaces)) {
+    const stat = await fs.stat(subProjectDir)
 
-      if (!stat.isDirectory()) {
-        continue
-      }
+    if (!stat.isDirectory()) {
+      continue
+    }
 
-      const manfest = path.join(subProjectDir, 'package.json')
+    const manfest = path.join(subProjectDir, 'package.json')
 
-      if (!fs.existsSync(manfest)) {
-        continue
-      }
+    if (!fs.existsSync(manfest)) {
+      continue
+    }
 
-      const pkg = fs.readJSONSync(manfest)
-      const homePage = `${repoUrl}/tree/${branchName}${subProjectDir.substring(projectDir.length)}`
+    const pkg = fs.readJSONSync(manfest)
+    const homePage = `${repoUrl}/tree/${branchName}${subProjectDir.substring(projectDir.length)}`
 
-      console.info('Found monorepo project', pkg.name)
+    console.info('Found monorepo project', pkg.name)
 
-      await processModule(subProjectDir, pkg, branchName, repoUrl, homePage, ciFile, manifest)
+    await processModule(subProjectDir, pkg, branchName, repoUrl, homePage, ciFile, manifest)
 
-      projectDirs.push(subProjectDir)
-    }
+    projectDirs.push(subProjectDir)
   }
 
   await alignMonorepoProjectDependencies(projectDirs)

diff --git a/src/release.js b/src/release.js
@@ -5,7 +5,7 @@ import { execa } from 'execa'
 import fs from 'fs-extra'
 import Listr from 'listr'
 import { calculateSiblingVersion } from './check-project/utils.js'
-import { isMonorepoProject, isMonorepoRoot, hasDocs, glob } from './utils.js'
+import { isMonorepoProject, isMonorepoRoot, hasDocs, getSubprojectDirectories } from './utils.js'
 
 /**
  * @typedef {import("./types").GlobalOptions} GlobalOptions
@@ -151,18 +151,13 @@ async function calculateSiblingVersions (rootDir, workspaces) {
   /** @type {Record<string, string>} */
   const siblingVersions = {}
 
-  for (const workspace of workspaces) {
-    for await (const subProjectDir of glob(rootDir, workspace, {
-      cwd: rootDir,
-      absolute: true
-    })) {
-      const pkg = JSON.parse(fs.readFileSync(path.join(subProjectDir, 'package.json'), {
-        encoding: 'utf-8'
-      }))
-
-      siblingVersions[pkg.name] = calculateSiblingVersion(pkg.version)
-      packageDirs.push(subProjectDir)
-    }
+  for (const subProjectDir of await getSubprojectDirectories(rootDir, workspaces)) {
+    const pkg = JSON.parse(fs.readFileSync(path.join(subProjectDir, 'package.json'), {
+      encoding: 'utf-8'
+    }))
+
+    siblingVersions[pkg.name] = calculateSiblingVersion(pkg.version)
+    packageDirs.push(subProjectDir)
   }
 
   return {

diff --git a/src/test-dependant/index.js b/src/test-dependant/index.js
@@ -4,8 +4,7 @@ import os from 'os'
 import path from 'path'
 import fs from 'fs-extra'
 import {
-  exec,
-  glob
+  exec, getSubprojectDirectories
 } from '../utils.js'
 
 /**
@@ -197,10 +196,8 @@ const testMonoRepo = async (targetDir, deps, scriptName) => {
   }
 
   // test each package that depends on passed deps
-  for (const pattern of config.workspaces) {
-    for await (const match of glob(targetDir, pattern)) {
-      await testModule(path.join(targetDir, match), deps, scriptName)
-    }
+  for (const match of await getSubprojectDirectories(targetDir, config.workspaces)) {
+    await testModule(path.join(targetDir, match), deps, scriptName)
   }
 }
 

diff --git a/src/utils.js b/src/utils.js
@@ -14,10 +14,10 @@ import { download } from '@electron/get'
 import envPaths from 'env-paths'
 import { execa } from 'execa'
 import extract from 'extract-zip'
+import fg from 'fast-glob'
 import fs from 'fs-extra'
 import kleur from 'kleur'
 import Listr from 'listr'
-import { minimatch } from 'minimatch'
 import PQueue from 'p-queue'
 import lockfile from 'proper-lockfile'
 import { readPackageUpSync } from 'read-pkg-up'
@@ -355,7 +355,7 @@ export async function everyMonorepoProject (projectDir, fn, opts) {
   }
 
   /** @type {Record<string, Project>} */
-  const projects = parseProjects(projectDir, workspaces)
+  const projects = await parseProjects(projectDir, workspaces)
 
   checkForCircularDependencies(projects)
 
@@ -402,40 +402,45 @@ export async function everyMonorepoProject (projectDir, fn, opts) {
  * @param {string} projectDir
  * @param {string[]} workspaces
  */
-export function parseProjects (projectDir, workspaces) {
+export const getSubprojectDirectories = async (projectDir, workspaces) => fg.glob(workspaces, {
+  cwd: projectDir,
+  onlyFiles: false
+})
+
+/**
+ *
+ * @param {string} projectDir
+ * @param {string[]} workspaces
+ */
+export async function parseProjects (projectDir, workspaces) {
   /** @type {Record<string, Project>} */
   const projects = {}
 
-  for (const workspace of workspaces) {
-    for (const subProjectDir of glob('.', workspace, {
-      cwd: projectDir,
-      absolute: true
-    })) {
-      const stat = fs.statSync(subProjectDir)
+  for (const subProjectDir of await getSubprojectDirectories(projectDir, workspaces)) {
+    const stat = fs.statSync(subProjectDir)
 
-      if (!stat.isDirectory()) {
-        continue
-      }
+    if (!stat.isDirectory()) {
+      continue
+    }
 
-      const manfest = path.join(subProjectDir, 'package.json')
+    const manfest = path.join(subProjectDir, 'package.json')
 
-      if (!fs.existsSync(manfest)) {
-        continue
-      }
+    if (!fs.existsSync(manfest)) {
+      continue
+    }
 
-      const pkg = fs.readJSONSync(manfest)
-
-      projects[pkg.name] = {
-        manifest: pkg,
-        dir: subProjectDir,
-        siblingDependencies: [],
-        dependencies: [
-          ...Object.keys(pkg.dependencies ?? {}),
-          ...Object.keys(pkg.devDependencies ?? {}),
-          ...Object.keys(pkg.optionalDependencies ?? {}),
-          ...Object.keys(pkg.peerDependencies ?? {})
-        ]
-      }
+    const pkg = fs.readJSONSync(manfest)
+
+    projects[pkg.name] = {
+      manifest: pkg,
+      dir: subProjectDir,
+      siblingDependencies: [],
+      dependencies: [
+        ...Object.keys(pkg.dependencies ?? {}),
+        ...Object.keys(pkg.devDependencies ?? {}),
+        ...Object.keys(pkg.optionalDependencies ?? {}),
+        ...Object.keys(pkg.peerDependencies ?? {})
+      ]
     }
   }
 
@@ -504,92 +509,6 @@ function checkForCircularDependencies (projects) {
   }
 }
 
-/**
- * @typedef {object} GlobOptions
- * @property {string} [cwd] The current working directory
- * @property {boolean} [absolute] If true produces absolute paths (default: false)
- * @property {boolean} [nodir] If true yields file paths and skip directories (default: false)
- *
- * Iterable filename pattern matcher
- *
- * @param {string} dir
- * @param {string} pattern
- * @param {GlobOptions & import('minimatch').MinimatchOptions} [options]
- * @returns {Generator<string, void, undefined>}
- */
-export function * glob (dir, pattern, options = {}) {
-  const absoluteDir = path.resolve(dir)
-  const relativeDir = path.relative(options.cwd ?? process.cwd(), dir)
-
-  const stats = fs.statSync(absoluteDir)
-
-  if (stats.isDirectory()) {
-    for (const entry of _glob(absoluteDir, '', pattern, options)) {
-      yield entry
-    }
-
-    return
-  }
-
-  if (minimatch(relativeDir, pattern, options)) {
-    yield options.absolute === true ? absoluteDir : relativeDir
-  }
-}
-
-/**
- * @param {string} base
- * @param {string} dir
- * @param {string} pattern
- * @param {GlobOptions & import('minimatch').MinimatchOptions} options
- * @returns {Generator<string, void, undefined>}
- */
-function * _glob (base, dir, pattern, options) {
-  const p = path.join(base, dir)
-
-  if (!fs.existsSync(p)) {
-    return
-  }
-
-  const stats = fs.statSync(p)
-
-  if (!stats.isDirectory()) {
-    return
-  }
-
-  const d = fs.opendirSync(p)
-
-  try {
-    while (true) {
-      const entry = d.readSync()
-
-      if (entry == null) {
-        break
-      }
-
-      const relativeEntryPath = path.join(dir, entry.name)
-      const absoluteEntryPath = path.join(base, dir, entry.name)
-
-      let match = minimatch(relativeEntryPath, pattern, options)
-
-      const isDirectory = entry.isDirectory()
-
-      if (isDirectory && options.nodir === true) {
-        match = false
-      }
-
-      if (match) {
-        yield options.absolute === true ? absoluteEntryPath : relativeEntryPath
-      }
-
-      if (isDirectory) {
-        yield * _glob(base, relativeEntryPath, pattern, options)
-      }
-    }
-  } finally {
-    d.closeSync()
-  }
-}
-
 /**
  *
  * @param {Error} error