diff --git a/.changeset/good-mirrors-bake.md b/.changeset/good-mirrors-bake.md new file mode 100644 index 0000000000000..c5968f5ba3daf --- /dev/null +++ b/.changeset/good-mirrors-bake.md @@ -0,0 +1,9 @@ +--- +'astro': minor +--- + +Improved image optimization performance + +Astro will now generate optimized images concurrently at build time, which can significantly speed up build times for sites with many images. Additionally, Astro will now reuse the same buffer for all variants of an image. This should improve performance for websites with many variants of the same image, especially when using remote images. + +No code changes are required to take advantage of these improvements. diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 041d475aa39d6..70c4b0ffa74d2 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -17,7 +17,12 @@ module.exports = { '@typescript-eslint/array-type': ['error', { default: 'array-simple' }], '@typescript-eslint/no-unused-vars': [ 'warn', - { argsIgnorePattern: '^_', ignoreRestSiblings: true }, + { + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + caughtErrorsIgnorePattern: '^_', + ignoreRestSiblings: true, + }, ], 'no-only-tests/no-only-tests': 'error', '@typescript-eslint/no-shadow': ['error'], diff --git a/packages/astro/package.json b/packages/astro/package.json index ab5dd25461238..d28fe9565ffbe 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -157,6 +157,7 @@ "mime": "^3.0.0", "ora": "^7.0.1", "p-limit": "^4.0.0", + "p-queue": "^7.4.1", "path-to-regexp": "^6.2.1", "preferred-pm": "^3.1.2", "probe-image-size": "^7.2.3", diff --git a/packages/astro/src/assets/build/generate.ts b/packages/astro/src/assets/build/generate.ts index 283e957a3ccb2..4447fa0a73d52 100644 --- a/packages/astro/src/assets/build/generate.ts +++ b/packages/astro/src/assets/build/generate.ts @@ -1,12 +1,18 @@ +import { dim, green } from 'kleur/colors'; import fs, { readFileSync } from 'node:fs'; import { basename, join } from 'node:path/posix'; +import PQueue from 'p-queue'; +import type { AstroConfig } from '../../@types/astro.js'; import type { BuildPipeline } from '../../core/build/buildPipeline.js'; import { getOutDirWithinCwd } from '../../core/build/common.js'; -import { prependForwardSlash } from '../../core/path.js'; +import { getTimeStat } from '../../core/build/util.js'; +import type { Logger } from '../../core/logger/core.js'; +import { isRemotePath, prependForwardSlash } from '../../core/path.js'; import { isServerLikeOutput } from '../../prerender/utils.js'; +import type { MapValue } from '../../type-utils.js'; import { getConfiguredImageService, isESMImportedImage } from '../internal.js'; import type { LocalImageService } from '../services/service.js'; -import type { ImageMetadata, ImageTransform } from '../types.js'; +import type { AssetsGlobalStaticImagesList, ImageMetadata, ImageTransform } from '../types.js'; import { loadRemoteImage, type RemoteCacheEntry } from './remote.js'; interface GenerationDataUncached { @@ -23,15 +29,28 @@ interface GenerationDataCached { type GenerationData = GenerationDataUncached | GenerationDataCached; -export async function generateImage( +type AssetEnv = { + logger: Logger; + count: { total: number; current: number }; + useCache: boolean; + assetsCacheDir: URL; + serverRoot: URL; + clientRoot: URL; + imageConfig: AstroConfig['image']; + assetsFolder: AstroConfig['build']['assets']; +}; + +type ImageData = { data: Buffer; expires: number }; + +export async function prepareAssetsGenerationEnv( pipeline: BuildPipeline, - options: ImageTransform, - filepath: string -): Promise { + totalCount: number +): Promise { const config = pipeline.getConfig(); const logger = pipeline.getLogger(); let useCache = true; const assetsCacheDir = new URL('assets/', config.cacheDir); + const count = { total: totalCount, current: 1 }; // Ensure that the cache directory exists try { @@ -53,113 +72,182 @@ export async function generateImage( clientRoot = config.outDir; } - const isLocalImage = isESMImportedImage(options.src); - - const finalFileURL = new URL('.' + filepath, clientRoot); - const finalFolderURL = new URL('./', finalFileURL); + return { + logger, + count, + useCache, + assetsCacheDir, + serverRoot, + clientRoot, + imageConfig: config.image, + assetsFolder: config.build.assets, + }; +} - // For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server - const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json'); - const cachedFileURL = new URL(cacheFile, assetsCacheDir); +export async function generateImagesForPath( + originalFilePath: string, + transforms: MapValue, + env: AssetEnv, + queue: PQueue +) { + const originalImageData = await loadImage(originalFilePath, env); + + for (const [_, transform] of transforms) { + queue.add(async () => + generateImage(originalImageData, transform.finalPath, transform.transform) + ); + } - await fs.promises.mkdir(finalFolderURL, { recursive: true }); + async function generateImage( + originalImage: ImageData, + filepath: string, + options: ImageTransform + ) { + const timeStart = performance.now(); + const generationData = await generateImageInternal(originalImage, filepath, options); + + const timeEnd = performance.now(); + const timeChange = getTimeStat(timeStart, timeEnd); + const timeIncrease = `(+${timeChange})`; + const statsText = generationData.cached + ? `(reused cache entry)` + : `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`; + const count = `(${env.count.current}/${env.count.total})`; + env.logger.info( + null, + ` ${green('▶')} ${filepath} ${dim(statsText)} ${dim(timeIncrease)} ${dim(count)}` + ); + env.count.current++; + } - // Check if we have a cached entry first - try { - if (isLocalImage) { - await fs.promises.copyFile(cachedFileURL, finalFileURL); + async function generateImageInternal( + originalImage: ImageData, + filepath: string, + options: ImageTransform + ): Promise { + const isLocalImage = isESMImportedImage(options.src); + const finalFileURL = new URL('.' + filepath, env.clientRoot); - return { - cached: true, - }; - } else { - const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry; + // For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server + const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json'); + const cachedFileURL = new URL(cacheFile, env.assetsCacheDir); - // If the cache entry is not expired, use it - if (JSONData.expires > Date.now()) { - await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64')); + // Check if we have a cached entry first + try { + if (isLocalImage) { + await fs.promises.copyFile(cachedFileURL, finalFileURL, fs.constants.COPYFILE_FICLONE); return { cached: true, }; + } else { + const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry; + + if (!JSONData.data || !JSONData.expires) { + await fs.promises.unlink(cachedFileURL); + + throw new Error( + `Malformed cache entry for ${filepath}, cache will be regenerated for this file.` + ); + } + + // If the cache entry is not expired, use it + if (JSONData.expires > Date.now()) { + await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64')); + + return { + cached: true, + }; + } else { + await fs.promises.unlink(cachedFileURL); + } } + } catch (e: any) { + if (e.code !== 'ENOENT') { + throw new Error(`An error was encountered while reading the cache file. Error: ${e}`); + } + // If the cache file doesn't exist, just move on, and we'll generate it } - } catch (e: any) { - if (e.code !== 'ENOENT') { - throw new Error(`An error was encountered while reading the cache file. Error: ${e}`); - } - // If the cache file doesn't exist, just move on, and we'll generate it - } - - // The original filepath or URL from the image transform - const originalImagePath = isLocalImage - ? (options.src as ImageMetadata).src - : (options.src as string); - let imageData; - let resultData: { data: Buffer | undefined; expires: number | undefined } = { - data: undefined, - expires: undefined, - }; - - // If the image is local, we can just read it directly, otherwise we need to download it - if (isLocalImage) { - imageData = await fs.promises.readFile( - new URL( - '.' + prependForwardSlash(join(config.build.assets, basename(originalImagePath))), - serverRoot + const finalFolderURL = new URL('./', finalFileURL); + await fs.promises.mkdir(finalFolderURL, { recursive: true }); + + // The original filepath or URL from the image transform + const originalImagePath = isLocalImage + ? (options.src as ImageMetadata).src + : (options.src as string); + + let resultData: Partial = { + data: undefined, + expires: originalImage.expires, + }; + + const imageService = (await getConfiguredImageService()) as LocalImageService; + resultData.data = ( + await imageService.transform( + originalImage.data, + { ...options, src: originalImagePath }, + env.imageConfig ) - ); - } else { - const remoteImage = await loadRemoteImage(originalImagePath); - resultData.expires = remoteImage.expires; - imageData = remoteImage.data; - } - - const imageService = (await getConfiguredImageService()) as LocalImageService; - resultData.data = ( - await imageService.transform(imageData, { ...options, src: originalImagePath }, config.image) - ).data; - - try { - // Write the cache entry - if (useCache) { - if (isLocalImage) { - await fs.promises.writeFile(cachedFileURL, resultData.data); - } else { - await fs.promises.writeFile( - cachedFileURL, - JSON.stringify({ - data: Buffer.from(resultData.data).toString('base64'), - expires: resultData.expires, - }) - ); + ).data; + + try { + // Write the cache entry + if (env.useCache) { + if (isLocalImage) { + await fs.promises.writeFile(cachedFileURL, resultData.data); + } else { + await fs.promises.writeFile( + cachedFileURL, + JSON.stringify({ + data: Buffer.from(resultData.data).toString('base64'), + expires: resultData.expires, + }) + ); + } } + } catch (e) { + env.logger.warn( + 'astro:assets', + `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}` + ); + } finally { + // Write the final file + await fs.promises.writeFile(finalFileURL, resultData.data); } - } catch (e) { - logger.warn( - 'astro:assets', - `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}` - ); - } finally { - // Write the final file - await fs.promises.writeFile(finalFileURL, resultData.data); - } - return { - cached: false, - weight: { - // Divide by 1024 to get size in kilobytes - before: Math.trunc(imageData.byteLength / 1024), - after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024), - }, - }; + return { + cached: false, + weight: { + // Divide by 1024 to get size in kilobytes + before: Math.trunc(originalImage.data.byteLength / 1024), + after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024), + }, + }; + } } -export function getStaticImageList(): Map { +export function getStaticImageList(): AssetsGlobalStaticImagesList { if (!globalThis?.astroAsset?.staticImages) { return new Map(); } return globalThis.astroAsset.staticImages; } + +async function loadImage(path: string, env: AssetEnv): Promise { + if (isRemotePath(path)) { + const remoteImage = await loadRemoteImage(path); + return { + data: remoteImage.data, + expires: remoteImage.expires, + }; + } + + return { + data: await fs.promises.readFile( + new URL('.' + prependForwardSlash(join(env.assetsFolder, basename(path))), env.serverRoot) + ), + expires: 0, + }; +} diff --git a/packages/astro/src/assets/types.ts b/packages/astro/src/assets/types.ts index 43bfe0f53c5e4..ec7393e504307 100644 --- a/packages/astro/src/assets/types.ts +++ b/packages/astro/src/assets/types.ts @@ -8,12 +8,17 @@ export type ImageQuality = ImageQualityPreset | number; export type ImageInputFormat = (typeof VALID_INPUT_FORMATS)[number]; export type ImageOutputFormat = (typeof VALID_OUTPUT_FORMATS)[number] | (string & {}); +export type AssetsGlobalStaticImagesList = Map< + string, + Map +>; + declare global { // eslint-disable-next-line no-var var astroAsset: { imageService?: ImageService; addStaticImage?: ((options: ImageTransform) => string) | undefined; - staticImages?: Map; + staticImages?: AssetsGlobalStaticImagesList; }; } diff --git a/packages/astro/src/assets/vite-plugin-assets.ts b/packages/astro/src/assets/vite-plugin-assets.ts index a4346edcea8ba..382d161fd76a8 100644 --- a/packages/astro/src/assets/vite-plugin-assets.ts +++ b/packages/astro/src/assets/vite-plugin-assets.ts @@ -12,6 +12,7 @@ import { } from '../core/path.js'; import { isServerLikeOutput } from '../prerender/utils.js'; import { VALID_INPUT_FORMATS, VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js'; +import { isESMImportedImage } from './internal.js'; import { emitESMImage } from './utils/emitAsset.js'; import { hashTransform, propsToFilename } from './utils/transformToPath.js'; @@ -80,27 +81,37 @@ export default function assets({ if (!globalThis.astroAsset.staticImages) { globalThis.astroAsset.staticImages = new Map< string, - { path: string; options: ImageTransform } + Map >(); } + const originalImagePath = ( + isESMImportedImage(options.src) ? options.src.src : options.src + ).replace(settings.config.build.assetsPrefix || '', ''); const hash = hashTransform(options, settings.config.image.service.entrypoint); - let filePath: string; - if (globalThis.astroAsset.staticImages.has(hash)) { - filePath = globalThis.astroAsset.staticImages.get(hash)!.path; + let finalFilePath: string; + let transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath); + let transformForHash = transformsForPath?.get(hash); + if (transformsForPath && transformForHash) { + finalFilePath = transformForHash.finalPath; } else { - filePath = prependForwardSlash( + finalFilePath = prependForwardSlash( joinPaths(settings.config.build.assets, propsToFilename(options, hash)) ); - globalThis.astroAsset.staticImages.set(hash, { path: filePath, options: options }); + if (!transformsForPath) { + globalThis.astroAsset.staticImages.set(originalImagePath, new Map()); + transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath)!; + } + + transformsForPath.set(hash, { finalPath: finalFilePath, transform: options }); } if (settings.config.build.assetsPrefix) { - return joinPaths(settings.config.build.assetsPrefix, filePath); + return joinPaths(settings.config.build.assetsPrefix, finalFilePath); } else { - return prependForwardSlash(joinPaths(settings.config.base, filePath)); + return prependForwardSlash(joinPaths(settings.config.base, finalFilePath)); } }; }, diff --git a/packages/astro/src/core/build/generate.ts b/packages/astro/src/core/build/generate.ts index a5b3165545113..258d02038d6be 100644 --- a/packages/astro/src/core/build/generate.ts +++ b/packages/astro/src/core/build/generate.ts @@ -1,7 +1,9 @@ import * as colors from 'kleur/colors'; import { bgGreen, black, cyan, dim, green, magenta } from 'kleur/colors'; import fs from 'node:fs'; +import os from 'node:os'; import { fileURLToPath } from 'node:url'; +import PQueue from 'p-queue'; import type { OutputAsset, OutputChunk } from 'rollup'; import type { BufferEncoding } from 'vfile'; import type { @@ -9,7 +11,6 @@ import type { AstroSettings, ComponentInstance, GetStaticPathsItem, - ImageTransform, MiddlewareEndpointHandler, RouteData, RouteType, @@ -18,8 +19,9 @@ import type { SSRManifest, } from '../../@types/astro.js'; import { - generateImage as generateImageInternal, + generateImagesForPath, getStaticImageList, + prepareAssetsGenerationEnv, } from '../../assets/build/generate.js'; import { hasPrerenderedPages, type BuildInternals } from '../../core/build/internal.js'; import { @@ -196,58 +198,35 @@ export async function generatePages(opts: StaticBuildOptions, internals: BuildIn } } - const staticImageList = getStaticImageList(); + logger.info(null, dim(`Completed in ${getTimeStat(timer, performance.now())}.\n`)); - if (staticImageList.size) + const staticImageList = getStaticImageList(); + if (staticImageList.size) { logger.info(null, `\n${bgGreen(black(` generating optimized images `))}`); - let count = 0; - for (const imageData of staticImageList.entries()) { - count++; - await generateImage( - pipeline, - imageData[1].options, - imageData[1].path, - count, - staticImageList.size - ); - } - - delete globalThis?.astroAsset?.addStaticImage; - await runHookBuildGenerated({ - config: opts.settings.config, - logger: pipeline.getLogger(), - }); + const totalCount = Array.from(staticImageList.values()) + .map((x) => x.size) + .reduce((a, b) => a + b, 0); + const cpuCount = os.cpus().length; + const assetsCreationEnvironment = await prepareAssetsGenerationEnv(pipeline, totalCount); + const queue = new PQueue({ concurrency: cpuCount }); - logger.info(null, dim(`Completed in ${getTimeStat(timer, performance.now())}.\n`)); -} + const assetsTimer = performance.now(); + for (const [originalPath, transforms] of staticImageList) { + await generateImagesForPath(originalPath, transforms, assetsCreationEnvironment, queue); + } -async function generateImage( - pipeline: BuildPipeline, - transform: ImageTransform, - path: string, - count: number, - totalCount: number -) { - const logger = pipeline.getLogger(); - let timeStart = performance.now(); - const generationData = await generateImageInternal(pipeline, transform, path); + await queue.onIdle(); + const assetsTimeEnd = performance.now(); + logger.info(null, dim(`Completed in ${getTimeStat(assetsTimer, assetsTimeEnd)}.\n`)); - if (!generationData) { - return; + delete globalThis?.astroAsset?.addStaticImage; } - const timeEnd = performance.now(); - const timeChange = getTimeStat(timeStart, timeEnd); - const timeIncrease = `(+${timeChange})`; - const statsText = generationData.cached - ? `(reused cache entry)` - : `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`; - const counter = `(${count}/${totalCount})`; - logger.info( - null, - ` ${green('▶')} ${path} ${dim(statsText)} ${dim(timeIncrease)} ${dim(counter)}` - ); + await runHookBuildGenerated({ + config: opts.settings.config, + logger: pipeline.getLogger(), + }); } async function generatePage( diff --git a/packages/astro/src/type-utils.ts b/packages/astro/src/type-utils.ts index 96970f7c4fb1c..d777a9f287893 100644 --- a/packages/astro/src/type-utils.ts +++ b/packages/astro/src/type-utils.ts @@ -27,3 +27,6 @@ export type KebabKeys = { [K in keyof T as K extends string ? Kebab : K]: // Similar to `keyof`, gets the type of all the values of an object export type ValueOf = T[keyof T]; + +// Gets the type of the values of a Map +export type MapValue = T extends Map ? V : never;