From df5dc1a067bfdd95300540a84e2bcbae677d3275 Mon Sep 17 00:00:00 2001 From: David Herges Date: Thu, 8 Oct 2026 21:58:27 +0200 Subject: [PATCH 1/9] test(@angular/build): add library builder scaling benchmark suite --- scripts/benchmark.mts | 72 ++++- scripts/benchmarks/library-builder/README.md | 77 ++++++ .../benchmarks/library-builder/fixtures.mts | 257 ++++++++++++++++++ scripts/benchmarks/library-builder/index.mts | 247 +++++++++++++++++ 4 files changed, 651 insertions(+), 2 deletions(-) create mode 100644 scripts/benchmarks/library-builder/README.md create mode 100644 scripts/benchmarks/library-builder/fixtures.mts create mode 100644 scripts/benchmarks/library-builder/index.mts diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts index 0237f5101bc4..a8318b84af11 100644 --- a/scripts/benchmark.mts +++ b/scripts/benchmark.mts @@ -9,6 +9,10 @@ import fs from 'node:fs'; import path from 'node:path'; import { type BenchmarkCliOptions, runI18nBenchmarks } from './benchmarks/i18n/index.mts'; +import { + type LibraryBuilderBenchmarkOptions, + runLibraryBuilderBenchmarks, +} from './benchmarks/library-builder/index.mts'; function checkBuildStatus(logger: Console): boolean { const distFile = 'dist/@angular/build/src/tools/i18n/i18n-inliner.js'; @@ -36,6 +40,50 @@ function checkBuildStatus(logger: Console): boolean { return true; } +function parseIntList(raw: string | undefined): number[] | undefined { + return raw?.split(',').map((s) => Number(s.trim())); +} + +async function runLibraryBuilderSubsystem(options: { + layout?: string; + style?: string; + sizes?: string; + depths?: string; + iterations?: string | number; + json?: boolean; +}): Promise { + if (options.layout !== undefined && options.layout !== 'flat' && options.layout !== 'deep') { + // eslint-disable-next-line no-console + console.error(`Error: --layout must be "flat" or "deep", got "${options.layout}".`); + + return 1; + } + if ( + options.style !== undefined && + !['inline', 'inline-scss', 'external'].includes(options.style) + ) { + // eslint-disable-next-line no-console + console.error( + `Error: --style must be one of "inline", "inline-scss", "external", got "${options.style}".`, + ); + + return 1; + } + + const cliOptions: LibraryBuilderBenchmarkOptions = { + layout: options.layout as LibraryBuilderBenchmarkOptions['layout'], + style: options.style as LibraryBuilderBenchmarkOptions['style'], + sizes: parseIntList(options.sizes), + depths: parseIntList(options.depths), + iterations: options.iterations !== undefined ? Number(options.iterations) : undefined, + json: Boolean(options.json), + }; + + const { exitCode } = await runLibraryBuilderBenchmarks(cliOptions); + + return exitCode; +} + export default async function ( options: { _?: string[]; @@ -51,6 +99,10 @@ export default async function ( 'save-baseline'?: string; compareBaseline?: string; 'compare-baseline'?: string; + layout?: string; + style?: string; + sizes?: string; + depths?: string; help?: boolean; [key: string]: unknown; }, @@ -69,8 +121,9 @@ Usage: Subsystems: i18n Run i18n inliner performance benchmarks (default) + library-builder Run @angular/build:library scaling benchmarks -Options: +Options (i18n): --scenario= Run a specific scenario (e.g. standard-app, enterprise-multilingual) --iterations= Number of measured iterations (default: 5) --warmup= Number of warmup iterations (default: 2) @@ -80,15 +133,30 @@ Options: --json Output results in machine-readable JSON --save-baseline= Save run results to a baseline JSON file --compare-baseline= Compare run results against an existing baseline JSON file + +Options (library-builder): + --layout= Fixture layout (default: flat) + --style= Component style variant (default: inline) + --sizes= Entry-point counts to sweep, layout=flat only (default: 10,50,300,1000,2000) + --depths= Tree depths to sweep, layout=deep only (default: 2,5,8,11) + --iterations= Measured iterations per size (default: 2) + --json Output raw measurements as JSON + --help Show this help message `); return 0; } + if (targetSubsystem === 'library-builder') { + return runLibraryBuilderSubsystem(options); + } + if (targetSubsystem !== 'i18n') { // eslint-disable-next-line no-console - console.error(`Unknown benchmark subsystem: "${targetSubsystem}". Supported subsystems: i18n`); + console.error( + `Unknown benchmark subsystem: "${targetSubsystem}". Supported subsystems: i18n, library-builder`, + ); return 1; } diff --git a/scripts/benchmarks/library-builder/README.md b/scripts/benchmarks/library-builder/README.md new file mode 100644 index 000000000000..407ba6710bc2 --- /dev/null +++ b/scripts/benchmarks/library-builder/README.md @@ -0,0 +1,77 @@ +# `@angular/build:library` scaling benchmark + +This benchmark measures how `@angular/build:library`'s cold build time scales with the number +of secondary entry points in an Angular Package Format (APF) library, across two structurally +different layouts (flat siblings vs. a deeply nested tree) and three component style variants +(inline template, inline template + styles, external template/stylesheet files). + +## Why + +A comparison against [ng-packagr](https://github.com/ng-packagr/ng-packagr) (a third-party APF +build tool) found that its build time **per entry point** grows super-linearly once a library +has more than roughly 300-500 secondary entry points -- doubling the library size past that +point costs noticeably more than double the build time. This suite exists to check whether +`@angular/build:library`, a from-scratch build pipeline unrelated to ng-packagr's implementation, +shows the same growth on equivalent fixtures. + +**Finding:** it does not. Across the same sweep (flat layout: 10/50/300/1000/2000 entries; deep +layout: tree depths 2/5/8/11, i.e. 4/32/256/2048 entries), `@angular/build:library`'s per-entry +cost keeps falling as the library grows, with no climb at the largest sizes tested. It is also +1.8-13x faster than ng-packagr at matching sizes, with the gap widening as size increases. See +the PR description this suite was added alongside for the full writeup and caveats. + +## Running it + +```sh +pnpm build # or: pnpm admin build --local -- the benchmark measures the compiled builder, + # not TypeScript source, so it must exist at dist/@angular/build and + # dist/@angular-devkit/architect first +pnpm admin benchmark library-builder +``` + +Options: + +``` +--layout= Fixture layout (default: flat) +--style= Component style variant (default: inline) +--sizes= Entry-point counts to sweep, layout=flat only (default: 10,50,300,1000,2000) +--depths= Tree depths to sweep, layout=deep only (default: 2,5,8,11) +--iterations= Measured iterations per size (default: 2) +--json Output raw measurements as JSON instead of a human-readable log +``` + +Example: sweep the external-style variant at a few sizes with 3 iterations each: + +```sh +pnpm admin benchmark library-builder --style=external --sizes=100,500,1000 --iterations=3 +``` + +## How it works + +- `fixtures.mts` generates a synthetic library at a given size/layout/style: a primary entry + point, N secondary entry points (each a minimal standalone Angular component), and the + `angular.json`/`tsconfig.lib.json`/`package.json` scaffolding `@angular/build:library` needs. + Secondary entry points are declared as a single flat `exports` map in the root `package.json` + (the builder's actual convention), not as separate per-entry config files. +- `index.mts` times `architect lib:build` (the standalone `@angular-devkit/architect` CLI, + invoked directly -- no `ng` CLI needed) once per iteration via `child_process.spawnSync`, + discards the generated project afterwards, and reports per-size median build time. +- Because `@angular/build` and `@angular-devkit/architect` are Bazel-built workspace packages + rather than ordinary npm dependencies, a plain `pnpm install` never links them (or their own + dependencies) into the repo's root `node_modules` -- nothing in `dist/` is resolvable via a + bare `require()`/`import` otherwise. On first run, `index.mts` symlinks `@angular/build` and + `@angular-devkit/architect` from `dist/` into the root `node_modules`, then walks their + declared `dependencies` and links any of those still missing from the pnpm store. This only + touches `node_modules` (already gitignored) and is idempotent -- safe to run repeatedly or + alongside other work in the same checkout. + +## Methodology notes + +- Each reported number is the **median** of the measured iterations, not the mean -- process-spawn + wall-clock timings are noisy and right-skewed, and the median is more representative of steady + state than an outlier-sensitive mean. +- Results are specific to the machine they were collected on. Run on one otherwise-idle machine + for a self-consistent sweep; absolute numbers will differ across hardware, though the _shape_ + of the curve (does per-entry cost grow with size, or not) is the more portable finding. +- This benchmark only covers **cold, one-shot builds**. It does not exercise incremental rebuilds + or watch mode. diff --git a/scripts/benchmarks/library-builder/fixtures.mts b/scripts/benchmarks/library-builder/fixtures.mts new file mode 100644 index 000000000000..33ffc91762b7 --- /dev/null +++ b/scripts/benchmarks/library-builder/fixtures.mts @@ -0,0 +1,257 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Generates synthetic Angular Package Format (APF) library fixtures for benchmarking +// `@angular/build:library`, and converts them into the builder's project shape (a flat +// `exports` map in one root package.json, rather than the per-entry-point `ng-package.json` +// convention used by third-party tools such as ng-packagr). + +import fs from 'node:fs'; +import path from 'node:path'; + +export type Layout = 'flat' | 'deep'; +export type Style = 'inline' | 'inline-scss' | 'external'; + +export interface FixtureOptions { + layout: Layout; + style: Style; + + /** Number of secondary entry points for `layout: 'flat'`. */ + count?: number; + + /** Binary-tree depth for `layout: 'deep'` (2^depth leaf entry points). */ + depth?: number; +} + +function buildComponentSource( + className: string, + selector: string, + compBase: string, + id: string, + style: Style, +): string { + const metadataLines = [` selector: '${selector}',`]; + + if (style === 'external') { + metadataLines.push(` templateUrl: './${compBase}.component.html',`); + metadataLines.push(` styleUrls: ['./${compBase}.component.scss'],`); + } else { + metadataLines.push(` template: '
Reference component ${id}
',`); + if (style === 'inline-scss') { + metadataLines.push(` styles: ['.ref-comp { display: block; padding: 4px; }'],`); + } + } + + return `import { Component } from '@angular/core'; + +@Component({ +${metadataLines.join('\n')} +}) +export class ${className} {} +`; +} + +/** Writes one secondary entry point's source files (component + public-api.ts) into `entryDir`. */ +function writeEntryPointSource(entryDir: string, id: string, style: Style): void { + fs.mkdirSync(entryDir, { recursive: true }); + + const className = `RefComponent${id}`; + const selector = `ref-comp-${id}`; + const compBase = `comp-${id}`; + + fs.writeFileSync( + path.join(entryDir, 'public-api.ts'), + `export * from './${compBase}.component';\n`, + ); + + if (style === 'external') { + fs.writeFileSync( + path.join(entryDir, `${compBase}.component.html`), + `
Reference component ${id}
\n`, + ); + fs.writeFileSync( + path.join(entryDir, `${compBase}.component.scss`), + `.ref-comp {\n display: block;\n padding: 4px;\n}\n`, + ); + } + + fs.writeFileSync( + path.join(entryDir, `${compBase}.component.ts`), + buildComponentSource(className, selector, compBase, id, style), + ); +} + +/** Generates `count` flat secondary entry points as direct siblings, returning their relative dirs. */ +function generateFlatEntries(outDir: string, count: number, style: Style): string[] { + const width = String(count).length; + const dirs: string[] = []; + for (let i = 1; i <= count; i++) { + const id = String(i).padStart(width, '0'); + const relDir = `comp-${id}`; + writeEntryPointSource(path.join(outDir, relDir), id, style); + dirs.push(relDir); + } + + return dirs; +} + +/** + * Generates a binary tree `depth` levels deep (`sub-a`/`sub-b` at each level), placing one + * secondary entry point at each of the 2^depth leaves. Returns their relative dirs. + */ +function generateDeepEntries(outDir: string, depth: number, style: Style): string[] { + const total = 2 ** depth; + const width = String(total).length; + const dirs: string[] = []; + let counter = 0; + + function build(currentRelDir: string, level: number): void { + if (level === depth) { + counter++; + const id = String(counter).padStart(width, '0'); + writeEntryPointSource(path.join(outDir, currentRelDir), id, style); + dirs.push(currentRelDir); + + return; + } + for (const child of ['sub-a', 'sub-b']) { + build(path.join(currentRelDir, child), level + 1); + } + } + + build('.', 0); + + return dirs.map((d) => path.normalize(d)); +} + +/** "sub-a/sub-b" -> "sub-a-sub-b"; "comp-0001" -> "comp-0001". Used as the exports map subpath key. */ +function entryKey(relDir: string): string { + return relDir.split(path.sep).join('-'); +} + +/** + * Generates a complete `@angular/build:library`-buildable project at `outDir`: a primary entry + * point, N secondary entry points per `options`, and the angular.json/tsconfig/package.json + * scaffolding the builder needs. Returns the number of secondary entry points generated. + */ +export function generateFixture(outDir: string, options: FixtureOptions): number { + fs.rmSync(outDir, { recursive: true, force: true }); + fs.mkdirSync(outDir, { recursive: true }); + + fs.writeFileSync( + path.join(outDir, 'public-api.ts'), + `export const REFERENCE_VERSION = '1.0.0';\n`, + ); + + const secondaryDirs = + options.layout === 'flat' + ? generateFlatEntries(outDir, options.count ?? 10, options.style) + : generateDeepEntries(outDir, options.depth ?? 2, options.style); + + const exportsMap: Record = { '.': './public-api.ts' }; + for (const relDir of secondaryDirs) { + exportsMap[`./${entryKey(relDir)}`] = `./${relDir.split(path.sep).join('/')}/public-api.ts`; + } + + const angularCoreVersion = JSON.parse( + fs.readFileSync( + path.resolve(import.meta.dirname, '../../../node_modules/@angular/core/package.json'), + 'utf8', + ), + ).version as string; + // e.g. "22.3.0-next.0" -> "22.3.0-next" (drop the trailing prerelease build number only). + const peerRange = `^${angularCoreVersion.replace(/\.\d+$/, '')}`; + + fs.writeFileSync( + path.join(outDir, 'package.json'), + JSON.stringify( + { + name: `ref-${options.layout}-${options.style}`, + version: '0.0.1', + peerDependencies: { + '@angular/core': peerRange, + '@angular/common': peerRange, + }, + exports: exportsMap, + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'tsconfig.json'), + JSON.stringify( + { + compileOnSave: false, + compilerOptions: { + outDir: './dist/out-tsc', + sourceMap: true, + declaration: false, + moduleResolution: 'bundler', + experimentalDecorators: true, + target: 'es2022', + module: 'es2022', + useDefineForClassFields: false, + lib: ['es2022', 'dom'], + }, + angularCompilerOptions: { + enableIvy: true, + disableTypeScriptVersionCheck: true, + }, + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'tsconfig.lib.json'), + JSON.stringify( + { + extends: './tsconfig.json', + compilerOptions: { + declaration: true, + declarationMap: true, + inlineSources: true, + types: [], + }, + exclude: ['**/*.spec.ts'], + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'angular.json'), + JSON.stringify( + { + version: 1, + cli: { cache: { enabled: false } }, + projects: { + lib: { + root: '.', + projectType: 'library', + sourceRoot: '.', + targets: { + build: { + builder: '@angular/build:library', + options: { tsConfig: './tsconfig.lib.json', outputPath: 'dist' }, + }, + }, + }, + }, + }, + null, + 2, + ) + '\n', + ); + + return secondaryDirs.length; +} diff --git a/scripts/benchmarks/library-builder/index.mts b/scripts/benchmarks/library-builder/index.mts new file mode 100644 index 000000000000..6e3aaaba03bd --- /dev/null +++ b/scripts/benchmarks/library-builder/index.mts @@ -0,0 +1,247 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Benchmarks `@angular/build:library`'s cold build time across a sweep of library sizes. +// +// Context: a comparison against ng-packagr (a third-party APF build tool) found that its +// per-entry-point build cost grows super-linearly past roughly 300-500 secondary entry points. +// This suite exists to check whether `@angular/build:library` -- a from-scratch build pipeline, +// unrelated to ng-packagr's implementation -- shows the same growth on equivalent fixtures. +// See scripts/benchmarks/library-builder/README.md for the full methodology and findings. + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { type FixtureOptions, type Layout, type Style, generateFixture } from './fixtures.mts'; + +export interface LibraryBuilderBenchmarkOptions { + layout?: Layout; + style?: Style; + + /** Comma-separated entry-point counts, for `layout: 'flat'`. */ + sizes?: number[]; + + /** Comma-separated tree depths, for `layout: 'deep'`. */ + depths?: number[]; + iterations?: number; + json?: boolean; +} + +export interface BuildMeasurement { + layout: Layout; + style: Style; + entryPoints: number; + iteration: number; + durationMs: number; + status: 'ok' | 'fail'; +} + +const repoRoot = path.resolve(import.meta.dirname, '../../..'); +const scratchRoot = path.join(repoRoot, 'dist', '.library-builder-benchmark-tmp'); +const rootNodeModules = path.join(repoRoot, 'node_modules'); +const pnpmStore = path.join(rootNodeModules, '.pnpm'); + +function checkBuilderIsBuilt( + logger: Console, +): { buildPkg: string; architectPkg: string; architectCli: string } | undefined { + const buildPkg = path.join(repoRoot, 'dist', '@angular', 'build'); + const architectPkg = path.join(repoRoot, 'dist', '@angular-devkit', 'architect'); + const architectCli = path.join(architectPkg, 'bin', 'cli.js'); + + if (!fs.existsSync(path.join(buildPkg, 'package.json')) || !fs.existsSync(architectCli)) { + logger.error( + 'Error: @angular/build and/or @angular-devkit/architect have not been built yet.\n' + + 'Run "pnpm build" (or "pnpm admin build --local" for a faster local build) first.', + ); + + return undefined; + } + + return { buildPkg, architectPkg, architectCli }; +} + +/** + * `@angular/build` and `@angular-devkit/architect` are Bazel-built workspace packages, not + * ordinary npm dependencies -- a plain `pnpm install` never links them (or their own + * dependencies) into the root node_modules. Resolving a plain `require('@angular/build')` + * from anywhere therefore needs these links to exist at least once, at the repo root (Node's + * module resolution walks up from each required file's real, on-disk location, so per-fixture + * symlinks are not enough once a dependency itself does a bare `require`). + * + * This walks the dependency graph starting at `@angular/build` and `@angular-devkit/architect` + * and links any package missing from node_modules, resolving each from the pnpm store so it + * doesn't go stale if a dependency version changes. + */ +function ensureBuilderDepsLinkedAtRepoRoot( + logger: Console, + { buildPkg, architectPkg }: { buildPkg: string; architectPkg: string }, +): void { + fs.mkdirSync(path.join(rootNodeModules, '@angular'), { recursive: true }); + fs.mkdirSync(path.join(rootNodeModules, '@angular-devkit'), { recursive: true }); + linkIfMissing('@angular/build', buildPkg); + linkIfMissing('@angular-devkit/architect', architectPkg); + + const seen = new Set(['@angular/build', '@angular-devkit/architect']); + const queue: string[] = ['@angular/build', '@angular-devkit/architect']; + while (queue.length > 0) { + const pkgName = queue.shift(); + if (pkgName === undefined) { + break; + } + const pkgJsonPath = path.join(rootNodeModules, pkgName, 'package.json'); + if (!fs.existsSync(pkgJsonPath)) { + continue; + } + const deps = Object.keys( + (JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8')).dependencies ?? {}) as Record< + string, + string + >, + ); + for (const dep of deps) { + if (seen.has(dep)) { + continue; + } + seen.add(dep); + queue.push(dep); + if (!fs.existsSync(path.join(rootNodeModules, dep))) { + linkFromPnpmStore(logger, dep); + } + } + } +} + +function linkIfMissing(name: string, target: string): void { + const linkPath = path.join(rootNodeModules, name); + if (!fs.existsSync(linkPath)) { + fs.symlinkSync(target, linkPath, 'dir'); + } +} + +/** Resolves `dep` (e.g. "rxjs" or "@angular-devkit/core") to its pnpm store dir and symlinks it. */ +function linkFromPnpmStore(logger: Console, dep: string): void { + const storeName = dep.startsWith('@') ? dep.replace('/', '+') : dep; + const candidates = fs.existsSync(pnpmStore) + ? fs.readdirSync(pnpmStore).filter((entry) => entry.startsWith(`${storeName}@`)) + : []; + if (candidates.length === 0) { + logger.warn(`Warning: could not resolve "${dep}" from the pnpm store; it may be missing.`); + + return; + } + // Prefer the lexically-last match (newest version) when more than one is installed. + candidates.sort(); + const storeDir = candidates[candidates.length - 1]; + const target = path.join(pnpmStore, storeDir, 'node_modules', dep); + if (dep.includes('/')) { + fs.mkdirSync(path.join(rootNodeModules, path.dirname(dep)), { recursive: true }); + } + linkIfMissing(dep, target); +} + +function buildOnce( + projectDir: string, + architectCli: string, +): { durationMs: number; status: 'ok' | 'fail' } { + const start = performance.now(); + const result = spawnSync(process.execPath, [architectCli, 'lib:build'], { + cwd: projectDir, + stdio: 'pipe', + maxBuffer: Infinity, + }); + const durationMs = performance.now() - start; + const status = result.status === 0 ? 'ok' : 'fail'; + if (status === 'fail') { + // eslint-disable-next-line no-console + console.error(result.stdout?.toString()); + // eslint-disable-next-line no-console + console.error(result.stderr?.toString()); + } + + return { durationMs, status }; +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} + +export async function runLibraryBuilderBenchmarks( + options: LibraryBuilderBenchmarkOptions = {}, +): Promise<{ measurements: BuildMeasurement[]; exitCode: number }> { + const builder = checkBuilderIsBuilt(console); + if (!builder) { + return { measurements: [], exitCode: 1 }; + } + ensureBuilderDepsLinkedAtRepoRoot(console, builder); + + const layout = options.layout ?? 'flat'; + const style = options.style ?? 'inline'; + const iterations = options.iterations ?? 2; + const sizes = + layout === 'flat' + ? (options.sizes ?? [10, 50, 300, 1000, 2000]) + : (options.depths ?? [2, 5, 8, 11]); + + fs.mkdirSync(scratchRoot, { recursive: true }); + const measurements: BuildMeasurement[] = []; + + for (const sizeOrDepth of sizes) { + const label = `${layout}-${style}-${layout === 'deep' ? `depth${sizeOrDepth}` : sizeOrDepth}`; + const projectDir = path.join(scratchRoot, label); + + const fixtureOptions: FixtureOptions = + layout === 'flat' + ? { layout, style, count: sizeOrDepth } + : { layout, style, depth: sizeOrDepth }; + const entryPoints = generateFixture(projectDir, fixtureOptions); + + if (!options.json) { + // eslint-disable-next-line no-console + console.log(`\n[library-builder] ${label} (${entryPoints} entry points)`); + } + + const durations: number[] = []; + for (let i = 1; i <= iterations; i++) { + const { durationMs, status } = buildOnce(projectDir, builder.architectCli); + if (!options.json) { + // eslint-disable-next-line no-console + console.log( + ` run ${i}/${iterations}: ${status === 'ok' ? 'done' : 'FAILED'} in ${(durationMs / 1000).toFixed(2)}s`, + ); + } + if (status === 'ok') { + durations.push(durationMs); + } + measurements.push({ layout, style, entryPoints, iteration: i, durationMs, status }); + } + + if (!options.json && durations.length > 0) { + const medianMs = median(durations); + // eslint-disable-next-line no-console + console.log( + ` median: ${(medianMs / 1000).toFixed(2)}s (${(medianMs / entryPoints).toFixed(2)}ms/entry)`, + ); + } + + fs.rmSync(projectDir, { recursive: true, force: true }); + } + + fs.rmSync(scratchRoot, { recursive: true, force: true }); + + const anyFailed = measurements.some((m) => m.status === 'fail'); + if (options.json) { + // eslint-disable-next-line no-console + console.log(JSON.stringify(measurements, null, 2)); + } + + return { measurements, exitCode: anyFailed ? 1 : 0 }; +} From 19f942ce322522ee6d7b407ebb379988cc47cf8e Mon Sep 17 00:00:00 2001 From: David Herges Date: Sat, 10 Oct 2026 05:54:26 +0000 Subject: [PATCH 2/9] fixup! test(@angular/build): add library builder scaling benchmark suite --- .../benchmarks/library-builder/fixtures.mts | 6 +++--- scripts/benchmarks/library-builder/index.mts | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/scripts/benchmarks/library-builder/fixtures.mts b/scripts/benchmarks/library-builder/fixtures.mts index 33ffc91762b7..940d570e6984 100644 --- a/scripts/benchmarks/library-builder/fixtures.mts +++ b/scripts/benchmarks/library-builder/fixtures.mts @@ -160,9 +160,9 @@ export function generateFixture(outDir: string, options: FixtureOptions): number const angularCoreVersion = JSON.parse( fs.readFileSync( - path.resolve(import.meta.dirname, '../../../node_modules/@angular/core/package.json'), - 'utf8', - ), + new URL('../../../node_modules/@angular/core/package.json', import.meta.url), + 'utf8' + ) ).version as string; // e.g. "22.3.0-next.0" -> "22.3.0-next" (drop the trailing prerelease build number only). const peerRange = `^${angularCoreVersion.replace(/\.\d+$/, '')}`; diff --git a/scripts/benchmarks/library-builder/index.mts b/scripts/benchmarks/library-builder/index.mts index 6e3aaaba03bd..9be2070b965f 100644 --- a/scripts/benchmarks/library-builder/index.mts +++ b/scripts/benchmarks/library-builder/index.mts @@ -18,6 +18,7 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { performance } from 'node:perf_hooks'; +import { fileURLToPath } from 'node:url'; import { type FixtureOptions, type Layout, type Style, generateFixture } from './fixtures.mts'; export interface LibraryBuilderBenchmarkOptions { @@ -42,7 +43,7 @@ export interface BuildMeasurement { status: 'ok' | 'fail'; } -const repoRoot = path.resolve(import.meta.dirname, '../../..'); +const repoRoot = fileURLToPath(new URL('../../../', import.meta.url)); const scratchRoot = path.join(repoRoot, 'dist', '.library-builder-benchmark-tmp'); const rootNodeModules = path.join(repoRoot, 'node_modules'); const pnpmStore = path.join(rootNodeModules, '.pnpm'); @@ -119,9 +120,21 @@ function ensureBuilderDepsLinkedAtRepoRoot( function linkIfMissing(name: string, target: string): void { const linkPath = path.join(rootNodeModules, name); - if (!fs.existsSync(linkPath)) { - fs.symlinkSync(target, linkPath, 'dir'); + try { + const stat = fs.lstatSync(linkPath); + if (stat.isSymbolicLink() && !fs.existsSync(linkPath)) { + // Remove broken symlink to avoid EEXIST error on recreation + fs.unlinkSync(linkPath); + } else { + // Symlink already exists and is valid + return; + } + } catch (e: any) { + if (e.code !== 'ENOENT') { + throw e; + } } + fs.symlinkSync(target, linkPath, 'dir'); } /** Resolves `dep` (e.g. "rxjs" or "@angular-devkit/core") to its pnpm store dir and symlinks it. */ From 136c57a796b535146e9c734f833530c17d94dea9 Mon Sep 17 00:00:00 2001 From: David Herges Date: Sat, 10 Oct 2026 06:08:50 +0000 Subject: [PATCH 3/9] fixup! test(@angular/build): add library builder scaling benchmark suite --- scripts/benchmark.mts | 36 ++++++++-- scripts/benchmarks/library-builder/index.mts | 72 +++++++++++--------- 2 files changed, 69 insertions(+), 39 deletions(-) diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts index a8318b84af11..9f3bb1e5353a 100644 --- a/scripts/benchmark.mts +++ b/scripts/benchmark.mts @@ -32,7 +32,7 @@ function checkBuildStatus(logger: Console): boolean { if (srcMtime > distMtime) { logger.warn( 'Warning: Source files in packages/angular/build are newer than dist/.\n' + - 'Run "pnpm build" to ensure your benchmark reflects your latest local edits.\n', + 'Run "pnpm build" to ensure your benchmark reflects your latest local edits.\n', ); } } @@ -54,7 +54,7 @@ async function runLibraryBuilderSubsystem(options: { }): Promise { if (options.layout !== undefined && options.layout !== 'flat' && options.layout !== 'deep') { // eslint-disable-next-line no-console - console.error(`Error: --layout must be "flat" or "deep", got "${options.layout}".`); + console.error("Error: --layout must be \"flat\" or \"deep\", got \"" + options.layout + "\"."); return 1; } @@ -64,18 +64,42 @@ async function runLibraryBuilderSubsystem(options: { ) { // eslint-disable-next-line no-console console.error( - `Error: --style must be one of "inline", "inline-scss", "external", got "${options.style}".`, + "Error: --style must be one of \"inline\", \"inline-scss\", \"external\", got \"" + options.style + "\"." ); return 1; } + const sizes = parseIntList(options.sizes); + if (sizes !== undefined && sizes.some((n) => !Number.isInteger(n) || n < 1)) { + // eslint-disable-next-line no-console + console.error("Error: --sizes must be a comma-separated list of positive integers, got \"" + options.sizes + "\"."); + + return 1; + } + + const depths = parseIntList(options.depths); + if (depths !== undefined && depths.some((n) => !Number.isInteger(n) || n < 1)) { + // eslint-disable-next-line no-console + console.error("Error: --depths must be a comma-separated list of positive integers, got \"" + options.depths + "\"."); + + return 1; + } + + const iterations = options.iterations !== undefined ? Number(options.iterations) : undefined; + if (iterations !== undefined && (!Number.isInteger(iterations) || iterations < 1)) { + // eslint-disable-next-line no-console + console.error("Error: --iterations must be a positive integer, got \"" + options.iterations + "\"."); + + return 1; + } + const cliOptions: LibraryBuilderBenchmarkOptions = { layout: options.layout as LibraryBuilderBenchmarkOptions['layout'], style: options.style as LibraryBuilderBenchmarkOptions['style'], - sizes: parseIntList(options.sizes), - depths: parseIntList(options.depths), - iterations: options.iterations !== undefined ? Number(options.iterations) : undefined, + sizes, + depths, + iterations, json: Boolean(options.json), }; diff --git a/scripts/benchmarks/library-builder/index.mts b/scripts/benchmarks/library-builder/index.mts index 9be2070b965f..2824d6c3aaf6 100644 --- a/scripts/benchmarks/library-builder/index.mts +++ b/scripts/benchmarks/library-builder/index.mts @@ -58,7 +58,7 @@ function checkBuilderIsBuilt( if (!fs.existsSync(path.join(buildPkg, 'package.json')) || !fs.existsSync(architectCli)) { logger.error( 'Error: @angular/build and/or @angular-devkit/architect have not been built yet.\n' + - 'Run "pnpm build" (or "pnpm admin build --local" for a faster local build) first.', + 'Run "pnpm build" (or "pnpm admin build --local" for a faster local build) first.', ); return undefined; @@ -171,6 +171,10 @@ function buildOnce( const durationMs = performance.now() - start; const status = result.status === 0 ? 'ok' : 'fail'; if (status === 'fail') { + if (result.error) { + // eslint-disable-next-line no-console + console.error(result.error); + } // eslint-disable-next-line no-console console.error(result.stdout?.toString()); // eslint-disable-next-line no-console @@ -207,49 +211,51 @@ export async function runLibraryBuilderBenchmarks( fs.mkdirSync(scratchRoot, { recursive: true }); const measurements: BuildMeasurement[] = []; - for (const sizeOrDepth of sizes) { - const label = `${layout}-${style}-${layout === 'deep' ? `depth${sizeOrDepth}` : sizeOrDepth}`; - const projectDir = path.join(scratchRoot, label); - - const fixtureOptions: FixtureOptions = - layout === 'flat' - ? { layout, style, count: sizeOrDepth } - : { layout, style, depth: sizeOrDepth }; - const entryPoints = generateFixture(projectDir, fixtureOptions); + try { + for (const sizeOrDepth of sizes) { + const label = layout + "-" + style + "-" + (layout === "deep" ? "depth" + sizeOrDepth : sizeOrDepth); + const projectDir = path.join(scratchRoot, label); - if (!options.json) { - // eslint-disable-next-line no-console - console.log(`\n[library-builder] ${label} (${entryPoints} entry points)`); - } + const fixtureOptions: FixtureOptions = + layout === 'flat' + ? { layout, style, count: sizeOrDepth } + : { layout, style, depth: sizeOrDepth }; + const entryPoints = generateFixture(projectDir, fixtureOptions); - const durations: number[] = []; - for (let i = 1; i <= iterations; i++) { - const { durationMs, status } = buildOnce(projectDir, builder.architectCli); if (!options.json) { + // eslint-disable-next-line no-console + console.log("\n[library-builder] " + label + " (" + entryPoints + " entry points)"); + } + + const durations: number[] = []; + for (let i = 1; i <= iterations; i++) { + const { durationMs, status } = buildOnce(projectDir, builder.architectCli); + if (!options.json) { + // eslint-disable-next-line no-console + console.log( + " run " + i + "/" + iterations + ": " + (status === 'ok' ? 'done' : 'FAILED') + " in " + (durationMs / 1000).toFixed(2) + "s" + ); + } + if (status === 'ok') { + durations.push(durationMs); + } + measurements.push({ layout, style, entryPoints, iteration: i, durationMs, status }); + } + + if (!options.json && durations.length > 0) { + const medianMs = median(durations); // eslint-disable-next-line no-console console.log( - ` run ${i}/${iterations}: ${status === 'ok' ? 'done' : 'FAILED'} in ${(durationMs / 1000).toFixed(2)}s`, + " median: " + (medianMs / 1000).toFixed(2) + "s (" + (medianMs / entryPoints).toFixed(2) + "ms/entry)" ); } - if (status === 'ok') { - durations.push(durationMs); - } - measurements.push({ layout, style, entryPoints, iteration: i, durationMs, status }); - } - if (!options.json && durations.length > 0) { - const medianMs = median(durations); - // eslint-disable-next-line no-console - console.log( - ` median: ${(medianMs / 1000).toFixed(2)}s (${(medianMs / entryPoints).toFixed(2)}ms/entry)`, - ); + fs.rmSync(projectDir, { recursive: true, force: true }); } - - fs.rmSync(projectDir, { recursive: true, force: true }); + } finally { + fs.rmSync(scratchRoot, { recursive: true, force: true }); } - fs.rmSync(scratchRoot, { recursive: true, force: true }); - const anyFailed = measurements.some((m) => m.status === 'fail'); if (options.json) { // eslint-disable-next-line no-console From 01111495a40ec5eb677d2e5c2a2d86ce1939439f Mon Sep 17 00:00:00 2001 From: David Herges Date: Sat, 10 Oct 2026 06:35:59 +0000 Subject: [PATCH 4/9] fixup! test(@angular/build): add library builder scaling benchmark suite --- scripts/benchmark.mts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts index 9f3bb1e5353a..8a8764744091 100644 --- a/scripts/benchmark.mts +++ b/scripts/benchmark.mts @@ -40,8 +40,11 @@ function checkBuildStatus(logger: Console): boolean { return true; } -function parseIntList(raw: string | undefined): number[] | undefined { - return raw?.split(',').map((s) => Number(s.trim())); +function parseIntList(raw: string | number | undefined): number[] | undefined { + if (raw === undefined) { + return undefined; + } + return String(raw).split(',').map((s) => Number(s.trim())); } async function runLibraryBuilderSubsystem(options: { From 90b8b5fc9675bcce45ece95284cebf55a1aac298 Mon Sep 17 00:00:00 2001 From: David Herges Date: Mon, 12 Oct 2026 10:27:15 +0200 Subject: [PATCH 5/9] fixup! test(@angular/build): add library builder scaling benchmark suite --- scripts/benchmark.mts | 66 +++--- scripts/benchmarks/library-builder/README.md | 166 ++++++++----- .../benchmarks/library-builder/fixtures.mts | 75 ++---- .../benchmarks/library-builder/harness.mts | 82 +++++++ scripts/benchmarks/library-builder/index.mts | 222 ++++-------------- 5 files changed, 288 insertions(+), 323 deletions(-) create mode 100644 scripts/benchmarks/library-builder/harness.mts diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts index 8a8764744091..9f5a55214eee 100644 --- a/scripts/benchmark.mts +++ b/scripts/benchmark.mts @@ -32,7 +32,7 @@ function checkBuildStatus(logger: Console): boolean { if (srcMtime > distMtime) { logger.warn( 'Warning: Source files in packages/angular/build are newer than dist/.\n' + - 'Run "pnpm build" to ensure your benchmark reflects your latest local edits.\n', + 'Run "pnpm build" to ensure your benchmark reflects your latest local edits.\n', ); } } @@ -44,47 +44,45 @@ function parseIntList(raw: string | number | undefined): number[] | undefined { if (raw === undefined) { return undefined; } - return String(raw).split(',').map((s) => Number(s.trim())); + return String(raw) + .split(',') + .map((s) => Number(s.trim())); } async function runLibraryBuilderSubsystem(options: { layout?: string; - style?: string; - sizes?: string; - depths?: string; + count?: string; + depth?: string; iterations?: string | number; json?: boolean; }): Promise { if (options.layout !== undefined && options.layout !== 'flat' && options.layout !== 'deep') { // eslint-disable-next-line no-console - console.error("Error: --layout must be \"flat\" or \"deep\", got \"" + options.layout + "\"."); + console.error('Error: --layout must be "flat" or "deep", got "' + options.layout + '".'); return 1; } - if ( - options.style !== undefined && - !['inline', 'inline-scss', 'external'].includes(options.style) - ) { + + const count = parseIntList(options.count); + if (count !== undefined && count.some((n) => !Number.isInteger(n) || n < 1)) { // eslint-disable-next-line no-console console.error( - "Error: --style must be one of \"inline\", \"inline-scss\", \"external\", got \"" + options.style + "\"." + 'Error: --count must be a comma-separated list of positive integers, got "' + + options.count + + '".', ); return 1; } - const sizes = parseIntList(options.sizes); - if (sizes !== undefined && sizes.some((n) => !Number.isInteger(n) || n < 1)) { - // eslint-disable-next-line no-console - console.error("Error: --sizes must be a comma-separated list of positive integers, got \"" + options.sizes + "\"."); - - return 1; - } - - const depths = parseIntList(options.depths); - if (depths !== undefined && depths.some((n) => !Number.isInteger(n) || n < 1)) { + const depth = parseIntList(options.depth); + if (depth !== undefined && depth.some((n) => !Number.isInteger(n) || n < 1)) { // eslint-disable-next-line no-console - console.error("Error: --depths must be a comma-separated list of positive integers, got \"" + options.depths + "\"."); + console.error( + 'Error: --depth must be a comma-separated list of positive integers, got "' + + options.depth + + '".', + ); return 1; } @@ -92,16 +90,17 @@ async function runLibraryBuilderSubsystem(options: { const iterations = options.iterations !== undefined ? Number(options.iterations) : undefined; if (iterations !== undefined && (!Number.isInteger(iterations) || iterations < 1)) { // eslint-disable-next-line no-console - console.error("Error: --iterations must be a positive integer, got \"" + options.iterations + "\"."); + console.error( + 'Error: --iterations must be a positive integer, got "' + options.iterations + '".', + ); return 1; } const cliOptions: LibraryBuilderBenchmarkOptions = { layout: options.layout as LibraryBuilderBenchmarkOptions['layout'], - style: options.style as LibraryBuilderBenchmarkOptions['style'], - sizes, - depths, + count, + depth, iterations, json: Boolean(options.json), }; @@ -127,9 +126,8 @@ export default async function ( compareBaseline?: string; 'compare-baseline'?: string; layout?: string; - style?: string; - sizes?: string; - depths?: string; + count?: string; + depth?: string; help?: boolean; [key: string]: unknown; }, @@ -163,10 +161,12 @@ Options (i18n): Options (library-builder): --layout= Fixture layout (default: flat) - --style= Component style variant (default: inline) - --sizes= Entry-point counts to sweep, layout=flat only (default: 10,50,300,1000,2000) - --depths= Tree depths to sweep, layout=deep only (default: 2,5,8,11) - --iterations= Measured iterations per size (default: 2) + --count= Entry-point count targets, drives both layouts + (default: 10,20,50,100,200,300,400,500,1000,2000) + --depth= Explicit tree depths, layout=deep only. Overrides the depths + that would otherwise be derived from --count (the smallest + depth with 2^depth >= each target count). + --iterations= Measured iterations (default: 3) --json Output raw measurements as JSON --help Show this help message diff --git a/scripts/benchmarks/library-builder/README.md b/scripts/benchmarks/library-builder/README.md index 407ba6710bc2..aa85db6f3ec7 100644 --- a/scripts/benchmarks/library-builder/README.md +++ b/scripts/benchmarks/library-builder/README.md @@ -1,77 +1,119 @@ -# `@angular/build:library` scaling benchmark +# Library Builds Scaling Benchmark -This benchmark measures how `@angular/build:library`'s cold build time scales with the number -of secondary entry points in an Angular Package Format (APF) library, across two structurally -different layouts (flat siblings vs. a deeply nested tree) and three component style variants -(inline template, inline template + styles, external template/stylesheet files). +This benchmark measures build time metrics for Angular libraries. -## Why +## Context -A comparison against [ng-packagr](https://github.com/ng-packagr/ng-packagr) (a third-party APF -build tool) found that its build time **per entry point** grows super-linearly once a library -has more than roughly 300-500 secondary entry points -- doubling the library size past that -point costs noticeably more than double the build time. This suite exists to check whether -`@angular/build:library`, a from-scratch build pipeline unrelated to ng-packagr's implementation, -shows the same growth on equivalent fixtures. +Angular CLI is in the process of transitioning from `ng-packagr` to Angular CLI's native library builder. +Both library build systems are registered Architect builders: +- `@angular/build:ng-packagr`: runs library builds through the legacy `ng-packagr` build pipeline. +- `@angular/build:library`: runs library builds through Angular CLI's native library builder. -**Finding:** it does not. Across the same sweep (flat layout: 10/50/300/1000/2000 entries; deep -layout: tree depths 2/5/8/11, i.e. 4/32/256/2048 entries), `@angular/build:library`'s per-entry -cost keeps falling as the library grows, with no climb at the largest sizes tested. It is also -1.8-13x faster than ng-packagr at matching sizes, with the gap widening as size increases. See -the PR description this suite was added alongside for the full writeup and caveats. +This benchmark suite is designed to quantify performance impacts and ensure scalability. -## Running it +### Previous Benchmarks + +For context, when running a similar benchmark suite directly against `ng-packagr`, it hinted at a "U-curve" in build times, +where build time per entry point in `ng-packagr` drops until entry point N (with N varying in the range 200..500). +This shape indicates an initial overhead as a possible explanation for a small number of entry points. +However, we discovered that `ng-packagr` build time per entry point increases +as the number of entry points scale past a certain N (in the test runs N=200+). + +[Landing Angular CLI's native `@angular/build:library` builder](https://github.com/angular/angular-cli/pull/34106#issuecomment-5744311681), +indicated very favourable performance metrics for the new library builder that can be explained by number of +architectural changes in the library build pipeline: +- Shared Stylesheet Bundling +- Public API Invalidation via Declaration Hashing +- Dedicated Worker Pool for Production Builds +- Single Pass Output Emission +- Direct Async Pipeline + +### Goals + +This benchmark suite runs explorative research on the performance of library builds in Angular CLI. +Interesting results from the previous benchmarks indicate 2 measurable goals: + +- Measure the new library builder against itself: How does it scale with the number of entry points? +- Enable A/B benchmarks of the `@angular/build:ng-packagr` builder against `@angular/build:library` builder: How much do library builds benefit from the transition to the native library builder? + +As the previous benchmark runs hint at significant performance improvements in the new library builder, +this benchmark suite aims to quantify the impact of that transition. + +## Benchmark Design + +This suite supports two primary layouts to test different filesystem and entry-point scaling scenarios: + +- **Flat (`--layout=flat`)**: Generates $N$ secondary entry points as direct siblings, where $N$ + comes straight from `--count`. + - Design: Each entry point is named `comp-i` (e.g., `comp-0001`, `comp-0002`). + +- **Deep (`--layout=deep`)**: Generates $2^N$ secondary entry points in a binary tree structure. + - Design: Creates a nested directory structure (`sub-a`/`sub-b` at each level). + - Depth $N$ is not an independent sweep parameter -- see "Sweep rounds" below. + +Each entry point consists of a `public-api.ts` and a component file with an inline template +containing a bit of per-entry content derived from the entry's id (e.g. `

Component no. +0042

`), so entries aren't byte-identical copies of each other. + +### Sweep rounds + +`--count` is the one sweep parameter: a comma-separated list of entry-point count targets. The +default sweep spans small to large entry-point counts in one round, chosen to resolve the +U-shaped scaling pattern observed with `ng-packagr` (see above): +`10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000`. + +For `--layout=deep`, there's no "number of entry points" parameter directly -- a binary tree of +depth $N$ always has exactly $2^N$ leaves/entry points. So `--count`'s targets are translated into +tree depths: for each target `n`, the smallest `depth` with `2^depth >= n`, de-duplicated (e.g. +`300`, `400`, and `500` all round up to the same `depth=9`, i.e. 512 entry points). This keeps +both layouts comparable at roughly the same entry-point scale without exposing depth as a second, +independently-tunable sweep axis. `--depth` still exists as an **explicit override** for +`--layout=deep` -- pass it directly to sweep exact tree depths instead of entry-point targets -- +but it is not part of the default sweep design and is ignored for `--layout=flat`. + +## Running the Benchmarks ```sh -pnpm build # or: pnpm admin build --local -- the benchmark measures the compiled builder, - # not TypeScript source, so it must exist at dist/@angular/build and - # dist/@angular-devkit/architect first -pnpm admin benchmark library-builder +# Ensure the builder is built first +pnpm build + +# Run the benchmark +pnpm admin benchmark library-builder [options] ``` -Options: +### Options -``` ---layout= Fixture layout (default: flat) ---style= Component style variant (default: inline) ---sizes= Entry-point counts to sweep, layout=flat only (default: 10,50,300,1000,2000) ---depths= Tree depths to sweep, layout=deep only (default: 2,5,8,11) ---iterations= Measured iterations per size (default: 2) ---json Output raw measurements as JSON instead of a human-readable log -``` +| Option | Description | Default | +| :--------------------- | :----------------------------------------------- | :---------------------------------------- | +| `--layout=` | Fixture layout | `flat` | +| `--count=` | Entry-point count targets, drives both layouts | `10,20,50,100,200,300,400,500,1000,2000` | +| `--depth=` | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count` | derived from `--count` | +| `--iterations=` | Measured iterations per size | `3` | +| `--json` | Output raw measurements as JSON | N/A | -Example: sweep the external-style variant at a few sizes with 3 iterations each: +> `--count` and `--depth` are not independent sweep axes for `layout=deep`: `--count` is translated +> into depths (smallest `depth` with `2^depth >= n` per target, de-duplicated); `--depth` only +> matters when you want to bypass that translation and name exact tree depths yourself. Passing +> both for `layout=deep` makes `--depth` win; `--depth` is ignored entirely for `layout=flat`. + +### Example + +Sweep the flat layout at a single size: ```sh -pnpm admin benchmark library-builder --style=external --sizes=100,500,1000 --iterations=3 +pnpm admin benchmark library-builder --layout=flat --count=500 ``` -## How it works - -- `fixtures.mts` generates a synthetic library at a given size/layout/style: a primary entry - point, N secondary entry points (each a minimal standalone Angular component), and the - `angular.json`/`tsconfig.lib.json`/`package.json` scaffolding `@angular/build:library` needs. - Secondary entry points are declared as a single flat `exports` map in the root `package.json` - (the builder's actual convention), not as separate per-entry config files. -- `index.mts` times `architect lib:build` (the standalone `@angular-devkit/architect` CLI, - invoked directly -- no `ng` CLI needed) once per iteration via `child_process.spawnSync`, - discards the generated project afterwards, and reports per-size median build time. -- Because `@angular/build` and `@angular-devkit/architect` are Bazel-built workspace packages - rather than ordinary npm dependencies, a plain `pnpm install` never links them (or their own - dependencies) into the repo's root `node_modules` -- nothing in `dist/` is resolvable via a - bare `require()`/`import` otherwise. On first run, `index.mts` symlinks `@angular/build` and - `@angular-devkit/architect` from `dist/` into the root `node_modules`, then walks their - declared `dependencies` and links any of those still missing from the pnpm store. This only - touches `node_modules` (already gitignored) and is idempotent -- safe to run repeatedly or - alongside other work in the same checkout. - -## Methodology notes - -- Each reported number is the **median** of the measured iterations, not the mean -- process-spawn - wall-clock timings are noisy and right-skewed, and the median is more representative of steady - state than an outlier-sensitive mean. -- Results are specific to the machine they were collected on. Run on one otherwise-idle machine - for a self-consistent sweep; absolute numbers will differ across hardware, though the _shape_ - of the curve (does per-entry cost grow with size, or not) is the more portable finding. -- This benchmark only covers **cold, one-shot builds**. It does not exercise incremental rebuilds - or watch mode. +## Benchmark Results + +_(To be filled in once a sweep has been run and reviewed. Out of scope for this iteration.)_ + +## Future Work + +- Style-flavor matrix: inline / inline-css / inline-scss / external / external-css / external-scss, swept across both layouts. +- A/B comparison against `ng-packagr` (both are already registered Architect builders — `@angular/build:ng-packagr` and `@angular/build:library` — so this is a same-builder-package comparison, not a separate tool invocation). +- Dependencies between entry points. +- Shared stylesheet tokens imported by multiple components. +- Watch mode. +- Peak memory (`ru_maxrss`) and CPU time (`ru_utime` + `ru_stime`) metrics per build, in addition to wall-clock build time. +- Library "taste" variants beyond plain flat/deep, e.g. flat/deep with inter-entry-point dependencies or shared SCSS tokens (currently out of scope, listed above). diff --git a/scripts/benchmarks/library-builder/fixtures.mts b/scripts/benchmarks/library-builder/fixtures.mts index 940d570e6984..8a9bd1085020 100644 --- a/scripts/benchmarks/library-builder/fixtures.mts +++ b/scripts/benchmarks/library-builder/fixtures.mts @@ -15,11 +15,9 @@ import fs from 'node:fs'; import path from 'node:path'; export type Layout = 'flat' | 'deep'; -export type Style = 'inline' | 'inline-scss' | 'external'; export interface FixtureOptions { layout: Layout; - style: Style; /** Number of secondary entry points for `layout: 'flat'`. */ count?: number; @@ -28,36 +26,19 @@ export interface FixtureOptions { depth?: number; } -function buildComponentSource( - className: string, - selector: string, - compBase: string, - id: string, - style: Style, -): string { - const metadataLines = [` selector: '${selector}',`]; - - if (style === 'external') { - metadataLines.push(` templateUrl: './${compBase}.component.html',`); - metadataLines.push(` styleUrls: ['./${compBase}.component.scss'],`); - } else { - metadataLines.push(` template: '
Reference component ${id}
',`); - if (style === 'inline-scss') { - metadataLines.push(` styles: ['.ref-comp { display: block; padding: 4px; }'],`); - } - } - +function buildComponentSource(className: string, selector: string, id: string): string { return `import { Component } from '@angular/core'; @Component({ -${metadataLines.join('\n')} + selector: '${selector}', + template: '

Component no. ${id}

', }) export class ${className} {} `; } /** Writes one secondary entry point's source files (component + public-api.ts) into `entryDir`. */ -function writeEntryPointSource(entryDir: string, id: string, style: Style): void { +function writeEntryPointSource(entryDir: string, id: string): void { fs.mkdirSync(entryDir, { recursive: true }); const className = `RefComponent${id}`; @@ -69,31 +50,20 @@ function writeEntryPointSource(entryDir: string, id: string, style: Style): void `export * from './${compBase}.component';\n`, ); - if (style === 'external') { - fs.writeFileSync( - path.join(entryDir, `${compBase}.component.html`), - `
Reference component ${id}
\n`, - ); - fs.writeFileSync( - path.join(entryDir, `${compBase}.component.scss`), - `.ref-comp {\n display: block;\n padding: 4px;\n}\n`, - ); - } - fs.writeFileSync( path.join(entryDir, `${compBase}.component.ts`), - buildComponentSource(className, selector, compBase, id, style), + buildComponentSource(className, selector, id), ); } /** Generates `count` flat secondary entry points as direct siblings, returning their relative dirs. */ -function generateFlatEntries(outDir: string, count: number, style: Style): string[] { +function generateFlatEntries(outDir: string, count: number): string[] { const width = String(count).length; const dirs: string[] = []; for (let i = 1; i <= count; i++) { const id = String(i).padStart(width, '0'); const relDir = `comp-${id}`; - writeEntryPointSource(path.join(outDir, relDir), id, style); + writeEntryPointSource(path.join(outDir, relDir), id); dirs.push(relDir); } @@ -104,7 +74,7 @@ function generateFlatEntries(outDir: string, count: number, style: Style): strin * Generates a binary tree `depth` levels deep (`sub-a`/`sub-b` at each level), placing one * secondary entry point at each of the 2^depth leaves. Returns their relative dirs. */ -function generateDeepEntries(outDir: string, depth: number, style: Style): string[] { +function generateDeepEntries(outDir: string, depth: number): string[] { const total = 2 ** depth; const width = String(total).length; const dirs: string[] = []; @@ -114,7 +84,7 @@ function generateDeepEntries(outDir: string, depth: number, style: Style): strin if (level === depth) { counter++; const id = String(counter).padStart(width, '0'); - writeEntryPointSource(path.join(outDir, currentRelDir), id, style); + writeEntryPointSource(path.join(outDir, currentRelDir), id); dirs.push(currentRelDir); return; @@ -150,28 +120,33 @@ export function generateFixture(outDir: string, options: FixtureOptions): number const secondaryDirs = options.layout === 'flat' - ? generateFlatEntries(outDir, options.count ?? 10, options.style) - : generateDeepEntries(outDir, options.depth ?? 2, options.style); + ? generateFlatEntries(outDir, options.count ?? 10) + : generateDeepEntries(outDir, options.depth ?? 2); const exportsMap: Record = { '.': './public-api.ts' }; for (const relDir of secondaryDirs) { exportsMap[`./${entryKey(relDir)}`] = `./${relDir.split(path.sep).join('/')}/public-api.ts`; } - const angularCoreVersion = JSON.parse( - fs.readFileSync( - new URL('../../../node_modules/@angular/core/package.json', import.meta.url), - 'utf8' - ) - ).version as string; - // e.g. "22.3.0-next.0" -> "22.3.0-next" (drop the trailing prerelease build number only). - const peerRange = `^${angularCoreVersion.replace(/\.\d+$/, '')}`; + // Fallback to a hardcoded version if the path lookup fails in the sandbox/environment. + let peerRange = '^22.0.0'; + try { + const angularCoreVersion = JSON.parse( + fs.readFileSync( + new URL('../../../node_modules/@angular/core/package.json', import.meta.url), + 'utf8', + ), + ).version as string; + peerRange = `^${angularCoreVersion.replace(/\.\d+$/, '')}`; + } catch { + // Ignore error, use fallback + } fs.writeFileSync( path.join(outDir, 'package.json'), JSON.stringify( { - name: `ref-${options.layout}-${options.style}`, + name: `ref-${options.layout}`, version: '0.0.1', peerDependencies: { '@angular/core': peerRange, diff --git a/scripts/benchmarks/library-builder/harness.mts b/scripts/benchmarks/library-builder/harness.mts new file mode 100644 index 000000000000..742d25e239b4 --- /dev/null +++ b/scripts/benchmarks/library-builder/harness.mts @@ -0,0 +1,82 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Runs and measures a single `@angular/build:library` cold build via Architect's standalone CLI. +// `fixtures.mts` owns generating the on-disk project; this module owns spawning the builder +// against it and reporting what happened. Kept separate so each file has one job. + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import type { Layout } from './fixtures.mts'; + +export interface BuildMeasurement { + layout: Layout; + entryPoints: number; + iteration: number; + durationMs: number; + status: 'ok' | 'fail'; +} + +export interface BuilderPaths { + buildPkg: string; + architectPkg: string; + architectCli: string; +} + +/** Verifies `@angular/build` and `@angular-devkit/architect` are built before attempting a build. */ +export function checkBuilderIsBuilt(repoRoot: string, logger: Console): BuilderPaths | undefined { + const buildPkg = path.join(repoRoot, 'dist', '@angular', 'build'); + const architectPkg = path.join(repoRoot, 'dist', '@angular-devkit', 'architect'); + const architectCli = path.join(architectPkg, 'bin', 'cli.js'); + + if (!fs.existsSync(path.join(buildPkg, 'package.json')) || !fs.existsSync(architectCli)) { + logger.error( + 'Error: @angular/build and/or @angular-devkit/architect have not been built yet.\n' + + 'Run "pnpm build" (or "pnpm admin build --local" for a faster local build) first.', + ); + + return undefined; + } + + return { buildPkg, architectPkg, architectCli }; +} + +/** Spawns one cold `lib:build` run against `projectDir` and times it. */ +export function buildOnce( + projectDir: string, + architectCli: string, +): { durationMs: number; status: 'ok' | 'fail' } { + const start = performance.now(); + const result = spawnSync(process.execPath, [architectCli, 'lib:build'], { + cwd: projectDir, + stdio: 'pipe', + maxBuffer: Infinity, + }); + const durationMs = performance.now() - start; + const status = result.status === 0 ? 'ok' : 'fail'; + if (status === 'fail') { + if (result.error) { + // eslint-disable-next-line no-console + console.error(result.error); + } + // eslint-disable-next-line no-console + console.error(result.stdout?.toString()); + // eslint-disable-next-line no-console + console.error(result.stderr?.toString()); + } + + return { durationMs, status }; +} + +export function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} diff --git a/scripts/benchmarks/library-builder/index.mts b/scripts/benchmarks/library-builder/index.mts index 2824d6c3aaf6..79358e8d1192 100644 --- a/scripts/benchmarks/library-builder/index.mts +++ b/scripts/benchmarks/library-builder/index.mts @@ -7,224 +7,78 @@ */ // Benchmarks `@angular/build:library`'s cold build time across a sweep of library sizes. -// -// Context: a comparison against ng-packagr (a third-party APF build tool) found that its -// per-entry-point build cost grows super-linearly past roughly 300-500 secondary entry points. -// This suite exists to check whether `@angular/build:library` -- a from-scratch build pipeline, -// unrelated to ng-packagr's implementation -- shows the same growth on equivalent fixtures. -// See scripts/benchmarks/library-builder/README.md for the full methodology and findings. +// This module only orchestrates the sweep: fixture generation lives in fixtures.mts, build +// execution and measurement live in harness.mts. -import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; -import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; -import { type FixtureOptions, type Layout, type Style, generateFixture } from './fixtures.mts'; +import { type FixtureOptions, type Layout, generateFixture } from './fixtures.mts'; +import { type BuildMeasurement, buildOnce, checkBuilderIsBuilt, median } from './harness.mts'; export interface LibraryBuilderBenchmarkOptions { layout?: Layout; - style?: Style; - - /** Comma-separated entry-point counts, for `layout: 'flat'`. */ - sizes?: number[]; - - /** Comma-separated tree depths, for `layout: 'deep'`. */ - depths?: number[]; + count?: number[]; + depth?: number[]; iterations?: number; json?: boolean; } -export interface BuildMeasurement { - layout: Layout; - style: Style; - entryPoints: number; - iteration: number; - durationMs: number; - status: 'ok' | 'fail'; -} - const repoRoot = fileURLToPath(new URL('../../../', import.meta.url)); -const scratchRoot = path.join(repoRoot, 'dist', '.library-builder-benchmark-tmp'); -const rootNodeModules = path.join(repoRoot, 'node_modules'); -const pnpmStore = path.join(rootNodeModules, '.pnpm'); - -function checkBuilderIsBuilt( - logger: Console, -): { buildPkg: string; architectPkg: string; architectCli: string } | undefined { - const buildPkg = path.join(repoRoot, 'dist', '@angular', 'build'); - const architectPkg = path.join(repoRoot, 'dist', '@angular-devkit', 'architect'); - const architectCli = path.join(architectPkg, 'bin', 'cli.js'); - - if (!fs.existsSync(path.join(buildPkg, 'package.json')) || !fs.existsSync(architectCli)) { - logger.error( - 'Error: @angular/build and/or @angular-devkit/architect have not been built yet.\n' + - 'Run "pnpm build" (or "pnpm admin build --local" for a faster local build) first.', - ); - - return undefined; - } - - return { buildPkg, architectPkg, architectCli }; -} +const scratchRoot = path.join(repoRoot, 'dist', '.library-benchmark-fixtures'); /** - * `@angular/build` and `@angular-devkit/architect` are Bazel-built workspace packages, not - * ordinary npm dependencies -- a plain `pnpm install` never links them (or their own - * dependencies) into the root node_modules. Resolving a plain `require('@angular/build')` - * from anywhere therefore needs these links to exist at least once, at the repo root (Node's - * module resolution walks up from each required file's real, on-disk location, so per-fixture - * symlinks are not enough once a dependency itself does a bare `require`). - * - * This walks the dependency graph starting at `@angular/build` and `@angular-devkit/architect` - * and links any package missing from node_modules, resolving each from the pnpm store so it - * doesn't go stale if a dependency version changes. + * Default entry-point counts for the `flat` layout, chosen to resolve a U-shaped scaling curve. + * Also drives the `deep` layout's default depths (see `depthsForEntryPointTargets`) -- `--count` + * is the one sweep parameter users pick; `--depth` is only ever an explicit override. */ -function ensureBuilderDepsLinkedAtRepoRoot( - logger: Console, - { buildPkg, architectPkg }: { buildPkg: string; architectPkg: string }, -): void { - fs.mkdirSync(path.join(rootNodeModules, '@angular'), { recursive: true }); - fs.mkdirSync(path.join(rootNodeModules, '@angular-devkit'), { recursive: true }); - linkIfMissing('@angular/build', buildPkg); - linkIfMissing('@angular-devkit/architect', architectPkg); - - const seen = new Set(['@angular/build', '@angular-devkit/architect']); - const queue: string[] = ['@angular/build', '@angular-devkit/architect']; - while (queue.length > 0) { - const pkgName = queue.shift(); - if (pkgName === undefined) { - break; - } - const pkgJsonPath = path.join(rootNodeModules, pkgName, 'package.json'); - if (!fs.existsSync(pkgJsonPath)) { - continue; - } - const deps = Object.keys( - (JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8')).dependencies ?? {}) as Record< - string, - string - >, - ); - for (const dep of deps) { - if (seen.has(dep)) { - continue; - } - seen.add(dep); - queue.push(dep); - if (!fs.existsSync(path.join(rootNodeModules, dep))) { - linkFromPnpmStore(logger, dep); - } - } - } -} +const DEFAULT_FLAT_COUNTS = [10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000]; -function linkIfMissing(name: string, target: string): void { - const linkPath = path.join(rootNodeModules, name); - try { - const stat = fs.lstatSync(linkPath); - if (stat.isSymbolicLink() && !fs.existsSync(linkPath)) { - // Remove broken symlink to avoid EEXIST error on recreation - fs.unlinkSync(linkPath); - } else { - // Symlink already exists and is valid - return; - } - } catch (e: any) { - if (e.code !== 'ENOENT') { - throw e; - } - } - fs.symlinkSync(target, linkPath, 'dir'); -} - -/** Resolves `dep` (e.g. "rxjs" or "@angular-devkit/core") to its pnpm store dir and symlinks it. */ -function linkFromPnpmStore(logger: Console, dep: string): void { - const storeName = dep.startsWith('@') ? dep.replace('/', '+') : dep; - const candidates = fs.existsSync(pnpmStore) - ? fs.readdirSync(pnpmStore).filter((entry) => entry.startsWith(`${storeName}@`)) - : []; - if (candidates.length === 0) { - logger.warn(`Warning: could not resolve "${dep}" from the pnpm store; it may be missing.`); - - return; - } - // Prefer the lexically-last match (newest version) when more than one is installed. - candidates.sort(); - const storeDir = candidates[candidates.length - 1]; - const target = path.join(pnpmStore, storeDir, 'node_modules', dep); - if (dep.includes('/')) { - fs.mkdirSync(path.join(rootNodeModules, path.dirname(dep)), { recursive: true }); - } - linkIfMissing(dep, target); -} - -function buildOnce( - projectDir: string, - architectCli: string, -): { durationMs: number; status: 'ok' | 'fail' } { - const start = performance.now(); - const result = spawnSync(process.execPath, [architectCli, 'lib:build'], { - cwd: projectDir, - stdio: 'pipe', - maxBuffer: Infinity, - }); - const durationMs = performance.now() - start; - const status = result.status === 0 ? 'ok' : 'fail'; - if (status === 'fail') { - if (result.error) { - // eslint-disable-next-line no-console - console.error(result.error); - } - // eslint-disable-next-line no-console - console.error(result.stdout?.toString()); - // eslint-disable-next-line no-console - console.error(result.stderr?.toString()); +/** + * Derives default `deep` layout tree depths from a set of target entry-point counts, picking for + * each target `n` the smallest `depth` with `2^depth >= n`, de-duplicated in ascending order. + * This keeps both layouts comparable at roughly the same entry-point scale instead of an + * arbitrary, independently-chosen depth list. + */ +function depthsForEntryPointTargets(targets: number[]): number[] { + const depths = new Set(); + for (const target of targets) { + depths.add(Math.max(1, Math.ceil(Math.log2(target)))); } - return { durationMs, status }; -} - -function median(values: number[]): number { - const sorted = [...values].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - - return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; + return [...depths].sort((a, b) => a - b); } export async function runLibraryBuilderBenchmarks( options: LibraryBuilderBenchmarkOptions = {}, ): Promise<{ measurements: BuildMeasurement[]; exitCode: number }> { - const builder = checkBuilderIsBuilt(console); + const builder = checkBuilderIsBuilt(repoRoot, console); if (!builder) { return { measurements: [], exitCode: 1 }; } - ensureBuilderDepsLinkedAtRepoRoot(console, builder); const layout = options.layout ?? 'flat'; - const style = options.style ?? 'inline'; - const iterations = options.iterations ?? 2; + const iterations = options.iterations ?? 3; const sizes = layout === 'flat' - ? (options.sizes ?? [10, 50, 300, 1000, 2000]) - : (options.depths ?? [2, 5, 8, 11]); + ? (options.count ?? DEFAULT_FLAT_COUNTS) + : (options.depth ?? depthsForEntryPointTargets(options.count ?? DEFAULT_FLAT_COUNTS)); fs.mkdirSync(scratchRoot, { recursive: true }); const measurements: BuildMeasurement[] = []; try { for (const sizeOrDepth of sizes) { - const label = layout + "-" + style + "-" + (layout === "deep" ? "depth" + sizeOrDepth : sizeOrDepth); + const label = layout + '-' + (layout === 'deep' ? 'depth' + sizeOrDepth : sizeOrDepth); const projectDir = path.join(scratchRoot, label); const fixtureOptions: FixtureOptions = - layout === 'flat' - ? { layout, style, count: sizeOrDepth } - : { layout, style, depth: sizeOrDepth }; + layout === 'flat' ? { layout, count: sizeOrDepth } : { layout, depth: sizeOrDepth }; const entryPoints = generateFixture(projectDir, fixtureOptions); if (!options.json) { // eslint-disable-next-line no-console - console.log("\n[library-builder] " + label + " (" + entryPoints + " entry points)"); + console.log('\n[library-builder] ' + label + ' (' + entryPoints + ' entry points)'); } const durations: number[] = []; @@ -233,20 +87,32 @@ export async function runLibraryBuilderBenchmarks( if (!options.json) { // eslint-disable-next-line no-console console.log( - " run " + i + "/" + iterations + ": " + (status === 'ok' ? 'done' : 'FAILED') + " in " + (durationMs / 1000).toFixed(2) + "s" + ' run ' + + i + + '/' + + iterations + + ': ' + + (status === 'ok' ? 'done' : 'FAILED') + + ' in ' + + (durationMs / 1000).toFixed(2) + + 's', ); } if (status === 'ok') { durations.push(durationMs); } - measurements.push({ layout, style, entryPoints, iteration: i, durationMs, status }); + measurements.push({ layout, entryPoints, iteration: i, durationMs, status }); } if (!options.json && durations.length > 0) { const medianMs = median(durations); // eslint-disable-next-line no-console console.log( - " median: " + (medianMs / 1000).toFixed(2) + "s (" + (medianMs / entryPoints).toFixed(2) + "ms/entry)" + ' median: ' + + (medianMs / 1000).toFixed(2) + + 's (' + + (medianMs / entryPoints).toFixed(2) + + 'ms/entry)', ); } From 25e62c916edbfc8e8cc6efff80377579481250ce Mon Sep 17 00:00:00 2001 From: David Herges Date: Mon, 12 Oct 2026 10:35:59 +0200 Subject: [PATCH 6/9] fixup! test(@angular/build): add library builder scaling benchmark suite --- scripts/benchmarks/library-builder/README.md | 238 ++++----- .../benchmarks/library-builder/fixtures.mts | 464 +++++++++--------- .../benchmarks/library-builder/harness.mts | 164 +++---- scripts/benchmarks/library-builder/index.mts | 264 +++++----- 4 files changed, 565 insertions(+), 565 deletions(-) diff --git a/scripts/benchmarks/library-builder/README.md b/scripts/benchmarks/library-builder/README.md index aa85db6f3ec7..73c56fe634ea 100644 --- a/scripts/benchmarks/library-builder/README.md +++ b/scripts/benchmarks/library-builder/README.md @@ -1,119 +1,119 @@ -# Library Builds Scaling Benchmark - -This benchmark measures build time metrics for Angular libraries. - -## Context - -Angular CLI is in the process of transitioning from `ng-packagr` to Angular CLI's native library builder. -Both library build systems are registered Architect builders: -- `@angular/build:ng-packagr`: runs library builds through the legacy `ng-packagr` build pipeline. -- `@angular/build:library`: runs library builds through Angular CLI's native library builder. - -This benchmark suite is designed to quantify performance impacts and ensure scalability. - -### Previous Benchmarks - -For context, when running a similar benchmark suite directly against `ng-packagr`, it hinted at a "U-curve" in build times, -where build time per entry point in `ng-packagr` drops until entry point N (with N varying in the range 200..500). -This shape indicates an initial overhead as a possible explanation for a small number of entry points. -However, we discovered that `ng-packagr` build time per entry point increases -as the number of entry points scale past a certain N (in the test runs N=200+). - -[Landing Angular CLI's native `@angular/build:library` builder](https://github.com/angular/angular-cli/pull/34106#issuecomment-5744311681), -indicated very favourable performance metrics for the new library builder that can be explained by number of -architectural changes in the library build pipeline: -- Shared Stylesheet Bundling -- Public API Invalidation via Declaration Hashing -- Dedicated Worker Pool for Production Builds -- Single Pass Output Emission -- Direct Async Pipeline - -### Goals - -This benchmark suite runs explorative research on the performance of library builds in Angular CLI. -Interesting results from the previous benchmarks indicate 2 measurable goals: - -- Measure the new library builder against itself: How does it scale with the number of entry points? -- Enable A/B benchmarks of the `@angular/build:ng-packagr` builder against `@angular/build:library` builder: How much do library builds benefit from the transition to the native library builder? - -As the previous benchmark runs hint at significant performance improvements in the new library builder, -this benchmark suite aims to quantify the impact of that transition. - -## Benchmark Design - -This suite supports two primary layouts to test different filesystem and entry-point scaling scenarios: - -- **Flat (`--layout=flat`)**: Generates $N$ secondary entry points as direct siblings, where $N$ - comes straight from `--count`. - - Design: Each entry point is named `comp-i` (e.g., `comp-0001`, `comp-0002`). - -- **Deep (`--layout=deep`)**: Generates $2^N$ secondary entry points in a binary tree structure. - - Design: Creates a nested directory structure (`sub-a`/`sub-b` at each level). - - Depth $N$ is not an independent sweep parameter -- see "Sweep rounds" below. - -Each entry point consists of a `public-api.ts` and a component file with an inline template -containing a bit of per-entry content derived from the entry's id (e.g. `

Component no. -0042

`), so entries aren't byte-identical copies of each other. - -### Sweep rounds - -`--count` is the one sweep parameter: a comma-separated list of entry-point count targets. The -default sweep spans small to large entry-point counts in one round, chosen to resolve the -U-shaped scaling pattern observed with `ng-packagr` (see above): -`10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000`. - -For `--layout=deep`, there's no "number of entry points" parameter directly -- a binary tree of -depth $N$ always has exactly $2^N$ leaves/entry points. So `--count`'s targets are translated into -tree depths: for each target `n`, the smallest `depth` with `2^depth >= n`, de-duplicated (e.g. -`300`, `400`, and `500` all round up to the same `depth=9`, i.e. 512 entry points). This keeps -both layouts comparable at roughly the same entry-point scale without exposing depth as a second, -independently-tunable sweep axis. `--depth` still exists as an **explicit override** for -`--layout=deep` -- pass it directly to sweep exact tree depths instead of entry-point targets -- -but it is not part of the default sweep design and is ignored for `--layout=flat`. - -## Running the Benchmarks - -```sh -# Ensure the builder is built first -pnpm build - -# Run the benchmark -pnpm admin benchmark library-builder [options] -``` - -### Options - -| Option | Description | Default | -| :--------------------- | :----------------------------------------------- | :---------------------------------------- | -| `--layout=` | Fixture layout | `flat` | -| `--count=` | Entry-point count targets, drives both layouts | `10,20,50,100,200,300,400,500,1000,2000` | -| `--depth=` | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count` | derived from `--count` | -| `--iterations=` | Measured iterations per size | `3` | -| `--json` | Output raw measurements as JSON | N/A | - -> `--count` and `--depth` are not independent sweep axes for `layout=deep`: `--count` is translated -> into depths (smallest `depth` with `2^depth >= n` per target, de-duplicated); `--depth` only -> matters when you want to bypass that translation and name exact tree depths yourself. Passing -> both for `layout=deep` makes `--depth` win; `--depth` is ignored entirely for `layout=flat`. - -### Example - -Sweep the flat layout at a single size: - -```sh -pnpm admin benchmark library-builder --layout=flat --count=500 -``` - -## Benchmark Results - -_(To be filled in once a sweep has been run and reviewed. Out of scope for this iteration.)_ - -## Future Work - -- Style-flavor matrix: inline / inline-css / inline-scss / external / external-css / external-scss, swept across both layouts. -- A/B comparison against `ng-packagr` (both are already registered Architect builders — `@angular/build:ng-packagr` and `@angular/build:library` — so this is a same-builder-package comparison, not a separate tool invocation). -- Dependencies between entry points. -- Shared stylesheet tokens imported by multiple components. -- Watch mode. -- Peak memory (`ru_maxrss`) and CPU time (`ru_utime` + `ru_stime`) metrics per build, in addition to wall-clock build time. -- Library "taste" variants beyond plain flat/deep, e.g. flat/deep with inter-entry-point dependencies or shared SCSS tokens (currently out of scope, listed above). +# Library Builds Scaling Benchmark + +This benchmark measures build time metrics for Angular libraries. + +## Context + +Angular CLI is in the process of transitioning from `ng-packagr` to Angular CLI's native library builder. +Both library build systems are registered Architect builders: +- `@angular/build:ng-packagr`: runs library builds through the legacy `ng-packagr` build pipeline. +- `@angular/build:library`: runs library builds through Angular CLI's native library builder. + +This benchmark suite is designed to quantify performance impacts and ensure scalability. + +### Previous Benchmarks + +For context, when running a similar benchmark suite directly against `ng-packagr`, it hinted at a "U-curve" in build times, +where build time per entry point in `ng-packagr` drops until entry point N (with N varying in the range 200..500). +This shape indicates an initial overhead as a possible explanation for a small number of entry points. +However, we discovered that `ng-packagr` build time per entry point increases +as the number of entry points scale past a certain N (in the test runs N=200+). + +[Landing Angular CLI's native `@angular/build:library` builder](https://github.com/angular/angular-cli/pull/34106#issuecomment-5744311681), +indicated very favourable performance metrics for the new library builder that can be explained by number of +architectural changes in the library build pipeline: +- Shared Stylesheet Bundling +- Public API Invalidation via Declaration Hashing +- Dedicated Worker Pool for Production Builds +- Single Pass Output Emission +- Direct Async Pipeline + +### Goals + +This benchmark suite runs explorative research on the performance of library builds in Angular CLI. +Interesting results from the previous benchmarks indicate 2 measurable goals: + +- Measure the new library builder against itself: How does it scale with the number of entry points? +- Enable A/B benchmarks of the `@angular/build:ng-packagr` builder against `@angular/build:library` builder: How much do library builds benefit from the transition to the native library builder? + +As the previous benchmark runs hint at significant performance improvements in the new library builder, +this benchmark suite aims to quantify the impact of that transition. + +## Benchmark Design + +This suite supports two primary layouts to test different filesystem and entry-point scaling scenarios: + +- **Flat (`--layout=flat`)**: Generates $N$ secondary entry points as direct siblings, where $N$ + comes straight from `--count`. + - Design: Each entry point is named `comp-i` (e.g., `comp-0001`, `comp-0002`). + +- **Deep (`--layout=deep`)**: Generates $2^N$ secondary entry points in a binary tree structure. + - Design: Creates a nested directory structure (`sub-a`/`sub-b` at each level). + - Depth $N$ is not an independent sweep parameter -- see "Sweep rounds" below. + +Each entry point consists of a `public-api.ts` and a component file with an inline template +containing a bit of per-entry content derived from the entry's id (e.g. `

Component no. +0042

`), so entries aren't byte-identical copies of each other. + +### Sweep rounds + +`--count` is the one sweep parameter: a comma-separated list of entry-point count targets. The +default sweep spans small to large entry-point counts in one round, chosen to resolve the +U-shaped scaling pattern observed with `ng-packagr` (see above): +`10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000`. + +For `--layout=deep`, there's no "number of entry points" parameter directly -- a binary tree of +depth $N$ always has exactly $2^N$ leaves/entry points. So `--count`'s targets are translated into +tree depths: for each target `n`, the smallest `depth` with `2^depth >= n`, de-duplicated (e.g. +`300`, `400`, and `500` all round up to the same `depth=9`, i.e. 512 entry points). This keeps +both layouts comparable at roughly the same entry-point scale without exposing depth as a second, +independently-tunable sweep axis. `--depth` still exists as an **explicit override** for +`--layout=deep` -- pass it directly to sweep exact tree depths instead of entry-point targets -- +but it is not part of the default sweep design and is ignored for `--layout=flat`. + +## Running the Benchmarks + +```sh +# Ensure the builder is built first +pnpm build + +# Run the benchmark +pnpm admin benchmark library-builder [options] +``` + +### Options + +| Option | Description | Default | +| :--------------------- | :----------------------------------------------- | :---------------------------------------- | +| `--layout=` | Fixture layout | `flat` | +| `--count=` | Entry-point count targets, drives both layouts | `10,20,50,100,200,300,400,500,1000,2000` | +| `--depth=` | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count` | derived from `--count` | +| `--iterations=` | Measured iterations per size | `3` | +| `--json` | Output raw measurements as JSON | N/A | + +> `--count` and `--depth` are not independent sweep axes for `layout=deep`: `--count` is translated +> into depths (smallest `depth` with `2^depth >= n` per target, de-duplicated); `--depth` only +> matters when you want to bypass that translation and name exact tree depths yourself. Passing +> both for `layout=deep` makes `--depth` win; `--depth` is ignored entirely for `layout=flat`. + +### Example + +Sweep the flat layout at a single size: + +```sh +pnpm admin benchmark library-builder --layout=flat --count=500 +``` + +## Benchmark Results + +_(To be filled in once a sweep has been run and reviewed. Out of scope for this iteration.)_ + +## Future Work + +- Style-flavor matrix: inline / inline-css / inline-scss / external / external-css / external-scss, swept across both layouts. +- A/B comparison against `ng-packagr` (both are already registered Architect builders — `@angular/build:ng-packagr` and `@angular/build:library` — so this is a same-builder-package comparison, not a separate tool invocation). +- Dependencies between entry points. +- Shared stylesheet tokens imported by multiple components. +- Watch mode. +- Peak memory (`ru_maxrss`) and CPU time (`ru_utime` + `ru_stime`) metrics per build, in addition to wall-clock build time. +- Library "taste" variants beyond plain flat/deep, e.g. flat/deep with inter-entry-point dependencies or shared SCSS tokens (currently out of scope, listed above). diff --git a/scripts/benchmarks/library-builder/fixtures.mts b/scripts/benchmarks/library-builder/fixtures.mts index 8a9bd1085020..9e661a00dc8e 100644 --- a/scripts/benchmarks/library-builder/fixtures.mts +++ b/scripts/benchmarks/library-builder/fixtures.mts @@ -1,232 +1,232 @@ -/** - * @license - * Copyright Google LLC All Rights Reserved. - * - * Use of this source code is governed by an MIT-style license that can be - * found in the LICENSE file at https://angular.dev/license - */ - -// Generates synthetic Angular Package Format (APF) library fixtures for benchmarking -// `@angular/build:library`, and converts them into the builder's project shape (a flat -// `exports` map in one root package.json, rather than the per-entry-point `ng-package.json` -// convention used by third-party tools such as ng-packagr). - -import fs from 'node:fs'; -import path from 'node:path'; - -export type Layout = 'flat' | 'deep'; - -export interface FixtureOptions { - layout: Layout; - - /** Number of secondary entry points for `layout: 'flat'`. */ - count?: number; - - /** Binary-tree depth for `layout: 'deep'` (2^depth leaf entry points). */ - depth?: number; -} - -function buildComponentSource(className: string, selector: string, id: string): string { - return `import { Component } from '@angular/core'; - -@Component({ - selector: '${selector}', - template: '

Component no. ${id}

', -}) -export class ${className} {} -`; -} - -/** Writes one secondary entry point's source files (component + public-api.ts) into `entryDir`. */ -function writeEntryPointSource(entryDir: string, id: string): void { - fs.mkdirSync(entryDir, { recursive: true }); - - const className = `RefComponent${id}`; - const selector = `ref-comp-${id}`; - const compBase = `comp-${id}`; - - fs.writeFileSync( - path.join(entryDir, 'public-api.ts'), - `export * from './${compBase}.component';\n`, - ); - - fs.writeFileSync( - path.join(entryDir, `${compBase}.component.ts`), - buildComponentSource(className, selector, id), - ); -} - -/** Generates `count` flat secondary entry points as direct siblings, returning their relative dirs. */ -function generateFlatEntries(outDir: string, count: number): string[] { - const width = String(count).length; - const dirs: string[] = []; - for (let i = 1; i <= count; i++) { - const id = String(i).padStart(width, '0'); - const relDir = `comp-${id}`; - writeEntryPointSource(path.join(outDir, relDir), id); - dirs.push(relDir); - } - - return dirs; -} - -/** - * Generates a binary tree `depth` levels deep (`sub-a`/`sub-b` at each level), placing one - * secondary entry point at each of the 2^depth leaves. Returns their relative dirs. - */ -function generateDeepEntries(outDir: string, depth: number): string[] { - const total = 2 ** depth; - const width = String(total).length; - const dirs: string[] = []; - let counter = 0; - - function build(currentRelDir: string, level: number): void { - if (level === depth) { - counter++; - const id = String(counter).padStart(width, '0'); - writeEntryPointSource(path.join(outDir, currentRelDir), id); - dirs.push(currentRelDir); - - return; - } - for (const child of ['sub-a', 'sub-b']) { - build(path.join(currentRelDir, child), level + 1); - } - } - - build('.', 0); - - return dirs.map((d) => path.normalize(d)); -} - -/** "sub-a/sub-b" -> "sub-a-sub-b"; "comp-0001" -> "comp-0001". Used as the exports map subpath key. */ -function entryKey(relDir: string): string { - return relDir.split(path.sep).join('-'); -} - -/** - * Generates a complete `@angular/build:library`-buildable project at `outDir`: a primary entry - * point, N secondary entry points per `options`, and the angular.json/tsconfig/package.json - * scaffolding the builder needs. Returns the number of secondary entry points generated. - */ -export function generateFixture(outDir: string, options: FixtureOptions): number { - fs.rmSync(outDir, { recursive: true, force: true }); - fs.mkdirSync(outDir, { recursive: true }); - - fs.writeFileSync( - path.join(outDir, 'public-api.ts'), - `export const REFERENCE_VERSION = '1.0.0';\n`, - ); - - const secondaryDirs = - options.layout === 'flat' - ? generateFlatEntries(outDir, options.count ?? 10) - : generateDeepEntries(outDir, options.depth ?? 2); - - const exportsMap: Record = { '.': './public-api.ts' }; - for (const relDir of secondaryDirs) { - exportsMap[`./${entryKey(relDir)}`] = `./${relDir.split(path.sep).join('/')}/public-api.ts`; - } - - // Fallback to a hardcoded version if the path lookup fails in the sandbox/environment. - let peerRange = '^22.0.0'; - try { - const angularCoreVersion = JSON.parse( - fs.readFileSync( - new URL('../../../node_modules/@angular/core/package.json', import.meta.url), - 'utf8', - ), - ).version as string; - peerRange = `^${angularCoreVersion.replace(/\.\d+$/, '')}`; - } catch { - // Ignore error, use fallback - } - - fs.writeFileSync( - path.join(outDir, 'package.json'), - JSON.stringify( - { - name: `ref-${options.layout}`, - version: '0.0.1', - peerDependencies: { - '@angular/core': peerRange, - '@angular/common': peerRange, - }, - exports: exportsMap, - }, - null, - 2, - ) + '\n', - ); - - fs.writeFileSync( - path.join(outDir, 'tsconfig.json'), - JSON.stringify( - { - compileOnSave: false, - compilerOptions: { - outDir: './dist/out-tsc', - sourceMap: true, - declaration: false, - moduleResolution: 'bundler', - experimentalDecorators: true, - target: 'es2022', - module: 'es2022', - useDefineForClassFields: false, - lib: ['es2022', 'dom'], - }, - angularCompilerOptions: { - enableIvy: true, - disableTypeScriptVersionCheck: true, - }, - }, - null, - 2, - ) + '\n', - ); - - fs.writeFileSync( - path.join(outDir, 'tsconfig.lib.json'), - JSON.stringify( - { - extends: './tsconfig.json', - compilerOptions: { - declaration: true, - declarationMap: true, - inlineSources: true, - types: [], - }, - exclude: ['**/*.spec.ts'], - }, - null, - 2, - ) + '\n', - ); - - fs.writeFileSync( - path.join(outDir, 'angular.json'), - JSON.stringify( - { - version: 1, - cli: { cache: { enabled: false } }, - projects: { - lib: { - root: '.', - projectType: 'library', - sourceRoot: '.', - targets: { - build: { - builder: '@angular/build:library', - options: { tsConfig: './tsconfig.lib.json', outputPath: 'dist' }, - }, - }, - }, - }, - }, - null, - 2, - ) + '\n', - ); - - return secondaryDirs.length; -} +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Generates synthetic Angular Package Format (APF) library fixtures for benchmarking +// `@angular/build:library`, and converts them into the builder's project shape (a flat +// `exports` map in one root package.json, rather than the per-entry-point `ng-package.json` +// convention used by third-party tools such as ng-packagr). + +import fs from 'node:fs'; +import path from 'node:path'; + +export type Layout = 'flat' | 'deep'; + +export interface FixtureOptions { + layout: Layout; + + /** Number of secondary entry points for `layout: 'flat'`. */ + count?: number; + + /** Binary-tree depth for `layout: 'deep'` (2^depth leaf entry points). */ + depth?: number; +} + +function buildComponentSource(className: string, selector: string, id: string): string { + return `import { Component } from '@angular/core'; + +@Component({ + selector: '${selector}', + template: '

Component no. ${id}

', +}) +export class ${className} {} +`; +} + +/** Writes one secondary entry point's source files (component + public-api.ts) into `entryDir`. */ +function writeEntryPointSource(entryDir: string, id: string): void { + fs.mkdirSync(entryDir, { recursive: true }); + + const className = `RefComponent${id}`; + const selector = `ref-comp-${id}`; + const compBase = `comp-${id}`; + + fs.writeFileSync( + path.join(entryDir, 'public-api.ts'), + `export * from './${compBase}.component';\n`, + ); + + fs.writeFileSync( + path.join(entryDir, `${compBase}.component.ts`), + buildComponentSource(className, selector, id), + ); +} + +/** Generates `count` flat secondary entry points as direct siblings, returning their relative dirs. */ +function generateFlatEntries(outDir: string, count: number): string[] { + const width = String(count).length; + const dirs: string[] = []; + for (let i = 1; i <= count; i++) { + const id = String(i).padStart(width, '0'); + const relDir = `comp-${id}`; + writeEntryPointSource(path.join(outDir, relDir), id); + dirs.push(relDir); + } + + return dirs; +} + +/** + * Generates a binary tree `depth` levels deep (`sub-a`/`sub-b` at each level), placing one + * secondary entry point at each of the 2^depth leaves. Returns their relative dirs. + */ +function generateDeepEntries(outDir: string, depth: number): string[] { + const total = 2 ** depth; + const width = String(total).length; + const dirs: string[] = []; + let counter = 0; + + function build(currentRelDir: string, level: number): void { + if (level === depth) { + counter++; + const id = String(counter).padStart(width, '0'); + writeEntryPointSource(path.join(outDir, currentRelDir), id); + dirs.push(currentRelDir); + + return; + } + for (const child of ['sub-a', 'sub-b']) { + build(path.join(currentRelDir, child), level + 1); + } + } + + build('.', 0); + + return dirs.map((d) => path.normalize(d)); +} + +/** "sub-a/sub-b" -> "sub-a-sub-b"; "comp-0001" -> "comp-0001". Used as the exports map subpath key. */ +function entryKey(relDir: string): string { + return relDir.split(path.sep).join('-'); +} + +/** + * Generates a complete `@angular/build:library`-buildable project at `outDir`: a primary entry + * point, N secondary entry points per `options`, and the angular.json/tsconfig/package.json + * scaffolding the builder needs. Returns the number of secondary entry points generated. + */ +export function generateFixture(outDir: string, options: FixtureOptions): number { + fs.rmSync(outDir, { recursive: true, force: true }); + fs.mkdirSync(outDir, { recursive: true }); + + fs.writeFileSync( + path.join(outDir, 'public-api.ts'), + `export const REFERENCE_VERSION = '1.0.0';\n`, + ); + + const secondaryDirs = + options.layout === 'flat' + ? generateFlatEntries(outDir, options.count ?? 10) + : generateDeepEntries(outDir, options.depth ?? 2); + + const exportsMap: Record = { '.': './public-api.ts' }; + for (const relDir of secondaryDirs) { + exportsMap[`./${entryKey(relDir)}`] = `./${relDir.split(path.sep).join('/')}/public-api.ts`; + } + + // Fallback to a hardcoded version if the path lookup fails in the sandbox/environment. + let peerRange = '^22.0.0'; + try { + const angularCoreVersion = JSON.parse( + fs.readFileSync( + new URL('../../../node_modules/@angular/core/package.json', import.meta.url), + 'utf8', + ), + ).version as string; + peerRange = `^${angularCoreVersion.replace(/\.\d+$/, '')}`; + } catch { + // Ignore error, use fallback + } + + fs.writeFileSync( + path.join(outDir, 'package.json'), + JSON.stringify( + { + name: `ref-${options.layout}`, + version: '0.0.1', + peerDependencies: { + '@angular/core': peerRange, + '@angular/common': peerRange, + }, + exports: exportsMap, + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'tsconfig.json'), + JSON.stringify( + { + compileOnSave: false, + compilerOptions: { + outDir: './dist/out-tsc', + sourceMap: true, + declaration: false, + moduleResolution: 'bundler', + experimentalDecorators: true, + target: 'es2022', + module: 'es2022', + useDefineForClassFields: false, + lib: ['es2022', 'dom'], + }, + angularCompilerOptions: { + enableIvy: true, + disableTypeScriptVersionCheck: true, + }, + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'tsconfig.lib.json'), + JSON.stringify( + { + extends: './tsconfig.json', + compilerOptions: { + declaration: true, + declarationMap: true, + inlineSources: true, + types: [], + }, + exclude: ['**/*.spec.ts'], + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'angular.json'), + JSON.stringify( + { + version: 1, + cli: { cache: { enabled: false } }, + projects: { + lib: { + root: '.', + projectType: 'library', + sourceRoot: '.', + targets: { + build: { + builder: '@angular/build:library', + options: { tsConfig: './tsconfig.lib.json', outputPath: 'dist' }, + }, + }, + }, + }, + }, + null, + 2, + ) + '\n', + ); + + return secondaryDirs.length; +} diff --git a/scripts/benchmarks/library-builder/harness.mts b/scripts/benchmarks/library-builder/harness.mts index 742d25e239b4..5220b95965e9 100644 --- a/scripts/benchmarks/library-builder/harness.mts +++ b/scripts/benchmarks/library-builder/harness.mts @@ -1,82 +1,82 @@ -/** - * @license - * Copyright Google LLC All Rights Reserved. - * - * Use of this source code is governed by an MIT-style license that can be - * found in the LICENSE file at https://angular.dev/license - */ - -// Runs and measures a single `@angular/build:library` cold build via Architect's standalone CLI. -// `fixtures.mts` owns generating the on-disk project; this module owns spawning the builder -// against it and reporting what happened. Kept separate so each file has one job. - -import { spawnSync } from 'node:child_process'; -import fs from 'node:fs'; -import path from 'node:path'; -import type { Layout } from './fixtures.mts'; - -export interface BuildMeasurement { - layout: Layout; - entryPoints: number; - iteration: number; - durationMs: number; - status: 'ok' | 'fail'; -} - -export interface BuilderPaths { - buildPkg: string; - architectPkg: string; - architectCli: string; -} - -/** Verifies `@angular/build` and `@angular-devkit/architect` are built before attempting a build. */ -export function checkBuilderIsBuilt(repoRoot: string, logger: Console): BuilderPaths | undefined { - const buildPkg = path.join(repoRoot, 'dist', '@angular', 'build'); - const architectPkg = path.join(repoRoot, 'dist', '@angular-devkit', 'architect'); - const architectCli = path.join(architectPkg, 'bin', 'cli.js'); - - if (!fs.existsSync(path.join(buildPkg, 'package.json')) || !fs.existsSync(architectCli)) { - logger.error( - 'Error: @angular/build and/or @angular-devkit/architect have not been built yet.\n' + - 'Run "pnpm build" (or "pnpm admin build --local" for a faster local build) first.', - ); - - return undefined; - } - - return { buildPkg, architectPkg, architectCli }; -} - -/** Spawns one cold `lib:build` run against `projectDir` and times it. */ -export function buildOnce( - projectDir: string, - architectCli: string, -): { durationMs: number; status: 'ok' | 'fail' } { - const start = performance.now(); - const result = spawnSync(process.execPath, [architectCli, 'lib:build'], { - cwd: projectDir, - stdio: 'pipe', - maxBuffer: Infinity, - }); - const durationMs = performance.now() - start; - const status = result.status === 0 ? 'ok' : 'fail'; - if (status === 'fail') { - if (result.error) { - // eslint-disable-next-line no-console - console.error(result.error); - } - // eslint-disable-next-line no-console - console.error(result.stdout?.toString()); - // eslint-disable-next-line no-console - console.error(result.stderr?.toString()); - } - - return { durationMs, status }; -} - -export function median(values: number[]): number { - const sorted = [...values].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - - return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; -} +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Runs and measures a single `@angular/build:library` cold build via Architect's standalone CLI. +// `fixtures.mts` owns generating the on-disk project; this module owns spawning the builder +// against it and reporting what happened. Kept separate so each file has one job. + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import type { Layout } from './fixtures.mts'; + +export interface BuildMeasurement { + layout: Layout; + entryPoints: number; + iteration: number; + durationMs: number; + status: 'ok' | 'fail'; +} + +export interface BuilderPaths { + buildPkg: string; + architectPkg: string; + architectCli: string; +} + +/** Verifies `@angular/build` and `@angular-devkit/architect` are built before attempting a build. */ +export function checkBuilderIsBuilt(repoRoot: string, logger: Console): BuilderPaths | undefined { + const buildPkg = path.join(repoRoot, 'dist', '@angular', 'build'); + const architectPkg = path.join(repoRoot, 'dist', '@angular-devkit', 'architect'); + const architectCli = path.join(architectPkg, 'bin', 'cli.js'); + + if (!fs.existsSync(path.join(buildPkg, 'package.json')) || !fs.existsSync(architectCli)) { + logger.error( + 'Error: @angular/build and/or @angular-devkit/architect have not been built yet.\n' + + 'Run "pnpm build" (or "pnpm admin build --local" for a faster local build) first.', + ); + + return undefined; + } + + return { buildPkg, architectPkg, architectCli }; +} + +/** Spawns one cold `lib:build` run against `projectDir` and times it. */ +export function buildOnce( + projectDir: string, + architectCli: string, +): { durationMs: number; status: 'ok' | 'fail' } { + const start = performance.now(); + const result = spawnSync(process.execPath, [architectCli, 'lib:build'], { + cwd: projectDir, + stdio: 'pipe', + maxBuffer: Infinity, + }); + const durationMs = performance.now() - start; + const status = result.status === 0 ? 'ok' : 'fail'; + if (status === 'fail') { + if (result.error) { + // eslint-disable-next-line no-console + console.error(result.error); + } + // eslint-disable-next-line no-console + console.error(result.stdout?.toString()); + // eslint-disable-next-line no-console + console.error(result.stderr?.toString()); + } + + return { durationMs, status }; +} + +export function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} diff --git a/scripts/benchmarks/library-builder/index.mts b/scripts/benchmarks/library-builder/index.mts index 79358e8d1192..348348f51df4 100644 --- a/scripts/benchmarks/library-builder/index.mts +++ b/scripts/benchmarks/library-builder/index.mts @@ -1,132 +1,132 @@ -/** - * @license - * Copyright Google LLC All Rights Reserved. - * - * Use of this source code is governed by an MIT-style license that can be - * found in the LICENSE file at https://angular.dev/license - */ - -// Benchmarks `@angular/build:library`'s cold build time across a sweep of library sizes. -// This module only orchestrates the sweep: fixture generation lives in fixtures.mts, build -// execution and measurement live in harness.mts. - -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { type FixtureOptions, type Layout, generateFixture } from './fixtures.mts'; -import { type BuildMeasurement, buildOnce, checkBuilderIsBuilt, median } from './harness.mts'; - -export interface LibraryBuilderBenchmarkOptions { - layout?: Layout; - count?: number[]; - depth?: number[]; - iterations?: number; - json?: boolean; -} - -const repoRoot = fileURLToPath(new URL('../../../', import.meta.url)); -const scratchRoot = path.join(repoRoot, 'dist', '.library-benchmark-fixtures'); - -/** - * Default entry-point counts for the `flat` layout, chosen to resolve a U-shaped scaling curve. - * Also drives the `deep` layout's default depths (see `depthsForEntryPointTargets`) -- `--count` - * is the one sweep parameter users pick; `--depth` is only ever an explicit override. - */ -const DEFAULT_FLAT_COUNTS = [10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000]; - -/** - * Derives default `deep` layout tree depths from a set of target entry-point counts, picking for - * each target `n` the smallest `depth` with `2^depth >= n`, de-duplicated in ascending order. - * This keeps both layouts comparable at roughly the same entry-point scale instead of an - * arbitrary, independently-chosen depth list. - */ -function depthsForEntryPointTargets(targets: number[]): number[] { - const depths = new Set(); - for (const target of targets) { - depths.add(Math.max(1, Math.ceil(Math.log2(target)))); - } - - return [...depths].sort((a, b) => a - b); -} - -export async function runLibraryBuilderBenchmarks( - options: LibraryBuilderBenchmarkOptions = {}, -): Promise<{ measurements: BuildMeasurement[]; exitCode: number }> { - const builder = checkBuilderIsBuilt(repoRoot, console); - if (!builder) { - return { measurements: [], exitCode: 1 }; - } - - const layout = options.layout ?? 'flat'; - const iterations = options.iterations ?? 3; - const sizes = - layout === 'flat' - ? (options.count ?? DEFAULT_FLAT_COUNTS) - : (options.depth ?? depthsForEntryPointTargets(options.count ?? DEFAULT_FLAT_COUNTS)); - - fs.mkdirSync(scratchRoot, { recursive: true }); - const measurements: BuildMeasurement[] = []; - - try { - for (const sizeOrDepth of sizes) { - const label = layout + '-' + (layout === 'deep' ? 'depth' + sizeOrDepth : sizeOrDepth); - const projectDir = path.join(scratchRoot, label); - - const fixtureOptions: FixtureOptions = - layout === 'flat' ? { layout, count: sizeOrDepth } : { layout, depth: sizeOrDepth }; - const entryPoints = generateFixture(projectDir, fixtureOptions); - - if (!options.json) { - // eslint-disable-next-line no-console - console.log('\n[library-builder] ' + label + ' (' + entryPoints + ' entry points)'); - } - - const durations: number[] = []; - for (let i = 1; i <= iterations; i++) { - const { durationMs, status } = buildOnce(projectDir, builder.architectCli); - if (!options.json) { - // eslint-disable-next-line no-console - console.log( - ' run ' + - i + - '/' + - iterations + - ': ' + - (status === 'ok' ? 'done' : 'FAILED') + - ' in ' + - (durationMs / 1000).toFixed(2) + - 's', - ); - } - if (status === 'ok') { - durations.push(durationMs); - } - measurements.push({ layout, entryPoints, iteration: i, durationMs, status }); - } - - if (!options.json && durations.length > 0) { - const medianMs = median(durations); - // eslint-disable-next-line no-console - console.log( - ' median: ' + - (medianMs / 1000).toFixed(2) + - 's (' + - (medianMs / entryPoints).toFixed(2) + - 'ms/entry)', - ); - } - - fs.rmSync(projectDir, { recursive: true, force: true }); - } - } finally { - fs.rmSync(scratchRoot, { recursive: true, force: true }); - } - - const anyFailed = measurements.some((m) => m.status === 'fail'); - if (options.json) { - // eslint-disable-next-line no-console - console.log(JSON.stringify(measurements, null, 2)); - } - - return { measurements, exitCode: anyFailed ? 1 : 0 }; -} +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Benchmarks `@angular/build:library`'s cold build time across a sweep of library sizes. +// This module only orchestrates the sweep: fixture generation lives in fixtures.mts, build +// execution and measurement live in harness.mts. + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { type FixtureOptions, type Layout, generateFixture } from './fixtures.mts'; +import { type BuildMeasurement, buildOnce, checkBuilderIsBuilt, median } from './harness.mts'; + +export interface LibraryBuilderBenchmarkOptions { + layout?: Layout; + count?: number[]; + depth?: number[]; + iterations?: number; + json?: boolean; +} + +const repoRoot = fileURLToPath(new URL('../../../', import.meta.url)); +const scratchRoot = path.join(repoRoot, 'dist', '.library-benchmark-fixtures'); + +/** + * Default entry-point counts for the `flat` layout, chosen to resolve a U-shaped scaling curve. + * Also drives the `deep` layout's default depths (see `depthsForEntryPointTargets`) -- `--count` + * is the one sweep parameter users pick; `--depth` is only ever an explicit override. + */ +const DEFAULT_FLAT_COUNTS = [10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000]; + +/** + * Derives default `deep` layout tree depths from a set of target entry-point counts, picking for + * each target `n` the smallest `depth` with `2^depth >= n`, de-duplicated in ascending order. + * This keeps both layouts comparable at roughly the same entry-point scale instead of an + * arbitrary, independently-chosen depth list. + */ +function depthsForEntryPointTargets(targets: number[]): number[] { + const depths = new Set(); + for (const target of targets) { + depths.add(Math.max(1, Math.ceil(Math.log2(target)))); + } + + return [...depths].sort((a, b) => a - b); +} + +export async function runLibraryBuilderBenchmarks( + options: LibraryBuilderBenchmarkOptions = {}, +): Promise<{ measurements: BuildMeasurement[]; exitCode: number }> { + const builder = checkBuilderIsBuilt(repoRoot, console); + if (!builder) { + return { measurements: [], exitCode: 1 }; + } + + const layout = options.layout ?? 'flat'; + const iterations = options.iterations ?? 3; + const sizes = + layout === 'flat' + ? (options.count ?? DEFAULT_FLAT_COUNTS) + : (options.depth ?? depthsForEntryPointTargets(options.count ?? DEFAULT_FLAT_COUNTS)); + + fs.mkdirSync(scratchRoot, { recursive: true }); + const measurements: BuildMeasurement[] = []; + + try { + for (const sizeOrDepth of sizes) { + const label = layout + '-' + (layout === 'deep' ? 'depth' + sizeOrDepth : sizeOrDepth); + const projectDir = path.join(scratchRoot, label); + + const fixtureOptions: FixtureOptions = + layout === 'flat' ? { layout, count: sizeOrDepth } : { layout, depth: sizeOrDepth }; + const entryPoints = generateFixture(projectDir, fixtureOptions); + + if (!options.json) { + // eslint-disable-next-line no-console + console.log('\n[library-builder] ' + label + ' (' + entryPoints + ' entry points)'); + } + + const durations: number[] = []; + for (let i = 1; i <= iterations; i++) { + const { durationMs, status } = buildOnce(projectDir, builder.architectCli); + if (!options.json) { + // eslint-disable-next-line no-console + console.log( + ' run ' + + i + + '/' + + iterations + + ': ' + + (status === 'ok' ? 'done' : 'FAILED') + + ' in ' + + (durationMs / 1000).toFixed(2) + + 's', + ); + } + if (status === 'ok') { + durations.push(durationMs); + } + measurements.push({ layout, entryPoints, iteration: i, durationMs, status }); + } + + if (!options.json && durations.length > 0) { + const medianMs = median(durations); + // eslint-disable-next-line no-console + console.log( + ' median: ' + + (medianMs / 1000).toFixed(2) + + 's (' + + (medianMs / entryPoints).toFixed(2) + + 'ms/entry)', + ); + } + + fs.rmSync(projectDir, { recursive: true, force: true }); + } + } finally { + fs.rmSync(scratchRoot, { recursive: true, force: true }); + } + + const anyFailed = measurements.some((m) => m.status === 'fail'); + if (options.json) { + // eslint-disable-next-line no-console + console.log(JSON.stringify(measurements, null, 2)); + } + + return { measurements, exitCode: anyFailed ? 1 : 0 }; +} From c822f4f4f25a0df5a434871593b8dd14534be4f9 Mon Sep 17 00:00:00 2001 From: David Herges Date: Mon, 12 Oct 2026 10:44:50 +0200 Subject: [PATCH 7/9] fixup! test(@angular/build): add library builder scaling benchmark suite --- scripts/benchmarks/library-builder/README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/benchmarks/library-builder/README.md b/scripts/benchmarks/library-builder/README.md index 73c56fe634ea..05b030974a56 100644 --- a/scripts/benchmarks/library-builder/README.md +++ b/scripts/benchmarks/library-builder/README.md @@ -6,6 +6,7 @@ This benchmark measures build time metrics for Angular libraries. Angular CLI is in the process of transitioning from `ng-packagr` to Angular CLI's native library builder. Both library build systems are registered Architect builders: + - `@angular/build:ng-packagr`: runs library builds through the legacy `ng-packagr` build pipeline. - `@angular/build:library`: runs library builds through Angular CLI's native library builder. @@ -22,6 +23,7 @@ as the number of entry points scale past a certain N (in the test runs N=200+). [Landing Angular CLI's native `@angular/build:library` builder](https://github.com/angular/angular-cli/pull/34106#issuecomment-5744311681), indicated very favourable performance metrics for the new library builder that can be explained by number of architectural changes in the library build pipeline: + - Shared Stylesheet Bundling - Public API Invalidation via Declaration Hashing - Dedicated Worker Pool for Production Builds @@ -83,13 +85,13 @@ pnpm admin benchmark library-builder [options] ### Options -| Option | Description | Default | -| :--------------------- | :----------------------------------------------- | :---------------------------------------- | -| `--layout=` | Fixture layout | `flat` | -| `--count=` | Entry-point count targets, drives both layouts | `10,20,50,100,200,300,400,500,1000,2000` | -| `--depth=` | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count` | derived from `--count` | -| `--iterations=` | Measured iterations per size | `3` | -| `--json` | Output raw measurements as JSON | N/A | +| Option | Description | Default | +| :---------------------- | :--------------------------------------------------------------------------------------------------- | :--------------------------------------- | +| `--layout=` | Fixture layout | `flat` | +| `--count=` | Entry-point count targets, drives both layouts | `10,20,50,100,200,300,400,500,1000,2000` | +| `--depth=` | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count` | derived from `--count` | +| `--iterations=` | Measured iterations per size | `3` | +| `--json` | Output raw measurements as JSON | N/A | > `--count` and `--depth` are not independent sweep axes for `layout=deep`: `--count` is translated > into depths (smallest `depth` with `2^depth >= n` per target, de-duplicated); `--depth` only From a39f001cea408a06292c970dfd2955f8ba8c7d97 Mon Sep 17 00:00:00 2001 From: David Herges Date: Mon, 12 Oct 2026 11:46:31 +0200 Subject: [PATCH 8/9] fixup! test(@angular/build): add library builder scaling benchmark suite Recorded reuslts for the Iteration 1 baseline sweep against the native libray builder, indicating that the new native library builder scales gracefully - far far better than ng-packagr. No U-shape curve detected. --- scripts/benchmarks/library-builder/README.md | 60 ++++++++++++++++++-- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/scripts/benchmarks/library-builder/README.md b/scripts/benchmarks/library-builder/README.md index 05b030974a56..d5232fcb2e43 100644 --- a/scripts/benchmarks/library-builder/README.md +++ b/scripts/benchmarks/library-builder/README.md @@ -1,6 +1,6 @@ # Library Builds Scaling Benchmark -This benchmark measures build time metrics for Angular libraries. +This benchmark suites measures build time metrics for Angular libraries. ## Context @@ -14,15 +14,15 @@ This benchmark suite is designed to quantify performance impacts and ensure scal ### Previous Benchmarks -For context, when running a similar benchmark suite directly against `ng-packagr`, it hinted at a "U-curve" in build times, +For context, when running a comparable benchmark suite directly against `ng-packagr`, it hinted at a "U-curve" in build times, where build time per entry point in `ng-packagr` drops until entry point N (with N varying in the range 200..500). This shape indicates an initial overhead as a possible explanation for a small number of entry points. However, we discovered that `ng-packagr` build time per entry point increases as the number of entry points scale past a certain N (in the test runs N=200+). [Landing Angular CLI's native `@angular/build:library` builder](https://github.com/angular/angular-cli/pull/34106#issuecomment-5744311681), -indicated very favourable performance metrics for the new library builder that can be explained by number of -architectural changes in the library build pipeline: +indicated very favourable performance metrics for the new library builder that can be explained by a number of +architectural changes in the design of the library build pipeline: - Shared Stylesheet Bundling - Public API Invalidation via Declaration Hashing @@ -30,6 +30,9 @@ architectural changes in the library build pipeline: - Single Pass Output Emission - Direct Async Pipeline +The benchmark design of this suite is built to not only verify the previous results, +but also to confirm or contradict the architectural choices in the library builder pipeline. + ### Goals This benchmark suite runs explorative research on the performance of library builds in Angular CLI. @@ -108,7 +111,54 @@ pnpm admin benchmark library-builder --layout=flat --count=500 ## Benchmark Results -_(To be filled in once a sweep has been run and reviewed. Out of scope for this iteration.)_ +Each entry below is one benchmark run, with its own summary and conclusions. + +### 2026-10-12: Iteration 1 baseline sweep (native library builder, flat & deep) + +First full run of the default sweep against `@angular/build:library` only — no `ng-packagr` A/B comparison yet (still Future Work). +Fixtures use inline HTML templates only, no stylesheets. +Default `--count` sweep, `--iterations=3`, both layouts. +Environment: 4-core / 16 GiB sandbox, NVMe SSD. +Raw log: `tmp/benchmark-run-1791795379.log` (gitignored; not included in this commit). + +#### Flat layout (inline templates only) + +| Entries | Median | ms/entry | +| ------: | -----: | -------: | +| 10 | 3.25s | 324.63 | +| 20 | 3.28s | 164.14 | +| 50 | 3.33s | 66.61 | +| 100 | 3.75s | 37.53 | +| 200 | 4.31s | 21.56 | +| 300 | 4.90s | 16.35 | +| 400 | 5.51s | 13.78 | +| 500 | 6.18s | 12.36 | +| 1000 | 10.29s | 10.29 | +| 2000 | 19.47s | 9.74 | + +#### Deep layout (inline templates only) + +| Entries | Median | ms/entry | +| ------: | -----: | -------: | +| 16 | 3.27s | 204.47 | +| 32 | 3.30s | 103.16 | +| 64 | 3.57s | 55.74 | +| 128 | 4.20s | 32.84 | +| 256 | 4.70s | 18.36 | +| 512 | 6.31s | 12.32 | +| 1024 | 10.25s | 10.01 | +| 2048 | 21.67s | 10.58 | + +**Conclusion:** per-entry build time decreases monotonically across both layouts and flattens out +around 9.6-10.6ms/entry at the largest sizes tested -- no U-shaped climb-back-up at any tested +size (up to 2000/2048 entries), unlike the pattern reported for `ng-packagr` past N≈200-500. Flat +and deep track each other closely at comparable entry-point scale, suggesting the directory +layout itself isn't a meaningful cost driver for the native builder, just raw entry-point count. +`deep`'s depth-11 run (2048 entries) showed slightly higher run-to-run variance (20.78s-22.46s) +than `flat`'s 2000-entry run (19.46s-19.63s) -- possibly directory-tree-walk overhead at depth, +but not a regression, and not yet investigated further. + +All 60 builds (10 sizes x 3 iterations x 2 layouts) completed with `status: "ok"`; zero failures. ## Future Work From fdb8f5a6fe5ddc57a95f45fb25b06e78e5525328 Mon Sep 17 00:00:00 2001 From: David Herges Date: Mon, 12 Oct 2026 19:39:57 +0200 Subject: [PATCH 9/9] fixup! test(@angular/build): add library builder scaling benchmark suite Reported benchmark results in csv file or json. --- scripts/benchmark.mts | 31 +++++++++++++------ scripts/benchmarks/library-builder/README.md | 10 ++++-- scripts/benchmarks/library-builder/index.mts | 12 ++++++- .../benchmarks/library-builder/reporters.mts | 25 +++++++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 scripts/benchmarks/library-builder/reporters.mts diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts index 9f5a55214eee..b51e6443e2c9 100644 --- a/scripts/benchmark.mts +++ b/scripts/benchmark.mts @@ -49,13 +49,17 @@ function parseIntList(raw: string | number | undefined): number[] | undefined { .map((s) => Number(s.trim())); } -async function runLibraryBuilderSubsystem(options: { - layout?: string; - count?: string; - depth?: string; - iterations?: string | number; - json?: boolean; -}): Promise { +async function runLibraryBuilderSubsystem( + options: { + layout?: string; + count?: string; + depth?: string; + iterations?: string | number; + json?: boolean; + csv?: string | boolean; + }, + cwd: string, +): Promise { if (options.layout !== undefined && options.layout !== 'flat' && options.layout !== 'deep') { // eslint-disable-next-line no-console console.error('Error: --layout must be "flat" or "deep", got "' + options.layout + '".'); @@ -97,12 +101,20 @@ async function runLibraryBuilderSubsystem(options: { return 1; } + if (options.csv !== undefined && (typeof options.csv !== 'string' || options.csv.length === 0)) { + // eslint-disable-next-line no-console + console.error('Error: --csv requires a file path, got "' + options.csv + '".'); + + return 1; + } + const cliOptions: LibraryBuilderBenchmarkOptions = { layout: options.layout as LibraryBuilderBenchmarkOptions['layout'], count, depth, iterations, json: Boolean(options.json), + csv: options.csv !== undefined ? path.resolve(cwd, options.csv as string) : undefined, }; const { exitCode } = await runLibraryBuilderBenchmarks(cliOptions); @@ -167,7 +179,8 @@ Options (library-builder): that would otherwise be derived from --count (the smallest depth with 2^depth >= each target count). --iterations= Measured iterations (default: 3) - --json Output raw measurements as JSON + --json Output raw measurements as JSON to stdout + --csv= Write raw per-iteration measurements as CSV to --help Show this help message `); @@ -176,7 +189,7 @@ Options (library-builder): } if (targetSubsystem === 'library-builder') { - return runLibraryBuilderSubsystem(options); + return runLibraryBuilderSubsystem(options, _cwd); } if (targetSubsystem !== 'i18n') { diff --git a/scripts/benchmarks/library-builder/README.md b/scripts/benchmarks/library-builder/README.md index d5232fcb2e43..b0bbc8cd87ea 100644 --- a/scripts/benchmarks/library-builder/README.md +++ b/scripts/benchmarks/library-builder/README.md @@ -94,7 +94,8 @@ pnpm admin benchmark library-builder [options] | `--count=` | Entry-point count targets, drives both layouts | `10,20,50,100,200,300,400,500,1000,2000` | | `--depth=` | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count` | derived from `--count` | | `--iterations=` | Measured iterations per size | `3` | -| `--json` | Output raw measurements as JSON | N/A | +| `--json` | Output raw per-iteration measurements as JSON to stdout | N/A | +| `--csv=` | Write raw per-iteration measurements as CSV to `` | N/A (opt-in only) | > `--count` and `--depth` are not independent sweep axes for `layout=deep`: `--count` is translated > into depths (smallest `depth` with `2^depth >= n` per target, de-duplicated); `--depth` only @@ -167,5 +168,10 @@ All 60 builds (10 sizes x 3 iterations x 2 layouts) completed with `status: "ok" - Dependencies between entry points. - Shared stylesheet tokens imported by multiple components. - Watch mode. -- Peak memory (`ru_maxrss`) and CPU time (`ru_utime` + `ru_stime`) metrics per build, in addition to wall-clock build time. +- Peak memory and CPU time metrics per build, in addition to wall-clock build time. Note: + `child.resourceUsage()` does **not** exist on Node's `ChildProcess` (only `process.resourceUsage()` + exists, and it reports the calling process's own usage, not a child's) -- the real path is + polling `/proc//status` (`VmHWM`) and `/proc//stat` (`utime`+`stime`) on a timer while + the build subprocess is alive, which is Linux-only and requires moving `buildOnce` off + `spawnSync` onto `spawn()` so a polling timer can interleave with the child's lifetime. - Library "taste" variants beyond plain flat/deep, e.g. flat/deep with inter-entry-point dependencies or shared SCSS tokens (currently out of scope, listed above). diff --git a/scripts/benchmarks/library-builder/index.mts b/scripts/benchmarks/library-builder/index.mts index 348348f51df4..7615f2b3598c 100644 --- a/scripts/benchmarks/library-builder/index.mts +++ b/scripts/benchmarks/library-builder/index.mts @@ -15,6 +15,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { type FixtureOptions, type Layout, generateFixture } from './fixtures.mts'; import { type BuildMeasurement, buildOnce, checkBuilderIsBuilt, median } from './harness.mts'; +import { formatCsvReport, formatJsonReport } from './reporters.mts'; export interface LibraryBuilderBenchmarkOptions { layout?: Layout; @@ -22,6 +23,7 @@ export interface LibraryBuilderBenchmarkOptions { depth?: number[]; iterations?: number; json?: boolean; + csv?: string; } const repoRoot = fileURLToPath(new URL('../../../', import.meta.url)); @@ -125,7 +127,15 @@ export async function runLibraryBuilderBenchmarks( const anyFailed = measurements.some((m) => m.status === 'fail'); if (options.json) { // eslint-disable-next-line no-console - console.log(JSON.stringify(measurements, null, 2)); + console.log(formatJsonReport(measurements)); + } + + if (options.csv) { + fs.writeFileSync(options.csv, formatCsvReport(measurements)); + if (!options.json) { + // eslint-disable-next-line no-console + console.log('CSV written to: ' + options.csv); + } } return { measurements, exitCode: anyFailed ? 1 : 0 }; diff --git a/scripts/benchmarks/library-builder/reporters.mts b/scripts/benchmarks/library-builder/reporters.mts new file mode 100644 index 000000000000..d175df7332bf --- /dev/null +++ b/scripts/benchmarks/library-builder/reporters.mts @@ -0,0 +1,25 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Pure output formatting for the library-builder benchmark's measurements. No console/file I/O +// here -- `index.mts` owns deciding when and where to print or write what this module formats. + +import type { BuildMeasurement } from './harness.mts'; + +export function formatJsonReport(measurements: BuildMeasurement[]): string { + return JSON.stringify(measurements, null, 2); +} + +export function formatCsvReport(measurements: BuildMeasurement[]): string { + const header = 'layout,entryPoints,iteration,durationMs,status'; + const rows = measurements.map((m) => + [m.layout, m.entryPoints, m.iteration, m.durationMs.toFixed(2), m.status].join(','), + ); + + return [header, ...rows].join('\n') + '\n'; +}