Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions packages/benchmark/__tests__/runner-edge-cases.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import path from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, test, vi } from "vitest";
import { benchmark } from "../src/index.ts";
import { calculateCompressorMetrics, runIterations } from "../src/runner.ts";
import type { BenchmarkOptions } from "../src/types.ts";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
Expand Down Expand Up @@ -276,4 +277,30 @@ describe("Runner Edge Cases", () => {
const result = results.files[0]?.results[0];
expect(result?.success).toBe(true);
});

describe("runIterations validation", () => {
const mockCompressor = vi.fn();

test("should throw when iterationCount is 0", async () => {
await expect(
runIterations(fixtureJS, mockCompressor, "/tmp/out.js", 0, {})
).rejects.toThrow("iterationCount must be at least 1, got 0");
});

test("should throw when iterationCount is negative", async () => {
await expect(
runIterations(fixtureJS, mockCompressor, "/tmp/out.js", -1, {})
).rejects.toThrow("iterationCount must be at least 1, got -1");
});
});

describe("calculateCompressorMetrics validation", () => {
test("should throw when times array is empty", async () => {
await expect(
calculateCompressorMetrics("test", [], fixtureJS, 1000, {})
).rejects.toThrow(
"Cannot calculate metrics for 'test': no timing data provided"
);
});
});
});
294 changes: 215 additions & 79 deletions packages/benchmark/src/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@

import { statSync, unlinkSync } from "node:fs";
import { minify } from "@node-minify/core";
import type { Compressor } from "@node-minify/types";
import {
getFilesizeBrotliInBytes,
getFilesizeBrotliRaw,
getFilesizeGzippedInBytes,
getFilesizeGzippedRaw,
prettyBytes,
Expand All @@ -22,6 +24,176 @@ import type {
FileResult,
} from "./types.ts";

/**
* Run warmup iterations for a compressor to stabilize JIT and caches.
*
* @param file - Input file path
* @param compressor - The compressor function
* @param warmupFile - Path for warmup output file
* @param warmupCount - Number of warmup iterations
* @param options - Benchmark options containing type and compressor options
*/
export async function runWarmup(
file: string,
compressor: Compressor,
warmupFile: string,
warmupCount: number,
options: Pick<BenchmarkOptions, "type" | "compressorOptions">
): Promise<void> {
for (let i = 0; i < warmupCount; i++) {
await minify({
compressor,
input: file,
output: warmupFile,
...(options.type && { type: options.type as "js" | "css" }),
options: options.compressorOptions,
});
}
}

/**
* Result of running timed iterations.
*/
export type IterationsResult = {
times: number[];
outputFile: string;
};

/**
* Run timed iterations of a compressor and return timing data.
*
* @param file - Input file path
* @param compressor - The compressor function
* @param outputFileBase - Base path for output files (will be used as final output)
* @param iterationCount - Number of iterations to run
* @param options - Benchmark options containing type and compressor options
* @returns The array of iteration times and the final output file path
*/
export async function runIterations(
file: string,
compressor: Compressor,
outputFileBase: string,
iterationCount: number,
options: Pick<BenchmarkOptions, "type" | "compressorOptions">
): Promise<IterationsResult> {
if (iterationCount < 1) {
throw new Error(
`iterationCount must be at least 1, got ${iterationCount}`
);
}

const times: number[] = [];

for (let i = 0; i < iterationCount; i++) {
const start = performance.now();
await minify({
compressor,
input: file,
output: outputFileBase,
...(options.type && { type: options.type as "js" | "css" }),
options: options.compressorOptions,
});
times.push(performance.now() - start);
}

return { times, outputFile: outputFileBase };
}

/**
* Calculate final metrics from iteration results.
*
* @param name - Compressor name
* @param times - Array of iteration times in ms
* @param outputFile - Path to the output file (for size measurement)
* @param originalSizeBytes - Original file size in bytes
* @param options - Benchmark options for optional gzip/brotli/verbose
* @returns Populated CompressorMetrics object
*/
export async function calculateCompressorMetrics(
name: string,
times: number[],
outputFile: string,
originalSizeBytes: number,
options: Pick<BenchmarkOptions, "includeGzip" | "includeBrotli" | "verbose">
): Promise<CompressorMetrics> {
if (times.length === 0) {
throw new Error(
`Cannot calculate metrics for '${name}': no timing data provided`
);
}

const avgTime = times.reduce((a, b) => a + b, 0) / times.length;
const outStats = statSync(outputFile);
const sizeBytes = outStats.size;

const metrics: CompressorMetrics = {
compressor: name,
sizeBytes,
size: prettyBytes(sizeBytes),
timeMs: avgTime,
timeMinMs: Math.min(...times),
timeMaxMs: Math.max(...times),
iterationTimes: options.verbose ? times : undefined,
reductionPercent: calculateReduction(originalSizeBytes, sizeBytes),
success: true,
};

if (options.includeGzip) {
metrics.gzipSize = await getFilesizeGzippedInBytes(outputFile);
metrics.gzipBytes = await getFilesizeGzippedRaw(outputFile);
}

if (options.includeBrotli) {
metrics.brotliSize = await getFilesizeBrotliInBytes(outputFile);
metrics.brotliBytes = await getFilesizeBrotliRaw(outputFile);
}

return metrics;
}

/**
* Create an error metrics object for failed compressor runs.
*
* @param name - Compressor name
* @param error - The error message or Error object
* @returns CompressorMetrics indicating failure
*/
export function createErrorMetrics(
name: string,
error: string | Error
): CompressorMetrics {
return {
compressor: name,
sizeBytes: 0,
size: "0 B",
timeMs: 0,
reductionPercent: 0,
success: false,
error: error instanceof Error ? error.message : error,
};
}

/**
* Clean up temporary files created during benchmarking.
*
* @param files - Array of file paths to delete
*/
export function cleanupTempFiles(files: string[]): void {
for (const file of files) {
try {
unlinkSync(file);
} catch {
// Ignore cleanup errors
}
}
}

/**
* Run a complete benchmark across all input files and compressors.
*
* @param options - Benchmark configuration
* @returns Complete benchmark results with file results and summary
*/
export async function runBenchmark(
options: BenchmarkOptions
): Promise<BenchmarkResult> {
Expand Down Expand Up @@ -55,11 +227,11 @@ export async function runBenchmark(
}

/**
* Benchmarks a single input file using the configured compressors and returns per-compressor metrics.
* Benchmarks a single input file using the configured compressors.
*
* @param file - Path to the input file to benchmark
* @param options - Benchmark configuration (compressors to run, iterations, callbacks, and metric options)
* @returns A FileResult containing the file path, original size in bytes and human-readable form, and an array of CompressorMetrics for each attempted compressor
* @param options - Benchmark configuration
* @returns FileResult with original size and per-compressor metrics
*/
async function benchmarkFile(
file: string,
Expand Down Expand Up @@ -94,6 +266,15 @@ async function benchmarkFile(
};
}

/**
* Benchmark a single compressor against a single file.
*
* @param file - Input file path
* @param name - Compressor name
* @param options - Benchmark options
* @param originalSizeBytes - Original file size for reduction calculation
* @returns CompressorMetrics with timing and size data
*/
async function benchmarkCompressor(
file: string,
name: string,
Expand All @@ -103,96 +284,59 @@ async function benchmarkCompressor(
const compressor = await loadCompressor(name);

if (!compressor) {
return {
compressor: name,
sizeBytes: 0,
size: "0 B",
timeMs: 0,
reductionPercent: 0,
success: false,
error: "Compressor not found or not installed",
};
return createErrorMetrics(
name,
"Compressor not found or not installed"
);
}

const iterations = options.iterations || 1;
const warmup = options.warmup ?? (iterations > 1 ? 1 : 0);
const times: number[] = [];
const tempFiles: string[] = [];
const uniqueId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;

try {
const warmupFile = `${file}.warmup.${uniqueId}.tmp`;
for (let i = 0; i < warmup; i++) {
await minify({
compressor,
input: file,
output: warmupFile,
...(options.type && { type: options.type as "js" | "css" }),
options: options.compressorOptions,
});
}
if (warmup > 0) {
await runWarmup(file, compressor, warmupFile, warmup, options);
tempFiles.push(warmupFile);
}

let lastOutputFile = "";
for (let i = 0; i < iterations; i++) {
const start = performance.now();
lastOutputFile = `${file}.${name}.${uniqueId}.tmp`;
await minify({
compressor,
input: file,
output: lastOutputFile,
...(options.type && { type: options.type as "js" | "css" }),
options: options.compressorOptions,
});
times.push(performance.now() - start);
}
tempFiles.push(lastOutputFile);

const avgTime = times.reduce((a, b) => a + b, 0) / times.length;
const outStats = statSync(lastOutputFile);
const sizeBytes = outStats.size;

const metrics: CompressorMetrics = {
compressor: name,
sizeBytes,
size: prettyBytes(sizeBytes),
timeMs: avgTime,
timeMinMs: Math.min(...times),
timeMaxMs: Math.max(...times),
iterationTimes: options.verbose ? times : undefined,
reductionPercent: calculateReduction(originalSizeBytes, sizeBytes),
success: true,
};

if (options.includeGzip) {
metrics.gzipSize = await getFilesizeGzippedInBytes(lastOutputFile);
metrics.gzipBytes = await getFilesizeGzippedRaw(lastOutputFile);
}
const outputFile = `${file}.${name}.${uniqueId}.tmp`;
const { times } = await runIterations(
file,
compressor,
outputFile,
iterations,
options
);
tempFiles.push(outputFile);

if (options.includeBrotli) {
metrics.brotliSize = await getFilesizeBrotliInBytes(lastOutputFile);
}
const metrics = await calculateCompressorMetrics(
name,
times,
outputFile,
originalSizeBytes,
options
);

cleanupTempFiles(tempFiles);

return metrics;
} catch (err) {
cleanupTempFiles(tempFiles);

return {
compressor: name,
sizeBytes: 0,
size: "0 B",
timeMs: 0,
reductionPercent: 0,
success: false,
error: err instanceof Error ? err.message : String(err),
};
return createErrorMetrics(
name,
err instanceof Error ? err.message : String(err)
);
}
}

/**
* Calculate summary statistics from all file results.
*
* @param files - Array of FileResult objects
* @returns Summary with best compression, performance, and recommended compressor
*/
function calculateSummary(files: FileResult[]): BenchmarkResult["summary"] {
const allResults = files.flatMap((f) => f.results).filter((r) => r.success);

Expand Down Expand Up @@ -230,11 +374,3 @@ function calculateSummary(files: FileResult[]): BenchmarkResult["summary"] {
recommended,
};
}

function cleanupTempFiles(files: string[]): void {
for (const file of files) {
try {
unlinkSync(file);
} catch {}
}
}
Loading