feat: add runSync method to Bench to force benchmarks to be synchronous (#210)

Author: rubennorte

src/bench.ts

@@ -20,7 +20,7 @@ import {
 } from './constants'
 import { createBenchEvent } from './event'
 import { Task } from './task'
-import { type JSRuntime, mToNs, now, runtime, runtimeVersion } from './utils'
+import { invariant, type JSRuntime, mToNs, now, runtime, runtimeVersion } from './utils'
 
 /**
  * The Bench class keeps track of the benchmark tasks and controls them.
@@ -207,6 +207,20 @@ export class Bench extends EventTarget {
     return values
   }
 
+  runSync (): Task[] {
+    invariant(this.concurrency === null, 'Cannot use `concurrency` option when using `runSync`')
+    if (this.opts.warmup) {
+      this.warmupTasksSync()
+    }
+    const values: Task[] = []
+    this.dispatchEvent(createBenchEvent('start'))
+    for (const task of this._tasks.values()) {
+      values.push(task.runSync())
+    }
+    this.dispatchEvent(createBenchEvent('complete'))
+    return values
+  }
+
   /**
    * table of the tasks results
    * @param convert - an optional callback to convert the task result to a table record
@@ -258,4 +272,14 @@ export class Bench extends EventTarget {
       }
     }
   }
+
+  /**
+   * warmup the benchmark tasks (sync version)
+   */
+  private warmupTasksSync (): void {
+    this.dispatchEvent(createBenchEvent('warmup'))
+    for (const task of this._tasks.values()) {
+      task.warmupSync()
+    }
+  }
 }

src/task.ts

@@ -12,7 +12,7 @@ import type {
 } from './types'
 
 import { createBenchEvent, createErrorEvent } from './event'
-import { getStatisticsSorted, isFnAsyncResource } from './utils'
+import { getStatisticsSorted, invariant, isFnAsyncResource, isPromiseLike } from './utils'
 
 /**
  * A class that represents each benchmark task in Tinybench. It keeps track of the
@@ -109,66 +109,38 @@ export class Task extends EventTarget {
     )) as { error?: Error; samples?: number[] }
     await this.bench.opts.teardown?.(this, 'run')
 
-    if (latencySamples) {
-      this.runs = latencySamples.length
-      const totalTime = latencySamples.reduce((a, b) => a + b, 0)
+    this.processRunResult({ error, latencySamples })
 
-      // Latency statistics
-      const latencyStatistics = getStatisticsSorted(
-        latencySamples.sort((a, b) => a - b)
-      )
+    return this
+  }
 
-      // Throughput statistics
-      const throughputSamples = latencySamples
-        .map(sample =>
-          sample !== 0 ? 1000 / sample : 1000 / latencyStatistics.mean
-        ) // Use latency average as imputed sample
-        .sort((a, b) => a - b)
-      const throughputStatistics = getStatisticsSorted(throughputSamples)
+  /**
+   * run the current task and write the results in `Task.result` object property
+   * @returns the current task
+   * @internal
+   */
+  runSync (): this {
+    if (this.result?.error) {
+      return this
+    }
 
-      if (this.bench.opts.signal?.aborted) {
-        return this
-      }
+    invariant(this.bench.concurrency === null, 'Cannot use `concurrency` option when using `runSync`')
+    this.dispatchEvent(createBenchEvent('start', this))
 
-      this.mergeTaskResult({
-        critical: latencyStatistics.critical,
-        df: latencyStatistics.df,
-        hz: throughputStatistics.mean,
-        latency: latencyStatistics,
-        max: latencyStatistics.max,
-        mean: latencyStatistics.mean,
-        min: latencyStatistics.min,
-        moe: latencyStatistics.moe,
-        p75: latencyStatistics.p75,
-        p99: latencyStatistics.p99,
-        p995: latencyStatistics.p995,
-        p999: latencyStatistics.p999,
-        period: totalTime / this.runs,
-        rme: latencyStatistics.rme,
-        runtime: this.bench.runtime,
-        runtimeVersion: this.bench.runtimeVersion,
-        samples: latencyStatistics.samples,
-        sd: latencyStatistics.sd,
-        sem: latencyStatistics.sem,
-        throughput: throughputStatistics,
-        totalTime,
-        variance: latencyStatistics.variance,
-      })
-    }
+    const setupResult = this.bench.opts.setup?.(this, 'run')
+    invariant(!isPromiseLike(setupResult), '`setup` function must be sync when using `runSync()`')
 
-    if (error) {
-      this.mergeTaskResult({ error })
-      this.dispatchEvent(createErrorEvent(this, error))
-      this.bench.dispatchEvent(createErrorEvent(this, error))
-      if (this.bench.opts.throws) {
-        throw error
-      }
-    }
+    const { error, samples: latencySamples } = (this.benchmarkSync(
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.bench.opts.time!,
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.bench.opts.iterations!
+    )) as { error?: Error; samples?: number[] }
 
-    this.dispatchEvent(createBenchEvent('cycle', this))
-    this.bench.dispatchEvent(createBenchEvent('cycle', this))
-    // cycle and complete are equal in Task
-    this.dispatchEvent(createBenchEvent('complete', this))
+    const teardownResult = this.bench.opts.teardown?.(this, 'run')
+    invariant(!isPromiseLike(teardownResult), '`teardown` function must be sync when using `runSync()`')
+
+    this.processRunResult({ error, latencySamples })
 
     return this
   }
@@ -191,14 +163,34 @@ export class Task extends EventTarget {
     )) as { error?: Error }
     await this.bench.opts.teardown?.(this, 'warmup')
 
-    if (error) {
-      this.mergeTaskResult({ error })
-      this.dispatchEvent(createErrorEvent(this, error))
-      this.bench.dispatchEvent(createErrorEvent(this, error))
-      if (this.bench.opts.throws) {
-        throw error
-      }
+    this.postWarmup(error)
+  }
+
+  /**
+   * warmup the current task (sync version)
+   * @internal
+   */
+  warmupSync (): void {
+    if (this.result?.error) {
+      return
     }
+
+    this.dispatchEvent(createBenchEvent('warmup', this))
+
+    const setupResult = this.bench.opts.setup?.(this, 'warmup')
+    invariant(!isPromiseLike(setupResult), '`setup` function must be sync when using `runSync()`')
+
+    const { error } = (this.benchmarkSync(
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.bench.opts.warmupTime!,
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.bench.opts.warmupIterations!
+    )) as { error?: Error }
+
+    const teardownResult = this.bench.opts.teardown?.(this, 'warmup')
+    invariant(!isPromiseLike(teardownResult), '`teardown` function must be sync when using `runSync()`')
+
+    this.postWarmup(error)
   }
 
   private async benchmark (
@@ -278,6 +270,69 @@ export class Task extends EventTarget {
     return { samples }
   }
 
+  private benchmarkSync (
+    time: number,
+    iterations: number
+  ): { error?: unknown; samples?: number[] } {
+    if (this.fnOpts.beforeAll != null) {
+      try {
+        const beforeAllResult = this.fnOpts.beforeAll.call(this)
+        invariant(!isPromiseLike(beforeAllResult), '`beforeAll` function must be sync when using `runSync()`')
+      } catch (error) {
+        return { error }
+      }
+    }
+
+    // TODO: factor out
+    let totalTime = 0 // ms
+    const samples: number[] = []
+    const benchmarkTask = () => {
+      if (this.fnOpts.beforeEach != null) {
+        const beforeEachResult = this.fnOpts.beforeEach.call(this)
+        invariant(!isPromiseLike(beforeEachResult), '`beforeEach` function must be sync when using `runSync()`')
+      }
+
+      let taskTime = 0 // ms;
+
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      const taskStart = this.bench.opts.now!()
+      // eslint-disable-next-line no-useless-call
+      const result = this.fn.call(this)
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      taskTime = this.bench.opts.now!() - taskStart
+
+      invariant(!isPromiseLike(result), 'task function must be sync when using `runSync()`')
+
+      samples.push(taskTime)
+      totalTime += taskTime
+
+      if (this.fnOpts.afterEach != null) {
+        const afterEachResult = this.fnOpts.afterEach.call(this)
+        invariant(!isPromiseLike(afterEachResult), '`afterEach` function must be sync when using `runSync()`')
+      }
+    }
+
+    try {
+      while (
+        // eslint-disable-next-line no-unmodified-loop-condition
+        (totalTime < time || samples.length < iterations)) {
+        benchmarkTask()
+      }
+    } catch (error) {
+      return { error }
+    }
+
+    if (this.fnOpts.afterAll != null) {
+      try {
+        const afterAllResult = this.fnOpts.afterAll.call(this)
+        invariant(!isPromiseLike(afterAllResult), '`afterAll` function must be sync when using `runSync()`')
+      } catch (error) {
+        return { error }
+      }
+    }
+    return { samples }
+  }
+
   /**
    * merge into the result object values
    * @param result - the task result object to merge with the current result object values
@@ -288,4 +343,78 @@ export class Task extends EventTarget {
       ...result,
     }) as Readonly<TaskResult>
   }
+
+  private postWarmup (error: Error | undefined): void {
+    if (error) {
+      this.mergeTaskResult({ error })
+      this.dispatchEvent(createErrorEvent(this, error))
+      this.bench.dispatchEvent(createErrorEvent(this, error))
+      if (this.bench.opts.throws) {
+        throw error
+      }
+    }
+  }
+
+  private processRunResult ({ error, latencySamples }: { error?: Error, latencySamples?: number[] }): void {
+    if (latencySamples) {
+      this.runs = latencySamples.length
+      const totalTime = latencySamples.reduce((a, b) => a + b, 0)
+
+      // Latency statistics
+      const latencyStatistics = getStatisticsSorted(
+        latencySamples.sort((a, b) => a - b)
+      )
+
+      // Throughput statistics
+      const throughputSamples = latencySamples
+        .map(sample =>
+          sample !== 0 ? 1000 / sample : 1000 / latencyStatistics.mean
+        ) // Use latency average as imputed sample
+        .sort((a, b) => a - b)
+      const throughputStatistics = getStatisticsSorted(throughputSamples)
+
+      if (this.bench.opts.signal?.aborted) {
+        return
+      }
+
+      this.mergeTaskResult({
+        critical: latencyStatistics.critical,
+        df: latencyStatistics.df,
+        hz: throughputStatistics.mean,
+        latency: latencyStatistics,
+        max: latencyStatistics.max,
+        mean: latencyStatistics.mean,
+        min: latencyStatistics.min,
+        moe: latencyStatistics.moe,
+        p75: latencyStatistics.p75,
+        p99: latencyStatistics.p99,
+        p995: latencyStatistics.p995,
+        p999: latencyStatistics.p999,
+        period: totalTime / this.runs,
+        rme: latencyStatistics.rme,
+        runtime: this.bench.runtime,
+        runtimeVersion: this.bench.runtimeVersion,
+        samples: latencyStatistics.samples,
+        sd: latencyStatistics.sd,
+        sem: latencyStatistics.sem,
+        throughput: throughputStatistics,
+        totalTime,
+        variance: latencyStatistics.variance,
+      })
+    }
+
+    if (error) {
+      this.mergeTaskResult({ error })
+      this.dispatchEvent(createErrorEvent(this, error))
+      this.bench.dispatchEvent(createErrorEvent(this, error))
+      if (this.bench.opts.throws) {
+        throw error
+      }
+    }
+
+    this.dispatchEvent(createBenchEvent('cycle', this))
+    this.bench.dispatchEvent(createBenchEvent('cycle', this))
+    // cycle and complete are equal in Task
+    this.dispatchEvent(createBenchEvent('complete', this))
+  }
 }

src/utils.ts

@@ -158,7 +158,7 @@ export const now = performanceNow
  * @param maybePromiseLike - the value to check
  * @returns true if the value is a promise-like object
  */
-const isPromiseLike = <T>(
+export const isPromiseLike = <T>(
   maybePromiseLike: unknown
 ): maybePromiseLike is PromiseLike<T> =>
     maybePromiseLike !== null &&
@@ -335,3 +335,9 @@ export const getStatisticsSorted = (samples: number[]): Statistics => {
     variance: vr,
   }
 }
+
+export const invariant = (condition: boolean, message: string): void => {
+  if (!condition) {
+    throw new Error(message)
+  }
+}