From e5706e09b33f2bc54323b70e502f675bb8b20c45 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Tue, 16 Sep 2025 18:19:14 -0700 Subject: [PATCH 1/3] Add eventually visible benchmark --- sdk/logs/build.gradle.kts | 27 +++++ .../sdk/logs/booleanstate/BooleanState.java | 13 +++ .../booleanstate/BooleanStateBenchmark.java | 110 ++++++++++++++++++ .../booleanstate/EventualBooleanState.java | 29 +++++ .../booleanstate/ImmediateBooleanState.java | 21 ++++ .../booleanstate/NonVolatileBooleanState.java | 21 ++++ .../VarHandleEventualBooleanState.java | 48 ++++++++ .../VarHandleImmediateBooleanState.java | 41 +++++++ 8 files changed, 310 insertions(+) create mode 100644 sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/BooleanState.java create mode 100644 sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/BooleanStateBenchmark.java create mode 100644 sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/EventualBooleanState.java create mode 100644 sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/ImmediateBooleanState.java create mode 100644 sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/NonVolatileBooleanState.java create mode 100644 sdk/logs/src/jmh/java9/io/opentelemetry/sdk/logs/booleanstate/VarHandleEventualBooleanState.java create mode 100644 sdk/logs/src/jmh/java9/io/opentelemetry/sdk/logs/booleanstate/VarHandleImmediateBooleanState.java diff --git a/sdk/logs/build.gradle.kts b/sdk/logs/build.gradle.kts index b205b03e90e..6f90228ebeb 100644 --- a/sdk/logs/build.gradle.kts +++ b/sdk/logs/build.gradle.kts @@ -9,6 +9,33 @@ plugins { description = "OpenTelemetry Log SDK" otelJava.moduleName.set("io.opentelemetry.sdk.logs") +sourceSets { + create("jmhJava9") { + java { + srcDirs("src/jmh/java9") + } + compileClasspath += sourceSets.jmh.get().compileClasspath + compileClasspath += sourceSets.jmh.get().output + } +} + +tasks { + named("compileJmhJava9Java") { + options.release = 9 + dependsOn("compileJmhJava") + } + + // Configure JMH jar as multi-release jar + named("jmhJar") { + into("META-INF/versions/9") { + from(sourceSets["jmhJava9"].output) + } + manifest.attributes( + "Multi-Release" to "true" + ) + } +} + dependencies { api(project(":api:all")) api(project(":sdk:common")) diff --git a/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/BooleanState.java b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/BooleanState.java new file mode 100644 index 00000000000..1f6ebb218d8 --- /dev/null +++ b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/BooleanState.java @@ -0,0 +1,13 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.logs.booleanstate; + +public interface BooleanState { + + boolean get(); + + void set(boolean state); +} diff --git a/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/BooleanStateBenchmark.java b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/BooleanStateBenchmark.java new file mode 100644 index 00000000000..f2df7c1ae5b --- /dev/null +++ b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/BooleanStateBenchmark.java @@ -0,0 +1,110 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.logs.booleanstate; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 10, time = 1) +@Fork(1) +@State(Scope.Benchmark) +public class BooleanStateBenchmark { + + @Param({ + "NonVolatileBooleanState", + "ImmediateBooleanState", + "EventualBooleanState", + "VarHandleImmediateBooleanState", // available with -PjmhJavaVersion=11 and higher + "VarHandleEventualBooleanState" // available with -PjmhJavaVersion=11 and higher + }) + private String implementation; + + private BooleanState state; + + @Setup(Level.Trial) + public void setup() { + switch (implementation) { + case "NonVolatileBooleanState": + state = new NonVolatileBooleanState(); + break; + case "ImmediateBooleanState": + state = new ImmediateBooleanState(); + break; + case "EventualBooleanState": + state = new EventualBooleanState(); + break; + case "VarHandleImmediateBooleanState": + state = createVarHandleImmediateBooleanState(); + break; + case "VarHandleEventualBooleanState": + state = createVarHandleEventualBooleanState(); + break; + default: + throw new IllegalArgumentException("Unknown implementation: " + implementation); + } + } + + private static BooleanState createVarHandleEventualBooleanState() { + try { + Class clazz = + Class.forName("io.opentelemetry.sdk.logs.booleanstate.VarHandleEventualBooleanState"); + return (BooleanState) clazz.getConstructor().newInstance(); + } catch (Exception e) { + throw new IllegalStateException( + "VarHandleEventualBooleanState not available on this Java version", e); + } + } + + private static BooleanState createVarHandleImmediateBooleanState() { + try { + Class clazz = + Class.forName("io.opentelemetry.sdk.logs.booleanstate.VarHandleImmediateBooleanState"); + return (BooleanState) clazz.getConstructor().newInstance(); + } catch (Exception e) { + throw new IllegalStateException( + "VarHandleImmediateBooleanState not available on this Java version", e); + } + } + + @Benchmark + @Threads(1) + public int read_singleThread() { + int count = 0; + for (int i = 0; i < 100; i++) { + if (state.get()) { + count++; + } + } + return count; + } + + @Benchmark + @Threads(2) // not expecting significant concurrent access + public int read_twoThreads() { + int count = 0; + for (int i = 0; i < 100; i++) { + if (state.get()) { + count++; + } + } + return count; + } +} diff --git a/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/EventualBooleanState.java b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/EventualBooleanState.java new file mode 100644 index 00000000000..522f844846e --- /dev/null +++ b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/EventualBooleanState.java @@ -0,0 +1,29 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.logs.booleanstate; + +public class EventualBooleanState implements BooleanState { + + private volatile boolean state; + private boolean cached; + + private int counter; + + @Override + public boolean get() { + if (counter++ > 1000) { + counter = 0; + cached = state; // Update cached value for visibility in this thread + } + return cached; + } + + @Override + public void set(boolean state) { + this.state = state; + this.cached = state; // Update cached value for immediate visibility in this thread + } +} diff --git a/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/ImmediateBooleanState.java b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/ImmediateBooleanState.java new file mode 100644 index 00000000000..5ea548157a9 --- /dev/null +++ b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/ImmediateBooleanState.java @@ -0,0 +1,21 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.logs.booleanstate; + +public class ImmediateBooleanState implements BooleanState { + + private volatile boolean state; + + @Override + public boolean get() { + return state; + } + + @Override + public void set(boolean state) { + this.state = state; + } +} diff --git a/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/NonVolatileBooleanState.java b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/NonVolatileBooleanState.java new file mode 100644 index 00000000000..33ea083432c --- /dev/null +++ b/sdk/logs/src/jmh/java/io/opentelemetry/sdk/logs/booleanstate/NonVolatileBooleanState.java @@ -0,0 +1,21 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.logs.booleanstate; + +public class NonVolatileBooleanState implements BooleanState { + + private boolean state; + + @Override + public boolean get() { + return state; + } + + @Override + public void set(boolean state) { + this.state = state; + } +} diff --git a/sdk/logs/src/jmh/java9/io/opentelemetry/sdk/logs/booleanstate/VarHandleEventualBooleanState.java b/sdk/logs/src/jmh/java9/io/opentelemetry/sdk/logs/booleanstate/VarHandleEventualBooleanState.java new file mode 100644 index 00000000000..a9cc9b79d50 --- /dev/null +++ b/sdk/logs/src/jmh/java9/io/opentelemetry/sdk/logs/booleanstate/VarHandleEventualBooleanState.java @@ -0,0 +1,48 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.logs.booleanstate; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; + +public class VarHandleEventualBooleanState implements BooleanState { + + private static final VarHandle STATE_HANDLE; + + static { + try { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + STATE_HANDLE = + lookup.findVarHandle(VarHandleEventualBooleanState.class, "state", boolean.class); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + @SuppressWarnings("UnusedVariable") // Used by VarHandle + private boolean state; + + private int counter = 0; + + public VarHandleEventualBooleanState() { + STATE_HANDLE.setRelease(this, false); + } + + @Override + public boolean get() { + if (counter++ > 1000) { + counter = 0; + return (boolean) STATE_HANDLE.getAcquire(this); + } + + return (boolean) STATE_HANDLE.getOpaque(this); + } + + @Override + public void set(boolean state) { + STATE_HANDLE.setRelease(this, state); + } +} diff --git a/sdk/logs/src/jmh/java9/io/opentelemetry/sdk/logs/booleanstate/VarHandleImmediateBooleanState.java b/sdk/logs/src/jmh/java9/io/opentelemetry/sdk/logs/booleanstate/VarHandleImmediateBooleanState.java new file mode 100644 index 00000000000..72a900d61e1 --- /dev/null +++ b/sdk/logs/src/jmh/java9/io/opentelemetry/sdk/logs/booleanstate/VarHandleImmediateBooleanState.java @@ -0,0 +1,41 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.logs.booleanstate; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; + +public class VarHandleImmediateBooleanState implements BooleanState { + + private static final VarHandle STATE_HANDLE; + + static { + try { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + STATE_HANDLE = + lookup.findVarHandle(VarHandleImmediateBooleanState.class, "state", boolean.class); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } + + @SuppressWarnings("UnusedVariable") // Used by VarHandle + private boolean state; + + public VarHandleImmediateBooleanState() { + STATE_HANDLE.setRelease(this, false); + } + + @Override + public boolean get() { + return (boolean) STATE_HANDLE.getAcquire(this); + } + + @Override + public void set(boolean state) { + STATE_HANDLE.setRelease(this, state); + } +} From 9be852cda88e6fffaef210bcc4336d4e39ce4310 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Tue, 23 Sep 2025 10:13:28 -0700 Subject: [PATCH 2/3] Respect testJavaVersion when running JMH benchmarks --- buildSrc/src/main/kotlin/otel.jmh-conventions.gradle.kts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/buildSrc/src/main/kotlin/otel.jmh-conventions.gradle.kts b/buildSrc/src/main/kotlin/otel.jmh-conventions.gradle.kts index 3e4ad43195c..70b661afabe 100644 --- a/buildSrc/src/main/kotlin/otel.jmh-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/otel.jmh-conventions.gradle.kts @@ -27,6 +27,15 @@ jmh { if (jmhIncludeSingleClass != null) { includes.add(jmhIncludeSingleClass as String) } + + val testJavaVersion = gradle.startParameter.projectProperties.get("testJavaVersion")?.let(JavaVersion::toVersion) + if (testJavaVersion != null) { + val javaExecutable = javaToolchains.launcherFor { + languageVersion.set(JavaLanguageVersion.of(testJavaVersion.majorVersion)) + }.get().executablePath.asFile.absolutePath + + jvm.set(javaExecutable) + } } jmhReport { From dd28501ed9e596e73675f57de9709239dd5e0fa3 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Tue, 23 Sep 2025 13:57:00 -0700 Subject: [PATCH 3/3] Add PR benchmark workflow --- .github/workflows/benchmark-pr.yml | 97 ++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 .github/workflows/benchmark-pr.yml diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml new file mode 100644 index 00000000000..4058a63581a --- /dev/null +++ b/.github/workflows/benchmark-pr.yml @@ -0,0 +1,97 @@ +name: Benchmark PR + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +env: + BENCHMARK_MODULE: sdk/logs + BENCHMARK_CLASSES: BooleanStateBenchmark + +jobs: + sdk-benchmark: + name: Benchmark SDK (Java ${{ matrix.test-java-version }}) + if: contains(github.event.pull_request.labels.*.name, 'run benchmarks') + strategy: + fail-fast: false + matrix: + test-java-version: + - 17 + - 24 + runs-on: oracle-bare-metal-64cpu-512gb-x86-64 + container: + image: ubuntu:24.04@sha256:353675e2a41babd526e2b837d7ec780c2a05bca0164f7ea5dbbd433d21d166fc + timeout-minutes: 20 # since there is only a single bare metal runner across all repos + steps: + - name: Install Git + run: | + apt-get update + apt-get install -y git + + - name: Configure Git safe directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - id: setup-java-test + name: Set up Java ${{ matrix.test-java-version }} for tests + uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 + with: + distribution: temurin + java-version: ${{ matrix.test-java-version }} + + - id: setup-java + name: Set up Java for build + uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 + with: + distribution: temurin + java-version: 17 + + - name: Set up gradle + uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3 + + - name: Build Benchmark + run: ./gradlew jmhJar + + - name: Run Benchmark + run: > + ${{ steps.setup-java-test.outputs.path }}/bin/java + -jar ${{ env.BENCHMARK_MODULE }}/build/libs/opentelemetry-*-jmh.jar + -rf json + ${{ env.BENCHMARK_CLASSES }} + + - name: Rename results + run: mv jmh-result.json jmh-result-pr.json + + - name: Switch to main branch + run: git checkout origin/main + + - name: Build Benchmark on main branch + run: ./gradlew jmhJar + + - name: Run Benchmark on main branch + run: > + ${{ steps.setup-java-test.outputs.path }}/bin/java + -jar ${{ env.BENCHMARK_MODULE }}/build/libs/opentelemetry-*-jmh.jar + -rf json + ${{ env.BENCHMARK_CLASSES }} + # Allow failure when benchmark doesn't exist on main branch (new benchmarks in PR) + continue-on-error: true + + - name: Rename results + run: mv jmh-result.json jmh-result-main.json + + - name: Upload benchmark results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: benchmark-results-java-${{ matrix.test-java-version }} + path: | + jmh-result-pr.json + jmh-result-main.json