Skip to content

Commit 89abb92

Browse files
jonahgrahamjonahgraham
committed
Add JMH benchmarks to jsonrpc
This commit adds some basic benchmarks to serve as a starting point for future benchmarking work. On the CI we build the benchmarks, but we don't run them.
1 parent fd40a78 commit 89abb92

File tree

7 files changed

+208
-2
lines changed

7 files changed

+208
-2
lines changed

‎.github/workflows/validate.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ jobs:
6565
cache: gradle
6666

6767
- name: Build with Gradle 🏗️
68-
run: ./gradlew build testOlderJavas
68+
run: ./gradlew build jmhCompileGeneratedClasses testOlderJavas
6969

7070
- name: Upload Test Results
7171
if: always()

‎documentation/benchmarking.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# JSON-RPC Benchmarking
2+
3+
The org.eclipse.lsp4j.jsonrpc project comes with some [JMH](https://github.com/openjdk/jmh) based benchmarks.
4+
This are useful to perform analysis when doing micro-optimization of the JSON-RPC implementation.
5+
6+
## Running Benchmarks
7+
8+
At the command line run `./gradlew jmh` to run the benchmarks.
9+
A lot of output, including results and caveats from JMH, is provided.
10+
Additionally, the result summaries are written to `org.eclipse.lsp4j.jsonrpc/build/results/jmh/results.txt`
11+
12+
See the [JMH Gradle Plug-in readme](https://github.com/melix/jmh-gradle-plugin?tab=readme-ov-file#readme) and the [OpenJDK JMH readme](https://github.com/openjdk/jmh) for more details on how best to use JMH.
13+
14+
## Running a single benchmark
15+
16+
To run a single benchmark, add `-PjmhIncludes=NameOfBenchmark` to the `./gradlew` command line.
17+
18+
## Building and running a jmh jar
19+
20+
A typical way of using jmh is to build a self-contained jar.
21+
Do this by running `./gradlew jmhJar`, the resulting jar is `org.eclipse.lsp4j.jsonrpc/build/libs/org.eclipse.lsp4j.jsonrpc-1.0.0-SNAPSHOT-jmh.jar` (with `1.0.0-SNAPSHOT` replaced with current version of LSP4J).
22+
This jar can then be used with the normal command line options of jmh, run with `-h` to see what they are:
23+
24+
```sh
25+
java -jar org.eclipse.lsp4j.jsonrpc/build/libs/org.eclipse.lsp4j.jsonrpc-1.0.0-SNAPSHOT-jmh.jar -h
26+
```
27+
28+
## Running jmh and gradle is doing nothing?
29+
30+
Gradle's incremental build system may prevent subsequent runs of the `jmh` task from doing anything.
31+
This happens because gradle does not think anything has changed, so there is nothing to do.
32+
Delete the results file (`org.eclipse.lsp4j.jsonrpc/build/results/jmh/results.txt`) or do a clean build (`./gradlew clean jmh`).
33+
34+
## Running jmh within Eclipse
35+
36+
At the time of writing there is no plug-in available for Eclipse to simplify running JMH within the IDE.
37+
LSP4J provides a Buildship launch configuration called `lsp4j-jmh` which can be used to run within the IDE.
38+
This task can be used once the project is imported in to your Eclipse workspace as explained in the [contribution guide](https://github.com/eclipse-lsp4j/lsp4j/blob/main/CONTRIBUTING.md#eclipse).

‎lsp4j-jmh.launch‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2+
<launchConfiguration type="org.eclipse.buildship.core.launch.runconfiguration">
3+
<listAttribute key="arguments"/>
4+
<booleanAttribute key="build_scans_enabled" value="false"/>
5+
<stringAttribute key="gradle_distribution" value="GRADLE_DISTRIBUTION(WRAPPER)"/>
6+
<stringAttribute key="gradle_user_home" value=""/>
7+
<stringAttribute key="java_home" value=""/>
8+
<listAttribute key="jvm_arguments"/>
9+
<booleanAttribute key="offline_mode" value="false"/>
10+
<booleanAttribute key="org.eclipse.debug.core.ATTR_FORCE_SYSTEM_CONSOLE_ENCODING" value="false"/>
11+
<booleanAttribute key="override_workspace_settings" value="false"/>
12+
<booleanAttribute key="show_console_view" value="true"/>
13+
<booleanAttribute key="show_execution_view" value="true"/>
14+
<listAttribute key="tasks">
15+
<listEntry value="jmh"/>
16+
</listAttribute>
17+
<stringAttribute key="working_dir" value="${workspace_loc:/lsp4j}"/>
18+
</launchConfiguration>

‎org.eclipse.lsp4j.jsonrpc/build.gradle‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
* SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
1111
******************************************************************************/
1212

13+
plugins {
14+
id("me.champeau.jmh") version "0.7.3"
15+
}
16+
1317
ext.title = 'LSP4J JSON-RPC'
1418
description = 'Generic JSON-RPC implementation'
1519

@@ -21,3 +25,14 @@ dependencies {
2125
jar.bundle.bnd(
2226
'Import-Package': "com.google.gson.*;version=\"$versions.gson\",*"
2327
)
28+
29+
// Add, for example, -PjmhIncludes=StreamMessageProducerBenchmark, to command line
30+
// to only run that one benchmark
31+
def jmhIncludes = project.findProperty("jmhIncludes")
32+
33+
jmh {
34+
profilers = ['gc']
35+
if (jmhIncludes != null) {
36+
includes = [jmhIncludes] // can be simple name or regex
37+
}
38+
}

‎org.eclipse.lsp4j.jsonrpc/src/jmh/java/org/eclipse/lsp4j/jsonrpc/jmh/StreamMessageConsumerBenchmark.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/******************************************************************************
2+
* Copyright (c) 2025 Kichwa Coders Canada, Inc. and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0,
7+
* or the Eclipse Distribution License v. 1.0 which is available at
8+
* http://www.eclipse.org/org/documents/edl-v10.php.
9+
*
10+
* SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
11+
******************************************************************************/
12+
package org.eclipse.lsp4j.jsonrpc.jmh;
13+
14+
import static java.util.Collections.emptyMap;
15+
16+
import java.io.OutputStream;
17+
import java.util.HashMap;
18+
import java.util.Map;
19+
import java.util.concurrent.TimeUnit;
20+
21+
import org.eclipse.lsp4j.jsonrpc.json.MessageJsonHandler;
22+
import org.eclipse.lsp4j.jsonrpc.json.StreamMessageConsumer;
23+
import org.eclipse.lsp4j.jsonrpc.messages.RequestMessage;
24+
import org.openjdk.jmh.annotations.Benchmark;
25+
import org.openjdk.jmh.annotations.BenchmarkMode;
26+
import org.openjdk.jmh.annotations.Fork;
27+
import org.openjdk.jmh.annotations.Measurement;
28+
import org.openjdk.jmh.annotations.Mode;
29+
import org.openjdk.jmh.annotations.OutputTimeUnit;
30+
import org.openjdk.jmh.annotations.Scope;
31+
import org.openjdk.jmh.annotations.Setup;
32+
import org.openjdk.jmh.annotations.State;
33+
import org.openjdk.jmh.annotations.Warmup;
34+
35+
@BenchmarkMode(Mode.AverageTime)
36+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
37+
@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
38+
@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
39+
@Fork(1)
40+
@State(Scope.Benchmark)
41+
public class StreamMessageConsumerBenchmark {
42+
private StreamMessageConsumer consumer;
43+
private RequestMessage message;
44+
45+
@SuppressWarnings("resource")
46+
@Setup
47+
public void setup() {
48+
consumer = new StreamMessageConsumer(OutputStream.nullOutputStream(), new MessageJsonHandler(emptyMap()));
49+
message = new RequestMessage();
50+
message.setId("1");
51+
message.setMethod("foo");
52+
Map<String, String> map = new HashMap<>();
53+
for (int i = 0; i < 100; i++) {
54+
map.put(String.valueOf(i), "X".repeat(i));
55+
}
56+
message.setParams(map);
57+
}
58+
59+
@Benchmark
60+
public void measure() {
61+
consumer.consume(message);
62+
}
63+
}

‎org.eclipse.lsp4j.jsonrpc/src/jmh/java/org/eclipse/lsp4j/jsonrpc/jmh/StreamMessageProducerBenchmark.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/******************************************************************************
2+
* Copyright (c) 2025 Kichwa Coders Canada, Inc. and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0,
7+
* or the Eclipse Distribution License v. 1.0 which is available at
8+
* http://www.eclipse.org/org/documents/edl-v10.php.
9+
*
10+
* SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
11+
******************************************************************************/
12+
package org.eclipse.lsp4j.jsonrpc.jmh;
13+
14+
import static java.util.Collections.emptyMap;
15+
16+
import java.io.ByteArrayInputStream;
17+
import java.io.ByteArrayOutputStream;
18+
import java.util.HashMap;
19+
import java.util.Map;
20+
import java.util.concurrent.TimeUnit;
21+
22+
import org.eclipse.lsp4j.jsonrpc.json.MessageJsonHandler;
23+
import org.eclipse.lsp4j.jsonrpc.json.StreamMessageConsumer;
24+
import org.eclipse.lsp4j.jsonrpc.json.StreamMessageProducer;
25+
import org.eclipse.lsp4j.jsonrpc.messages.RequestMessage;
26+
import org.openjdk.jmh.annotations.Benchmark;
27+
import org.openjdk.jmh.annotations.BenchmarkMode;
28+
import org.openjdk.jmh.annotations.Fork;
29+
import org.openjdk.jmh.annotations.Measurement;
30+
import org.openjdk.jmh.annotations.Mode;
31+
import org.openjdk.jmh.annotations.OutputTimeUnit;
32+
import org.openjdk.jmh.annotations.Scope;
33+
import org.openjdk.jmh.annotations.Setup;
34+
import org.openjdk.jmh.annotations.State;
35+
import org.openjdk.jmh.annotations.Warmup;
36+
import org.openjdk.jmh.infra.Blackhole;
37+
38+
@BenchmarkMode(Mode.AverageTime)
39+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
40+
@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
41+
@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
42+
@Fork(1)
43+
@State(Scope.Benchmark)
44+
public class StreamMessageProducerBenchmark {
45+
46+
private StreamMessageProducer messageProducer;
47+
private ByteArrayInputStream bais;
48+
49+
@Setup
50+
public void setup() {
51+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
52+
StreamMessageConsumer consumer = new StreamMessageConsumer(baos, new MessageJsonHandler(emptyMap()));
53+
RequestMessage message = new RequestMessage();
54+
message.setId("1");
55+
message.setMethod("foo");
56+
Map<String, String> map = new HashMap<>();
57+
for (int i = 0; i < 100; i++) {
58+
map.put(String.valueOf(i), "X".repeat(i));
59+
}
60+
message.setParams(map);
61+
consumer.consume(message);
62+
byte[] byteArray = baos.toByteArray();
63+
bais = new ByteArrayInputStream(byteArray);
64+
messageProducer = new StreamMessageProducer(bais, new MessageJsonHandler(emptyMap()));
65+
}
66+
67+
@Benchmark
68+
public void measure(Blackhole bh) {
69+
bais.reset();
70+
messageProducer.listen(bh::consume);
71+
}
72+
}

‎releng/build.Jenkinsfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ pipeline {
4040
-PignoreTestFailures=true \
4141
--refresh-dependencies \
4242
--continue \
43-
clean build testOlderJavas signJar publish \
43+
clean build jmhCompileGeneratedClasses testOlderJavas signJar publish \
4444
"
4545
}
4646
}

0 commit comments

Comments
 (0)