New: Add benchmarks to compare coverage approaches.

Author: LlamaLad7

benchmarks/jmh.gradle

@@ -74,27 +74,38 @@ def configureBenchmark(JavaExec benchmark, Closure<List<String>> jvmArgs = {[]})
   }
 }
 
+ext.configureBenchmark = this.&configureBenchmark
+
 def benchmarkReport(Task benchmark) {
   file("$benchmark.temporaryDir/${benchmark.name}.json")
 }
 
-
-ext.configureCompareWith = { benchmark, Closure<List<String>> jvmArgs, basicBenchmark ->
-  configureBenchmark(benchmark, jvmArgs)
-  benchmark.with {
-    dependsOn(basicBenchmark)
+ext.configureComparison = { combined, benchmark1, benchmark2 ->
+  combined.with {
+    group = 'benchmarks'
+    if (benchmark1 != combined) {
+      dependsOn benchmark1
+    }
+    if (benchmark2 != combined) {
+      dependsOn benchmark2
+    }
     doLast {
-      def noAgentReport = ResourceGroovyMethods.getText(benchmarkReport(basicBenchmark), 'UTF-8')
-      def currentReport = ResourceGroovyMethods.getText(benchmarkReport(benchmark), 'UTF-8')
+      def firstReport = ResourceGroovyMethods.getText(benchmarkReport(benchmark1), 'UTF-8')
+      def secondReport = ResourceGroovyMethods.getText(benchmarkReport(benchmark2), 'UTF-8')
 
       project.logger.quiet """Benchmark score:
-      $basicBenchmark.name vs $benchmark.name:
-      ${ReportReader.readScore(noAgentReport, currentReport, secondaryMetrics)}
+      $benchmark1.name vs $benchmark2.name:
+      ${ReportReader.readScore(firstReport, secondReport, secondaryMetrics)}
       """
     }
   }
 }
 
-ext.configureCompare = { benchmark, Closure<List<String>> jvmArgs ->
+ext.configureCompareWith = { benchmark, Closure<List<String>> jvmArgs, basicBenchmark ->
+  configureBenchmark(benchmark, jvmArgs)
+  configureComparison(benchmark, basicBenchmark, benchmark)
+}
+
+ext.configureCompareWithNoAgent = { benchmark, Closure<List<String>> jvmArgs ->
   configureCompareWith(benchmark, jvmArgs, NoAgent)
 }

test-discovery/jmh.gradle

@@ -44,10 +44,10 @@ def agentParams(Configuration configuration) {
 }
 
 task Baseline(type: JavaExec) {
-  configureCompare(it) { agentParams(configurations.baseline) }
+  configureCompareWithNoAgent(it) { agentParams(configurations.baseline) }
 }
 
 task Head(type: JavaExec) {
   dependsOn configurations.head.buildDependencies
-  configureCompare(it) { agentParams(configurations.head) }
+  configureCompareWithNoAgent(it) { agentParams(configurations.head) }
 }

tests/jmh.gradle

@@ -34,25 +34,25 @@ dependencies {
   baseline "$group:$coverage_jar_name:$baselineVersion"
 }
 
-def ijAgentParams(Configuration configuration, boolean branchCoverage) {
+def ijAgentParams(Configuration configuration, boolean branchCoverage, String... jvmArgs) {
   def agentPath = configuration == configurations.head
       ? rootProject.file("dist").listFiles().find { it.name.startsWith(coverage_jar_name) }.absolutePath
       : configuration.find { it.name.startsWith(coverage_jar_name) }.absolutePath
   return [
       "-javaagent:${agentPath}=${coverageFile} false false false ${!branchCoverage} org.joda.* org.apache.commons.*",
-      "-Didea.new.sampling.coverage=true", "-Didea.new.tracing.coverage=true"
+      *jvmArgs
   ]
 }
 
 def branchCoverage = true
 
 task BaselineCoverage(type: JavaExec) {
-  configureCompare(it) { ijAgentParams(configurations.baseline, branchCoverage) }
+  configureCompareWithNoAgent(it) { ijAgentParams(configurations.baseline, branchCoverage) }
   clear(it)
 }
 
 task HeadCoverage(type: JavaExec) {
-  configureCompare(it) { ijAgentParams(configurations.head, branchCoverage) }
+  configureCompareWithNoAgent(it) { ijAgentParams(configurations.head, branchCoverage) }
   clear(it)
 }
 
@@ -66,6 +66,54 @@ task LineVsBranchCoverage(type: JavaExec) {
   clear(it)
 }
 
+task CondyCoverage(type: JavaExec) {
+  configureBenchmark(it) {
+    ijAgentParams(configurations.head, branchCoverage)
+  }
+  clear(it)
+}
+
+task IndyCoverage(type: JavaExec) {
+  configureBenchmark(it) {
+    ijAgentParams(configurations.head, branchCoverage, "-Dcoverage.condy.enable=false")
+  }
+  clear(it)
+}
+
+task FieldCoverage(type: JavaExec) {
+  configureBenchmark(it) {
+    ijAgentParams(configurations.head, branchCoverage, "-Dcoverage.condy.enable=false", "-Dcoverage.indy.enable=false")
+  }
+  clear(it)
+}
+
+task OldCoverage(type: JavaExec) {
+  configureBenchmark(it) {
+    ijAgentParams(configurations.head, branchCoverage, "-Didea.new.sampling.coverage=false", "-Didea.new.tracing.coverage=false")
+  }
+  clear(it)
+}
+
+task IndyVsCondyCoverage {
+  configureComparison(it, IndyCoverage, CondyCoverage)
+  clear(it)
+}
+
+task FieldVsIndyCoverage {
+  configureComparison(it, FieldCoverage, IndyCoverage)
+  clear(it)
+}
+
+task OldVsFieldCoverage {
+  configureComparison(it, OldCoverage, FieldCoverage)
+  clear(it)
+}
+
+task FieldVsCondyCoverage {
+  configureComparison(it, FieldCoverage, CondyCoverage)
+  clear(it)
+}
+
 def clear(Task task) {
   task.doLast {
     delete(coverageFile)