feat: added basic benchmark and ci skeleton

.github/workflows/benchmark.yaml

@@ -0,0 +1,81 @@
+name: Benchmarks
+
+on:
+  pull_request: {}
+
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  benchmark:
+    timeout-minutes: ${{ fromJSON(vars.GHA_DEFAULT_TIMEOUT) }}
+    strategy:
+      matrix:
+        kong_image:
+        - 'kong:3.9'
+    env:
+      KONG_ANONYMOUS_REPORTS: "off"
+      KONG_IMAGE: ${{ matrix.kong_image }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout current branch
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 #v4
+        with:
+          path: current
+      - name: Checkout main branch
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 #v4
+        with:
+          path: main
+          ref: main
+      - name: Setup Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5
+        with:
+          go-version-file: current/go.mod
+      - name: Install benchstat
+        run: go install golang.org/x/perf/cmd/benchstat@latest
+      - name: Setup Kong
+        working-directory: current
+        run: make setup-kong
+      - name: Run benchmarks on current branch
+        working-directory: current
+        run: |
+          make benchmark-save
+          mv benchmark_results.txt ../benchmark_results_current.txt
+      - name: Run benchmarks on main branch
+        working-directory: main
+        run: |
+          if grep -q "benchmark-save" Makefile; then
+            make benchmark-save
+            mv benchmark_results.txt ../benchmark_results_main.txt
+          else
+            echo "benchmark-save target not found in main branch, creating placeholder results"
+            # Create a placeholder benchmark result
+            echo "no benchmark data available in main branch" > ../benchmark_results_main.txt
+          fi
+      - name: Compare benchmark results
+        run: |
+          benchstat -confidence 0.6 benchmark_results_main.txt benchmark_results_current.txt | tee benchmark_comparison.txt
+      - name: Comment on PR
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const fs = require('fs');
+            const comparison = fs.readFileSync('benchmark_comparison.txt', 'utf8');
+
+            const comment = `## Benchmark Comparison Results
+
+            \`\`\`
+            ${comparison}
+            \`\`\`
+            `;
+
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: comment
+            });

Makefile

@@ -45,3 +45,11 @@ test-integration:
 .PHONY: clean
 clean:
 	bash .ci/clean_kong.sh
+
+.PHONY: benchmark
+benchmark:
+	go test -tags benchmark -bench . -benchmem -count 5 -benchtime=20x ./tests/benchmarkutils/...
+
+.PHONY: benchmark-save
+benchmark-save:
+	go test -tags benchmark -bench . -benchmem -count 5 -benchtime=20x ./tests/benchmarkutils/... | tee benchmark_results.txt

tests/benchmarkutils/benchmarking_test.go

@@ -0,0 +1,44 @@
+//go:build benchmark
+
+package benchmarkutils
+
+import (
+	"strings"
+	"sync/atomic"
+	"testing"
+)
+
+func runBenchmark(b *testing.B, name string, fn func(b *testing.B) error) {
+	var errCount int64
+
+	b.Run(name, func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			err := fn(b)
+			if err != nil {
+				if !strings.Contains(err.Error(), "EOF") {
+					atomic.AddInt64(&errCount, 1)
+				}
+			}
+		}
+
+		total := b.N
+		errors := atomic.LoadInt64(&errCount)
+
+		b.ReportMetric(float64(errors)/float64(total), "errors/op")
+	})
+}
+
+func BenchmarkSync(b *testing.B) {
+	runBenchmark(b, "sync", func(b *testing.B) error {
+		setup(b)
+
+		b.StartTimer()
+		err := sync(b.Context(), "testdata/sync/kong-huge.yaml")
+		if err != nil {
+			return err
+		}
+		b.StopTimer()
+
+		return nil
+	})
+}

tests/benchmarkutils/test_utils.go

@@ -0,0 +1,49 @@
+package benchmarkutils
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/fatih/color"
+	"github.com/kong/deck/cmd"
+)
+
+func setup(b *testing.B) {
+	b.Setenv("DECK_ANALYTICS", "off")
+	b.Cleanup(func() {
+		reset(b)
+	})
+}
+
+func reset(b *testing.B, opts ...string) {
+	b.Helper()
+
+	deckCmd := cmd.NewRootCmd()
+	args := []string{"gateway", "reset", "-f"}
+	if len(opts) > 0 {
+		args = append(args, opts...)
+	}
+	deckCmd.SetArgs(args)
+	err := deckCmd.Execute()
+	if err != nil {
+		b.Fatalf("failed to reset Kong's state: %s", err.Error())
+	}
+}
+
+func sync(ctx context.Context, kongFile string, opts ...string) error {
+	deckCmd := cmd.NewRootCmd()
+	args := []string{"gateway", "sync", kongFile}
+	if len(opts) > 0 {
+		args = append(args, opts...)
+	}
+	deckCmd.SetArgs(args)
+	_, w, _ := os.Pipe()
+	color.Output = w
+
+	cmdErr := deckCmd.ExecuteContext(ctx)
+
+	w.Close()
+
+	return cmdErr
+}