[Test] Add load tests and behavioral checks to incremental upgrade E2E

.buildkite/test-e2e.yml

@@ -94,7 +94,7 @@
     - set -o pipefail
     - mkdir -p "$(pwd)/tmp" && export KUBERAY_TEST_OUTPUT_DIR=$(pwd)/tmp
     - echo "KUBERAY_TEST_OUTPUT_DIR=$$KUBERAY_TEST_OUTPUT_DIR"
-    - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v ./test/e2eincrementalupgrade 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-operator.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-log.tar -T - && exit 1)
+    - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=10m KUBERAY_TEST_TIMEOUT_LONG=20m go test -timeout 60m -v ./test/e2eincrementalupgrade 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-operator.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-log.tar -T - && exit 1)
     - echo "--- END:RayService Incremental Upgrade E2E (nightly operator) tests finished"
 
 - label: 'Test Autoscaler E2E Part 1 (nightly operator)'

ray-operator/test/e2eincrementalupgrade/constant.go

@@ -0,0 +1,121 @@
+package e2eincrementalupgrade
+
+import "k8s.io/utils/ptr"
+
+// These parameters control capacity scaling and gradual traffic migration during the upgrade.
+type incrementalUpgradeParams struct {
+	Name     string
+	StepSize int32
+	Interval int32
+	MaxSurge int32
+}
+
+// incrementalUpgradeCombinations defines diverse (stepSize, interval, maxSurge) combinations
+// to exercise different upgrade behaviors. Each combination targets a distinct scenario.
+var incrementalUpgradeCombinations = []incrementalUpgradeParams{
+	{
+		// Scenario: Instant cutover.
+		// All capacity and traffic shift in one step, which behaves like a blue/green deployment.
+		StepSize: 100,
+		Interval: 1,
+		MaxSurge: 100,
+		Name:     "BlueGreen",
+	},
+	{
+		// Scenario: Aggressive gradual upgrade.
+		// Larger traffic migration steps with shorter intervals.
+		StepSize: 25,
+		Interval: 5,
+		MaxSurge: 50,
+		Name:     "AggressiveGradual",
+	},
+	{
+		// Scenario: Conservative gradual upgrade.
+		// Smaller traffic migration steps with longer intervals.
+		StepSize: 5,
+		Interval: 10,
+		MaxSurge: 25,
+		Name:     "ConservativeGradual",
+	},
+}
+
+// ptrs returns (*stepSize, *interval, *maxSurge) for use with the RayService bootstrap helper.
+func (p incrementalUpgradeParams) ptrs() (*int32, *int32, *int32) {
+	return ptr.To(p.StepSize), ptr.To(p.Interval), ptr.To(p.MaxSurge)
+}
+
+// The following defines the Serve configurations for different types of incremental upgrade tests, including:
+//   - Functional test
+//   - High-RPS Locust load test
+//
+// NOTE: working_dir is coupled with the external GitHub repos, which might lead to CI flakiness considering the
+// availability and stability of these repos and specific commit hashes.
+
+type serveConfigV2 string
+
+// defaultIncrementalUpgradeServeConfigV2 configures a Serve app for functional tests.
+const defaultIncrementalUpgradeServeConfigV2 serveConfigV2 = `applications:
+  - name: fruit_app
+    import_path: fruit.deployment_graph
+    route_prefix: /fruit
+    runtime_env:
+      working_dir: "https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip"
+    deployments:
+      - name: MangoStand
+        num_replicas: 1
+        user_config:
+          price: 3
+        ray_actor_options:
+          num_cpus: 0.1
+      - name: OrangeStand
+        num_replicas: 1
+        user_config:
+          price: 2
+        ray_actor_options:
+          num_cpus: 0.1
+      - name: FruitMarket
+        num_replicas: 1
+        ray_actor_options:
+          num_cpus: 0.1
+  - name: math_app
+    import_path: conditional_dag.serve_dag
+    route_prefix: /calc
+    runtime_env:
+      working_dir: "https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip"
+    deployments:
+      - name: Adder
+        num_replicas: 1
+        user_config:
+          increment: 3
+        ray_actor_options:
+          num_cpus: 0.1
+      - name: Multiplier
+        num_replicas: 1
+        user_config:
+          factor: 5
+        ray_actor_options:
+          num_cpus: 0.1
+      - name: Router
+        num_replicas: 1
+        ray_actor_options:
+          num_cpus: 0.1
+`
+
+// highRPSServeConfigV2 configures a minimal high-RPS Serve app (SimpleDeployment) for Locust load tests.
+const highRPSServeConfigV2 serveConfigV2 = `applications:
+  - name: simple_app
+    import_path: simple_serve.app
+    route_prefix: /test
+    runtime_env:
+      working_dir: "https://github.com/jiangjiawei1103/incr-upgrade-locust/archive/a185bb29374388e801db4331ae73af3ad1e79a5f.zip"
+    deployments:
+      - name: SimpleDeployment
+        autoscaling_config:
+          min_replicas: 1
+          max_replicas: 3
+          target_ongoing_requests: 2
+          max_ongoing_requests: 6
+          upscale_delay_s: 0.5
+        ray_actor_options:
+          num_cpus: 2
+`