fix: move Makefile out of report structure as it can generate the structure

aslakknutsen · aslakknutsen · commit 23a6c79a0e60 · 2025-07-23T12:21:01.000+02:00
diff --git a/conformance/reports/v0.4.0/gateway/istio/README.md b/conformance/reports/v0.4.0/gateway/istio/README.md
@@ -1,153 +1,23 @@
-# Istio (Gateway Profile Conformance)
+# Istio (Gateway Profile Conformance) - v0.4.0
 
+## Test Results
 
-## Table of Contents
+This directory contains conformance test results for Gateway API Inference Extension v0.4.0 testing against Istio implementations using the Gateway profile.
 
 | Extension Version Tested | Profile Tested | Implementation Version | Mode    | Report | Status |
 |--------------------------|----------------|------------------------|---------|--------|--------|
 | v1.3.0 | Gateway | 1.27-alpha.0551127f00634403cddd4634567e65a8ecc499a7 | default | [1.27-alpha.0551127f00634403cddd4634567e65a8ecc499a7-default-gateway-report.yaml](./1.27-alpha.0551127f00634403cddd4634567e65a8ecc499a7-default-gateway-report.yaml) | PASS |
 | ...                      | ...            | ...                    | ...     | ...    | ...    |
 
-## Reproduce
+## Running the Tests
 
-To reproduce the test environment and run the conformance tests, choose **one** of these deployment environments:
+For instructions on how to reproduce these test results and run the conformance tests yourself, see the [Istio Conformance Testing README](../../../../scripts/istio/README.md).
 
-### Option A: Minikube Environment
+## About This Version
 
-```bash
-# Set up the complete environment (minikube, istio, CRDs, TLS)
-make setup-env-minikube
+- **Extension Version**: v0.4.0
+- **Profile**: Gateway
+- **Implementation**: Istio
+- **Test Mode**: Default
 
-# Run conformance tests
-make run-tests
-
-# Clean up resources when done
-make clean
-```
-
-### Option B: Kind Environment
-
-```bash
-# Set up the complete environment (kind, istio, CRDs, TLS)
-make setup-env-kind
-
-# Run conformance tests
-make run-tests
-
-# Clean up resources when done
-make clean
-```
-
-### Option C: OpenShift Environment
-
-```bash
-# Set up the complete environment (openshift, istio, CRDs, TLS)
-make setup-env-openshift
-
-# Run conformance tests
-make run-tests
-
-# Clean up resources when done
-make clean
-```
-
-### Individual Setup Steps
-
-If you prefer to run individual setup steps:
-
-```bash
-# Choose ONE of these deployment environments:
-make setup-minikube     # Option A: Set up minikube with metallb
-# OR
-make setup-kind         # Option B: Set up kind cluster with metallb
-# OR
-make setup-openshift    # Option C: Set up OpenShift environment
-
-# Use specific Kubernetes version (works for both kind and minikube):
-make setup-minikube KUBERNETES_VERSION=v1.31.9
-make setup-kind KUBERNETES_VERSION=v1.31.9
-
-# Then continue with common setup steps:
-make setup-istio                        # Install Istio (automatically detects hub)
-# OR use environment-specific targets:
-make setup-istio-minikube               # Install Istio for minikube environment
-make setup-istio-kind                   # Install Istio for kind environment
-make setup-istio-openshift              # Install Istio for OpenShift environment
-
-make setup-gateway-api-crds             # Apply Gateway API CRDs
-make setup-inference-extension-crds     # Apply Inference Extension CRDs
-# OR
-make setup-crds                         # Apply all CRDs (convenience target)
-make setup-tls                          # Set up TLS for EPP
-```
-
-### CRD Setup Options
-
-The CRD setup is split into two targets for flexibility:
-
-- `make setup-gateway-api-crds` - Install only Gateway API CRDs (if you have a base Gateway API environment)
-- `make setup-inference-extension-crds` - Install only Inference Extension CRDs (if Gateway API is already installed)
-- `make setup-crds` - Install both (convenience target for full setup)
-
-Different environments might require different combinations depending on what's already installed.
-
-### Version Configuration
-
-The Makefile uses configurable version variables that can be overridden:
-
-```bash
-# Use default versions
-make setup-env-minikube
-
-# Override specific versions
-make setup-env-minikube ISTIO_VERSION=1.28.0
-make setup-env-kind GATEWAY_API_VERSION=v1.4.0 INFERENCE_EXTENSION_VERSION=v0.5.0
-make setup-env-openshift GATEWAY_API_VERSION=v1.4.0 INFERENCE_EXTENSION_VERSION=v0.5.0
-
-# Use specific Kubernetes version (works for both kind and minikube)
-make setup-env-minikube KUBERNETES_VERSION=v1.31.9
-make setup-env-kind KUBERNETES_VERSION=v1.31.9
-
-# Combine version overrides
-make setup-env-minikube KUBERNETES_VERSION=v1.31.9 ISTIO_VERSION=1.28.0
-
-# Use environment-specific Istio installation
-make setup-istio-minikube                  # Install Istio for minikube environment
-make setup-istio-kind                      # Install Istio for kind environment
-make setup-istio-openshift                 # Install Istio for OpenShift environment
-
-# Run conformance tests
-make run-tests                             # Run all conformance tests
-make run-tests RUN_TEST=InferencePoolAccepted  # Run specific test only
-
-# See all available options
-make help
-```
-
-### Available Variables:
-
-**Main Variables:**
-- `GATEWAY_API_VERSION` - Gateway API version (default: v1.3.0)
-- `INFERENCE_EXTENSION_VERSION` - Inference Extension version (default: v0.4.0)  
-- `KUBERNETES_VERSION` - Kubernetes version for kind/minikube (default: v1.30.13)
-- `ISTIO_VERSION` - Istio version (default: 1.27-alpha.0551127f00634403cddd4634567e65a8ecc499a7)
-
-**Conformance Test Variables:**
-- `MODE` - Test mode (default: default)
-- `PROFILE` - Conformance profile (default: gateway)
-- `ORGANIZATION` - Organization name (default: istio)
-- `PROJECT` - Project name (default: istio)
-- `URL` - Project URL (default: https://istio.io)
-- `CONTACT` - Contact information (default: @istio/maintainers)
-- `RUN_TEST` - Run specific test (default: empty, runs all tests)
-
-**Advanced Variables:**
-For advanced users who need to customize internal behavior, additional variables are available including `ISTIO_HUB`, `ISTIO_PROFILE`, `TEST_BASE_DIR`, `REPORT_BASE_DIR`, and others. See `make help` for a complete list.
-
-> **Note**: This Makefile expects to be run from within the `gateway-api-inference-extension` repository structure. The test suite is located at the repository root, while conformance reports are generated in the current directory.
-
-### Conformance Testing
-
-For detailed information about conformance testing, report generation, and requirements, see the [conformance README](../../../README.md).
-
-For available commands and options specific to this Istio implementation, run `make help` or see the `Makefile` in this directory.
+For detailed information about conformance testing, report generation, and requirements, see the [main conformance README](../../../../../README.md).
diff --git a/conformance/scripts/istio/Makefile b/conformance/scripts/istio/Makefile
@@ -17,8 +17,8 @@ CONTACT ?= @istio/maintainers
 RUN_TEST ?=
 
 # Directory variables
-TEST_BASE_DIR ?= ../../../../..
-REPORT_BASE_DIR ?= .
+TEST_BASE_DIR ?= ../../..
+REPORT_BASE_DIR ?= ../../reports/$(INFERENCE_EXTENSION_VERSION)/$(PROFILE)/$(PROJECT)
 
 # Kubernetes version (shared between kind and minikube)
 KUBERNETES_VERSION ?= v1.30.13
@@ -91,7 +91,7 @@ spec:
         insecureSkipVerify: true
 endef
 
-.PHONY: setup-env-minikube setup-env-kind setup-env-openshift setup-minikube setup-kind setup-openshift setup-istio setup-istio-minikube setup-istio-kind setup-istio-openshift setup-gateway-api-crds setup-inference-extension-crds setup-crds setup-tls run-tests update-readme clean clean-reports help
+.PHONY: setup-env-minikube setup-env-kind setup-env-openshift setup-minikube setup-kind setup-openshift setup-istio setup-istio-minikube setup-istio-kind setup-istio-openshift setup-gateway-api-crds setup-inference-extension-crds setup-crds setup-tls run-tests update-readme clean clean-reports help ensure-report-dir
 
 # Setup complete environment with minikube
 setup-env-minikube: setup-minikube setup-istio-minikube setup-gateway-api-crds setup-inference-extension-crds setup-tls
@@ -157,6 +157,12 @@ help:
 	@echo "  INFERENCE_EXTENSION_VERSION      - Inference Extension version (current: $(INFERENCE_EXTENSION_VERSION))"
 	@echo "  KUBERNETES_VERSION               - Kubernetes version for kind/minikube (current: $(KUBERNETES_VERSION))"
 	@echo "  ISTIO_VERSION                    - Istio version (current: $(ISTIO_VERSION))"
+	@echo "  PROFILE                          - Conformance profile (current: $(PROFILE))"
+	@echo "  PROJECT                          - Project name (current: $(PROJECT))"
+	@echo ""
+	@echo "📁 REPORT STRUCTURE:"
+	@echo "  Reports will be generated in: $(REPORT_BASE_DIR)/"
+	@echo "  This structure is based on: reports/[INFERENCE_EXTENSION_VERSION]/[PROFILE]/[PROJECT]/"
 	@echo ""
 	@echo "📥 Automatic istioctl Management:"
 	@echo "  • istioctl binary will be automatically downloaded based on ISTIO_VERSION"
@@ -167,9 +173,7 @@ help:
 	@echo ""
 	@echo "Conformance test variables (can be overridden):"
 	@echo "  MODE                             - Test mode (current: $(MODE))"
-	@echo "  PROFILE                          - Conformance profile (current: $(PROFILE))"
 	@echo "  ORGANIZATION                     - Organization name (current: $(ORGANIZATION))"
-	@echo "  PROJECT                          - Project name (current: $(PROJECT))"
 	@echo "  URL                              - Project URL (current: $(URL))"
 	@echo "  CONTACT                          - Contact information (current: $(CONTACT))"
 	@echo "  RUN_TEST                         - Run specific test (current: $(RUN_TEST))"
@@ -217,11 +221,17 @@ help:
 	@echo ""
 	@echo "🧪 Testing Examples:"
 	@echo "  make run-tests                             - Run conformance tests with default settings"
-	@echo "  make run-tests TEST_BASE_DIR=../../../../../ - Run tests from custom directory"
+	@echo "  make run-tests TEST_BASE_DIR=..            - Run tests from custom directory"
 	@echo "  make run-tests RUN_TEST=InferencePoolAccepted - Run specific test only"
+	@echo "  make run-tests PROJECT=envoy PROFILE=mesh  - Run tests for different project/profile"
 	@echo "  make update-readme                         - Update README table with all available reports"
 	@echo ""
 
+# Ensure report directory exists
+ensure-report-dir:
+	@echo "Ensuring report directory exists: $(REPORT_BASE_DIR)"
+	@mkdir -p $(REPORT_BASE_DIR)
+
 # Setup minikube with metallb
 setup-minikube:
 	@echo "Setting up minikube with metallb..."
@@ -388,7 +398,7 @@ setup-tls:
 	@rm -f /tmp/tls-destination-rules.yaml
 
 # Run conformance tests
-run-tests:
+run-tests: ensure-report-dir
 	@echo "Running conformance tests..."
 	@echo "Test base directory: $(TEST_BASE_DIR)"
 	@echo "Report will be saved as: $(REPORT_BASE_DIR)/$(IMPLEMENTATION_VERSION)-$(MODE)-$(PROFILE)-report.yaml"
@@ -399,7 +409,7 @@ run-tests:
 	fi
 	cd $(TEST_BASE_DIR) && go test ./conformance -args -gateway-class istio \
 		-cleanup-base-resources=false \
-		-report-output=$(shell pwd)/$(IMPLEMENTATION_VERSION)-$(MODE)-$(PROFILE)-report.yaml \
+		-report-output=$(shell pwd)/$(REPORT_BASE_DIR)/$(IMPLEMENTATION_VERSION)-$(MODE)-$(PROFILE)-report.yaml \
 		-organization="$(ORGANIZATION)" \
 		-project="$(PROJECT)" \
 		-url="$(URL)" \
@@ -408,14 +418,15 @@ run-tests:
 		-mode="$(MODE)" \
 		-run-test="$(RUN_TEST)"
 	@echo ""
-	@echo "Test completed! To update the README table, run:"
+	@echo "Test completed! Report saved to: $(REPORT_BASE_DIR)/$(IMPLEMENTATION_VERSION)-$(MODE)-$(PROFILE)-report.yaml"
+	@echo "To update the README table, run:"
 	@echo "  make update-readme"
 
 # Update README table with all available reports
 update-readme:
 	@echo "Updating README table with all available reports..."
-	@echo "Scanning for conformance reports..."
-	@REPORTS=$$(ls *-*-*-report.yaml 2>/dev/null || echo ""); \
+	@echo "Scanning for conformance reports in reports/ directory..."
+	@cd reports && REPORTS=$$(find . -name "*-*-*-report.yaml" -type f 2>/dev/null | sort || echo ""); \
 	if [ -z "$$REPORTS" ]; then \
 		echo "No conformance reports found. Run 'make run-tests' to generate reports first."; \
 		exit 1; \
@@ -458,7 +469,7 @@ update-readme:
 			TABLE_ENTRIES="$$TABLE_ENTRIES\n$$TABLE_ENTRY"; \
 		fi; \
 	done; \
-	echo "Updating README.md..."; \
+	cd .. && echo "Updating README.md..."; \
 	sed -i '/^| Extension Version Tested | Profile Tested | Implementation Version | Mode    | Report | Status |$$/,/^| \.\.\. *| \.\.\. *| \.\.\. *| \.\.\. *| \.\.\. *| \.\.\. *|$$/{//!d;}' README.md; \
 	sed -i '/^| Extension Version Tested | Profile Tested | Implementation Version | Mode    | Report | Status |$$/a\|--------------------------|----------------|------------------------|---------|--------|--------|\n'"$$TABLE_ENTRIES" README.md; \
 	echo "README table updated successfully!"
@@ -477,11 +488,7 @@ clean:
 # Clean up generated conformance reports
 clean-reports:
 	@echo "Cleaning up generated conformance reports..."
-	rm -f *-*-*-report.yaml
+	rm -f reports/*/*/*/*-*-*-report.yaml
 	@echo "Cleaning up README table entries..."
 	@sed -i '/^| [^E][^t].*\.yaml.*|$$/d' README.md
-	@echo "Conformance reports and README table cleaned up"
-
-
-
- 
+	@echo "Conformance reports and README table cleaned up" 
diff --git a/conformance/scripts/istio/README.md b/conformance/scripts/istio/README.md
