diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..9f98cb8c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +# Exclude unnecessary files from Docker build context +.git +.gitignore +.travis.yml +.github +*.md +*.sh +*.go +go.mod +go.sum +bin/ +dist/ +examples/ +scripts/ +docs/ +*.tf +terraform.tfstate* +.terraform/ +.DS_Store +*.log +*.backup +Makefile +GNUmakefile +GORELEASER_GPGSIGNING_PLAN.md +TERRAFORM_BINARY_OPTIMIZATION_PLAN.md +TESTCONTAINERS_*.md +TIDB_*.md +WORKFLOW_OPTIMIZATION_ANALYSIS.md +terraform-registry-manifest.json +VERSION diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9c65000..eea327a1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -75,8 +75,28 @@ jobs: - name: Vendor Go dependencies run: go mod vendor + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and cache TiUP Playground Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.tiup-playground + tags: terraform-provider-mysql-tiup-playground:latest + cache-from: type=gha + cache-to: type=gha,mode=max + push: false + load: true + + - name: Save TiUP Playground Docker image + run: | + docker save terraform-provider-mysql-tiup-playground:latest | gzip > tiup-playground-image.tar.gz + echo "Image saved: $(du -h tiup-playground-image.tar.gz | cut -f1)" + # Note: Tests now use testcontainers - no mysql-client or Docker Buildx caching needed # Testcontainers handles container lifecycle and image pulling automatically + # TiUP Playground image is pre-built above and saved as artifact for test jobs - name: Upload Terraform binary uses: actions/upload-artifact@v4 @@ -92,6 +112,14 @@ jobs: path: vendor/ retention-days: 1 compression-level: 6 + + - name: Upload TiUP Playground Docker image + uses: actions/upload-artifact@v4 + with: + name: tiup-playground-image + path: tiup-playground-image.tar.gz + retention-days: 1 + compression-level: 6 tests: runs-on: ubuntu-22.04 @@ -103,43 +131,49 @@ jobs: # MySQL versions - db_type: mysql db_version: "5.6" - docker_image: "mysql:5.6" + make_target: "test-mysql-5.6" - db_type: mysql db_version: "5.7" - docker_image: "mysql:5.7" + make_target: "test-mysql-5.7" - db_type: mysql db_version: "8.0" - docker_image: "mysql:8.0" + make_target: "test-mysql-8.0" # Percona versions - db_type: percona db_version: "5.7" - docker_image: "percona:5.7" + make_target: "test-percona-5.7" - db_type: percona db_version: "8.0" - docker_image: "percona:8.0" + make_target: "test-percona-8.0" # MariaDB versions - db_type: mariadb db_version: "10.3" - docker_image: "mariadb:10.3" + make_target: "test-mariadb-10.3" - db_type: mariadb db_version: "10.8" - docker_image: "mariadb:10.8" + make_target: "test-mariadb-10.8" - db_type: mariadb db_version: "10.10" - docker_image: "mariadb:10.10" + make_target: "test-mariadb-10.10" # TiDB versions - must match env.TIDB_VERSIONS: 6.1.7 6.5.12 7.1.6 7.5.7 8.1.2 8.5.3 - db_type: tidb db_version: "6.1.7" + make_target: "test-tidb-6.1.7" - db_type: tidb db_version: "6.5.12" + make_target: "test-tidb-6.5.12" - db_type: tidb db_version: "7.1.6" + make_target: "test-tidb-7.1.6" - db_type: tidb db_version: "7.5.7" + make_target: "test-tidb-7.5.7" - db_type: tidb db_version: "8.1.2" + make_target: "test-tidb-8.1.2" - db_type: tidb db_version: "8.5.3" + make_target: "test-tidb-8.5.3" steps: - name: Checkout Git repo uses: actions/checkout@v4 @@ -169,56 +203,131 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Pre-pull Docker images for caching - if: matrix.db_type != 'tidb' - run: | - docker pull ${{ matrix.docker_image }} || true + - name: Download TiUP Playground Docker image + uses: actions/download-artifact@v4 + with: + name: tiup-playground-image + path: ./ - - name: Pre-pull TiDB images for caching - if: matrix.db_type == 'tidb' + - name: Load TiUP Playground Docker image run: | - docker pull pingcap/tidb:v${{ matrix.db_version }} || true - docker pull pingcap/pd:v${{ matrix.db_version }} || true - docker pull pingcap/tikv:v${{ matrix.db_version }} || true + echo "Loading pre-built TiUP Playground Docker image..." + gunzip -c tiup-playground-image.tar.gz | docker load + docker images | grep terraform-provider-mysql-tiup-playground + echo "✓ TiUP Playground image loaded successfully" - - name: Run testcontainers tests + # Note: TiUP Playground image is pre-built in prepare-dependencies and loaded here + # This avoids rebuilding the image during each test run + # Testcontainers handles container lifecycle and image pulling automatically + + - name: Run testcontainers tests via Makefile env: GOFLAGS: -mod=vendor TF_ACC: 1 GOTOOLCHAIN: auto run: | export PATH="${{ github.workspace }}/bin:$PATH" - if [ "${{ matrix.db_type }}" == "tidb" ]; then - TIDB_VERSION=${{ matrix.db_version }} go test -tags=testcontainers -v ./mysql/... -run WithTestcontainers -timeout=30m - else - DOCKER_IMAGE=${{ matrix.docker_image }} go test -tags=testcontainers -v ./mysql/... -run WithTestcontainers -timeout=30m - fi - # DISABLED to figure out GPG signing issue on Github Actions - # possibly due to lack of TTY inside docker? - # release: - # name: Release - # needs: [tests] - # # Can't use non-semvar for the testing tag - # # https://github.com/orgs/goreleaser/discussions/3708 - # if: ( startsWith( github.ref, 'refs/tags/v' ) || - # startsWith(github.ref, 'refs/tags/v0.0.0-rc') ) - # runs-on: ubuntu-22.04 - # steps: - # - name: Checkout Git repo - # uses: actions/checkout@v4 - - # # Goreleaser - # - name: Set up Go - # uses: actions/setup-go@v4 - # - name: Run GoReleaser - # uses: goreleaser/goreleaser-action@v6 - # with: - # distribution: goreleaser - # version: '~> v2' - # # Run goreleaser and ignore non-committed files (downloaded artifacts) - # args: release --clean --skip=validate --verbose - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + echo "Running ${{ matrix.db_type }} ${{ matrix.db_version }} tests using Makefile target: ${{ matrix.make_target }}" + make ${{ matrix.make_target }} + release: + name: Release + needs: [tests] + # Can't use non-semvar for the testing tag + # https://github.com/orgs/goreleaser/discussions/3708 + if: ( startsWith( github.ref, 'refs/tags/v' ) || + startsWith(github.ref, 'refs/tags/v0.0.0-rc') ) + runs-on: ubuntu-22.04 + permissions: + contents: write # Required for creating releases + steps: + - name: Checkout Git repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history needed for changelog + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: go.mod + + - name: Import GPG Subkey + env: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + GPG_FINGERPRINT: ${{ secrets.GPG_FINGERPRINT }} + run: | + # Install gnupg2 if not already available + sudo apt-get update && sudo apt-get install -y gnupg2 || true + + # Create GPG directory + mkdir -p ~/.gnupg + chmod 700 ~/.gnupg + + # Remove any existing gpg.conf to avoid conflicts + rm -f ~/.gnupg/gpg.conf + + # Configure GPG for non-interactive use + cat > ~/.gnupg/gpg.conf < ~/.gnupg/gpg-agent.conf </dev/null || true + gpgconf --kill dirmngr 2>/dev/null || true + sleep 1 + gpg-agent --daemon --allow-loopback-pinentry > /dev/null 2>&1 || true + sleep 2 # Give gpg-agent time to start + + # Import the subkey (no passphrase required) + KEY_FILE=$(mktemp) + echo "$GPG_PRIVATE_KEY" > "$KEY_FILE" + gpg --batch --yes --import "$KEY_FILE" + rm -f "$KEY_FILE" + + # Trust the key (required for signing) + # Format: fingerprint:trust-level: (fingerprint must be uppercase, no spaces, no colons) + # Use ultimate trust (6) for the subkey + FINGERPRINT_UPPER=$(echo "$GPG_FINGERPRINT" | tr '[:lower:]' '[:upper:]' | tr -d ' ' | tr -d ':') + echo "$FINGERPRINT_UPPER:6:" | gpg --batch --import-ownertrust + + # Verify key is available + gpg --list-secret-keys --keyid-format LONG + + # Verify signing works (subkey has no passphrase) + echo "test" | gpg --batch --no-tty --pinentry-mode loopback --sign --local-user "$FINGERPRINT_UPPER" -o /dev/null 2>&1 && echo "✓ Test signing successful" || echo "⚠ Test signing failed" + + echo "✓ GPG key imported successfully" + + - name: Verify GPG setup before GoReleaser + env: + GPG_FINGERPRINT: ${{ secrets.GPG_FINGERPRINT }} + run: | + echo "Verifying GPG setup..." + echo "GPG_FINGERPRINT length: ${#GPG_FINGERPRINT}" + gpg --list-secret-keys --keyid-format LONG + # Test signing (subkey has no passphrase) + echo "test" | gpg --batch --yes --no-tty --pinentry-mode loopback --local-user "$GPG_FINGERPRINT" --sign -o /tmp/test.sig - 2>&1 && echo "✓ Test signing successful" || echo "⚠ Test signing failed" + rm -f /tmp/test.sig + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: '~> v2' + # Run goreleaser and ignore non-committed files (downloaded artifacts) + args: release --clean --skip=validate + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GPG_FINGERPRINT: ${{ secrets.GPG_FINGERPRINT }} + GPG_TTY: $(tty) # terraform-provider-release: # needs: [release] diff --git a/.gitignore b/.gitignore index dea93704..dceaec88 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ website/vendor # Test exclusions !command/test-fixtures/**/*.tfstate !command/test-fixtures/**/.terraform/ +test-runner diff --git a/.goreleaser.yml b/.goreleaser.yml index 63ad53a4..f99d11b9 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -39,17 +39,35 @@ checksum: name_template: "{{ .ProjectName }}_{{ .Version }}_SHA256SUMS" algorithm: sha256 signs: - - artifacts: checksum + - id: checksum + artifacts: checksum args: - # if you are using this is a GitHub action or some other automated pipeline, you - # need to pass the batch flag to indicate its not interactive. + # Subkey has no passphrase - no --passphrase flag needed - "--batch" + - "--yes" + - "--no-tty" + - "--pinentry-mode" + - "loopback" - "--local-user" - "{{ .Env.GPG_FINGERPRINT }}" # set this environment variable for your signing key - "--output" - "${signature}" - "--detach-sign" - "${artifact}" + - id: archive + artifacts: archive + args: + - "--batch" + - "--yes" + - "--no-tty" + - "--pinentry-mode" + - "loopback" + - "--local-user" + - "{{ .Env.GPG_FINGERPRINT }}" + - "--output" + - "${signature}" + - "--detach-sign" + - "${artifact}" release: # If you want to manually examine the release before its live, uncomment this line: draft: true diff --git a/Dockerfile.tiup-playground b/Dockerfile.tiup-playground new file mode 100644 index 00000000..f481f42e --- /dev/null +++ b/Dockerfile.tiup-playground @@ -0,0 +1,26 @@ +# Dockerfile for TiUP Playground container +# This image contains TiUP and can run TiDB Playground inside a container + +FROM ubuntu:22.04 + +# Install dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + mysql-client \ + && rm -rf /var/lib/apt/lists/* + +# Install TiUP +RUN curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh + +# Add TiUP to PATH +ENV PATH="/root/.tiup/bin:${PATH}" + +# Update TiUP and playground component +RUN /root/.tiup/bin/tiup update --self && \ + /root/.tiup/bin/tiup update playground || true + +# Default command runs TiUP Playground +# This will be overridden by testcontainers with specific version and port +CMD ["/root/.tiup/bin/tiup", "playground"] diff --git a/Makefile b/Makefile index a36848a5..253d5d09 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,37 @@ default: help build: fmtcheck ## Build the provider go install +clean: ## Aggressively clear Docker cache and test artifacts + @echo "Clearing Docker cache and test artifacts..." + @# Remove testcontainers-related images (mysql, percona, mariadb, tidb) + @docker images --format "{{.Repository}}:{{.Tag}}" | grep -E "(mysql|percona|mariadb|tidb|pingcap)" | xargs -r docker rmi -f 2>/dev/null || true + @# Remove Docker manifests for problematic images (force remove even if they don't exist) + @for img in mysql:5.6 mysql:5.7 percona:5.7 percona:8.0; do \ + docker manifest rm $$img 2>/dev/null || true; \ + done + @# Prune build cache (all, not just 24h) + @docker builder prune -af 2>/dev/null || true + @# Prune unused images (all, not just 24h) + @docker image prune -af 2>/dev/null || true + @# Prune unused containers + @docker container prune -f 2>/dev/null || true + @# Prune unused networks (but keep default networks) + @docker network prune -f 2>/dev/null || true + @# Clear testcontainers temp files + @rm -rf /tmp/testcontainers-* 2>/dev/null || true + @# Clear Docker's content-addressable storage for problematic images (if possible) + @echo "Docker cache cleared. Note: For MySQL 5.6/5.7 and Percona on Apple Silicon," + @echo "you may need to restart Docker Desktop to fully clear manifest cache." + +build-tiup-playground-image: ## Pre-build TiUP Playground Docker image for caching + @echo "Building TiUP Playground Docker image..." + @if [ ! -f Dockerfile.tiup-playground ]; then \ + echo "ERROR: Dockerfile.tiup-playground not found"; \ + exit 1; \ + fi + @docker build -f Dockerfile.tiup-playground -t terraform-provider-mysql-tiup-playground:latest . + @echo "✓ TiUP Playground image built successfully: terraform-provider-mysql-tiup-playground:latest" + test: testcontainers-matrix ## Run all acceptance tests test-sequential: acceptance @@ -62,8 +93,8 @@ testcontainers-matrix: fmtcheck bin/terraform ## Run test matrix across all data # Run testcontainers tests for a specific database image # Usage: make testcontainers-image DOCKER_IMAGE=mysql:8.0 -# make testcontainers-image TIDB_VERSION=8.5.3 -testcontainers-image: fmtcheck bin/terraform ## Run tests for a specific database image (set DOCKER_IMAGE or TIDB_VERSION) +# make testcontainers-image DOCKER_IMAGE=tidb:8.5.3 +testcontainers-image: fmtcheck bin/terraform ## Run tests for a specific database image (set DOCKER_IMAGE) @PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers $(TEST) -v $(TESTARGS) -timeout=15m bin/terraform: ## Download Terraform binary @@ -79,18 +110,44 @@ testacc: fmtcheck bin/terraform ## Run acceptance tests (requires MYSQL_ENDPOINT acceptance: testversion5.6 testversion5.7 testversion8.0 testpercona5.7 testpercona8.0 testmariadb10.3 testmariadb10.8 testmariadb10.10 testtidb6.1.7 testtidb6.5.12 testtidb7.1.6 testtidb7.5.7 testtidb8.1.2 testtidb8.5.3 ## Run all acceptance tests across all database versions # MySQL test targets - use testcontainers -testversion%: ## Run tests against MySQL version (e.g., testversion8.0) - @DOCKER_IMAGE=mysql:$* PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS).*WithTestcontainers",-run WithTestcontainers) -timeout=30m +# Preferred format: test-mysql-VERSION (e.g., test-mysql-5.6) +test-mysql-%: ## Run tests against MySQL version (e.g., test-mysql-8.0) + @$(MAKE) testversion$* + +testversion%: ## Run tests against MySQL version (e.g., testversion8.0) [backwards compatible] + @# MySQL 5.6 and 5.7 don't have ARM64 builds - Docker Desktop on Apple Silicon has manifest cache issues + @# The workaround: restart Docker Desktop or use CI (GitHub Actions uses linux/amd64) + @if [ "$*" = "5.6" ] || [ "$*" = "5.7" ]; then \ + echo "WARNING: MySQL $* doesn't have ARM64 support. Docker Desktop manifest cache may cause issues."; \ + echo "If tests fail with 'no match for platform in manifest', try: make clean && restart Docker Desktop"; \ + docker rmi mysql:$* 2>/dev/null || true; \ + docker manifest rm mysql:$* 2>/dev/null || true; \ + docker pull --platform linux/amd64 mysql:$* 2>&1 | grep -v "no match" || true; \ + DOCKER_DEFAULT_PLATFORM=linux/amd64 DOCKER_IMAGE=mysql:$* PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS)",) -timeout=30m; \ + else \ + DOCKER_IMAGE=mysql:$* PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS)",) -timeout=30m; \ + fi testversion: ## Run tests against MySQL version (set MYSQL_VERSION) - @DOCKER_IMAGE=mysql:$(MYSQL_VERSION) PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS).*WithTestcontainers",-run WithTestcontainers) -timeout=30m + @DOCKER_IMAGE=mysql:$(MYSQL_VERSION) PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS)",) -timeout=30m # Percona test targets - use testcontainers -testpercona%: ## Run tests against Percona version (e.g., testpercona8.0) - @DOCKER_IMAGE=percona:$* PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS).*WithTestcontainers",-run WithTestcontainers) -timeout=30m +# Preferred format: test-percona-VERSION (e.g., test-percona-8.0) +test-percona-%: ## Run tests against Percona version (e.g., test-percona-8.0) + @$(MAKE) testpercona$* + +testpercona%: ## Run tests against Percona version (e.g., testpercona8.0) [backwards compatible] + @# Percona 5.7 and 8.0 don't have ARM64 builds, so pre-pull with platform specification for Apple Silicon + @if [ "$*" = "5.7" ] || [ "$*" = "8.0" ]; then \ + echo "Pre-pulling percona:$* with platform linux/amd64 for Apple Silicon compatibility..."; \ + docker rmi percona:$* 2>/dev/null || true; \ + docker manifest rm percona:$* 2>/dev/null || true; \ + docker pull --platform linux/amd64 percona:$* || true; \ + fi + @DOCKER_IMAGE=percona:$* PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS)",) -timeout=30m testpercona: ## Run tests against Percona version (set MYSQL_VERSION) - @DOCKER_IMAGE=percona:$(MYSQL_VERSION) PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS).*WithTestcontainers",-run WithTestcontainers) -timeout=30m + @DOCKER_IMAGE=percona:$(MYSQL_VERSION) PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS)",) -timeout=30m testrdsdb%: ## Run tests against RDS MySQL version (requires MYSQL_ENDPOINT env vars) $(MAKE) MYSQL_VERSION=$* MYSQL_USERNAME=${MYSQL_USERNAME} MYSQL_HOST=$(shell echo ${MYSQL_ENDPOINT} | cut -d: -f1) MYSQL_PASSWORD=${MYSQL_PASSWORD} MYSQL_PORT=$(shell echo ${MYSQL_ENDPOINT} | cut -d: -f2) testrdsdb @@ -101,18 +158,26 @@ testrdsdb: ## Run tests against Amazon RDS (requires MYSQL_ENDPOINT env vars) $(MAKE) testacc # TiDB test targets - use testcontainers -testtidb%: ## Run tests against TiDB version (e.g., testtidb8.5.3) - @TIDB_VERSION=$* PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS).*WithTestcontainers",-run WithTestcontainers) -timeout=30m +# Preferred format: test-tidb-VERSION (e.g., test-tidb-8.5.3) +test-tidb-%: ## Run tests against TiDB version (e.g., test-tidb-8.5.3) + @$(MAKE) testtidb$* + +testtidb%: ## Run tests against TiDB version (e.g., testtidb8.5.3) [backwards compatible] + @DOCKER_IMAGE=tidb:$* PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS)",) -timeout=30m testtidb: ## Run tests against TiDB version (set MYSQL_VERSION) - @TIDB_VERSION=$(MYSQL_VERSION) PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS).*WithTestcontainers",-run WithTestcontainers) -timeout=30m + @DOCKER_IMAGE=tidb:$(MYSQL_VERSION) PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS)",) -timeout=30m # MariaDB test targets - use testcontainers -testmariadb%: ## Run tests against MariaDB version (e.g., testmariadb10.10) - @DOCKER_IMAGE=mariadb:$* PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS).*WithTestcontainers",-run WithTestcontainers) -timeout=30m +# Preferred format: test-mariadb-VERSION (e.g., test-mariadb-10.10) +test-mariadb-%: ## Run tests against MariaDB version (e.g., test-mariadb-10.10) + @$(MAKE) testmariadb$* + +testmariadb%: ## Run tests against MariaDB version (e.g., testmariadb10.10) [backwards compatible] + @DOCKER_IMAGE=mariadb:$* PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS)",) -timeout=30m testmariadb: ## Run tests against MariaDB version (set MYSQL_VERSION) - @DOCKER_IMAGE=mariadb:$(MYSQL_VERSION) PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS).*WithTestcontainers",-run WithTestcontainers) -timeout=30m + @DOCKER_IMAGE=mariadb:$(MYSQL_VERSION) PATH="$(CURDIR)/bin:${PATH}" TF_ACC=1 GOTOOLCHAIN=auto go test -tags=testcontainers ./mysql/... -v $(if $(TESTARGS),-run "$(TESTARGS)",) -timeout=30m vet: ## Run go vet @echo "go vet ." diff --git a/TERRAFORM_BINARY_OPTIMIZATION_PLAN.md b/TERRAFORM_BINARY_OPTIMIZATION_PLAN.md deleted file mode 100644 index 2504f3e0..00000000 --- a/TERRAFORM_BINARY_OPTIMIZATION_PLAN.md +++ /dev/null @@ -1,394 +0,0 @@ -# Plan: Optimize Terraform Binary Build in GitHub Actions - -## Problem Statement - -Currently, each integration test job in the GitHub Actions matrix independently: -1. Downloads and extracts the Terraform binary (version 1.5.6) -2. Downloads all Go module dependencies (~90+ packages, ~100-200MB) - -This results in: -- **Terraform**: Redundant downloads (13 test jobs × ~20MB = ~260MB downloaded repeatedly) -- **Go Modules**: Redundant downloads (13 test jobs × ~100-200MB = ~1.3-2.6GB downloaded repeatedly!) -- Increased test execution time (each job waits for downloads/extraction) -- Unnecessary network usage and costs - -## Current Architecture - -### Current Flow: -``` -Each test job (testversion5.6, testversion8.0, etc.): - 1. Checkout repo - 2. Install mysql-client - 3. Run `make testversion5.6` (or other target) - → `make testacc` (dependency) - → `make bin/terraform` (downloads & extracts Terraform) - → Run actual tests -``` - -### Makefile Dependencies: -- `testacc` depends on `bin/terraform` -- `bin/terraform` target downloads Terraform 1.5.6 for Linux amd64 -- Binary is placed in `$(CURDIR)/bin/terraform` -- Tests use `PATH="$(CURDIR)/bin:${PATH}"` to find terraform - -## Solution: Pre-build Job with Artifact Sharing - -### Proposed Architecture: -``` -1. prepare-terraform job: - - Downloads Terraform binary once - - Uploads as GitHub Actions artifact - -2. Each test job: - - Downloads Terraform artifact - - Extracts to bin/ - - Runs tests (no download needed) -``` - -## Implementation Plan - -### Step 1: Create `prepare-terraform` Job - -**New job that runs before tests:** - -```yaml -prepare-terraform: - name: Prepare Terraform Binary - runs-on: ubuntu-22.04 - steps: - - name: Checkout Git repo - uses: actions/checkout@v4 - - - name: Set up Go (for go mod download if needed) - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - - - name: Download Terraform - run: | - mkdir -p bin - curl -sfL https://releases.hashicorp.com/terraform/1.5.6/terraform_1.5.6_linux_amd64.zip > bin/terraform.zip - cd bin && unzip terraform.zip && rm terraform.zip - chmod +x bin/terraform - - - name: Upload Terraform binary - uses: actions/upload-artifact@v4 - with: - name: terraform-binary - path: bin/terraform - retention-days: 1 -``` - -### Step 2: Update `tests` Job to Download Artifact and Cache Go Modules - -**Modify existing tests job:** - -```yaml -tests: - runs-on: ubuntu-22.04 - needs: [prepare-terraform] # Add dependency - strategy: - fail-fast: false - matrix: - target: - - testversion5.6 - - testversion5.7 - # ... rest of targets - steps: - - name: Checkout Git repo - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - - - name: Cache Go modules - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Download Terraform binary - uses: actions/download-artifact@v4 - with: - name: terraform-binary - path: bin/ - - - name: Make Terraform executable - run: chmod +x bin/terraform - - - name: Install mysql client - run: | - sudo apt-get update - sudo apt-get -f -y install mysql-client - - - name: Run tests {{ matrix.target }} - run: make ${{ matrix.target }} -``` - -### Step 3: Handle Makefile Compatibility - -**The Makefile already supports this!** - -The `bin/terraform` target in the Makefile: -```makefile -bin/terraform: - mkdir -p "$(CURDIR)/bin" - curl -sfL https://releases.hashicorp.com/terraform/$(TERRAFORM_VERSION)/terraform_$(TERRAFORM_VERSION)_$(TERRAFORM_OS)_$(ARCH).zip > $(CURDIR)/bin/terraform.zip - (cd $(CURDIR)/bin/ ; unzip terraform.zip) -``` - -**Solution Options:** - -**Option A: Create bin/terraform before running make (Recommended)** -- Download artifact to `bin/terraform` before running make -- Make will see the file exists and skip the target -- No Makefile changes needed - -**Option B: Modify Makefile to check for existing binary** -- Add check: `if [ -f "$(CURDIR)/bin/terraform" ]; then exit 0; fi` -- More robust but requires Makefile change - -**Option C: Use Makefile variable to skip download** -- Add `SKIP_TERRAFORM_DOWNLOAD=true` environment variable -- Modify Makefile to check this variable -- More explicit but requires Makefile change - -**Recommended: Option A** - Simplest, no Makefile changes needed. - -### Step 4: Update Makefile (Optional Enhancement) - -**If we want to make it more explicit, add a check:** - -```makefile -bin/terraform: - @if [ -f "$(CURDIR)/bin/terraform" ]; then \ - echo "Terraform binary already exists, skipping download"; \ - exit 0; \ - fi - mkdir -p "$(CURDIR)/bin" - curl -sfL https://releases.hashicorp.com/terraform/$(TERRAFORM_VERSION)/terraform_$(TERRAFORM_VERSION)_$(TERRAFORM_OS)_$(ARCH).zip > $(CURDIR)/bin/terraform.zip - (cd $(CURDIR)/bin/ ; unzip terraform.zip) -``` - -This makes the Makefile idempotent and works for both CI and local development. - -## Benefits - -### Terraform Binary Optimization: -1. **Time Savings**: - - Current: ~5-10 seconds per job × 13 jobs = 65-130 seconds - - Optimized: ~5-10 seconds once + ~1 second download per job = ~18 seconds total - - **Savings: ~50-110 seconds per workflow run** - -2. **Network Savings**: - - Current: ~260MB downloaded (13 × ~20MB) - - Optimized: ~20MB downloaded once + ~20MB uploaded once - - **Savings: ~220MB per workflow run** - -### Go Module Caching Optimization: -3. **Time Savings**: - - Current: ~10-30 seconds per job × 13 jobs = 130-390 seconds - - Optimized: First run downloads (~10-30s), subsequent runs use cache (~1-2s) - - **Savings: ~120-360 seconds per workflow run** (after first run) - -4. **Network Savings**: - - Current: ~1.3-2.6GB downloaded (13 × ~100-200MB) - - Optimized: ~100-200MB downloaded once, cached for all jobs - - **Savings: ~1.1-2.4GB per workflow run** - -### Combined Total Savings: -- **Time**: ~170-470 seconds (~3-8 minutes) per workflow run -- **Network**: ~1.3-2.6GB per workflow run - -3. **Reliability**: - - Single point of download reduces chance of network failures - - Artifact caching can further improve reliability - -4. **Cost**: - - Reduced network egress costs - - Faster test execution = lower compute costs - -## Implementation Details - -### Artifact Considerations - -**Artifact Size**: Terraform binary is ~20MB (compressed), ~50MB uncompressed - -**Retention**: Set to 1 day (artifacts are only needed during workflow execution) - -**Artifact Name**: `terraform-binary` (unique per workflow run) - -### Parallel Execution - -The `prepare-terraform` job will run first, then all test jobs run in parallel. This is optimal because: -- Tests can't run without Terraform anyway -- Parallel test execution is maintained -- Minimal impact on total workflow time - -### Error Handling - -If `prepare-terraform` fails: -- All dependent test jobs will be skipped (GitHub Actions behavior) -- Clear error message in workflow UI -- Easy to identify the root cause - -### Fallback Behavior - -If artifact download fails in a test job: -- The Makefile will still attempt to download (if Option A) -- Or we can add explicit error handling in the workflow - -## Testing Strategy - -1. **Test locally first**: - - Manually download Terraform to `bin/terraform` - - Run `make testversion8.0` to verify it works - -2. **Test in GitHub Actions**: - - Create a test branch - - Run workflow and verify: - - `prepare-terraform` completes successfully - - Artifact is uploaded - - Test jobs download artifact - - Tests run successfully - - No redundant downloads occur - -3. **Verify performance**: - - Compare workflow execution times before/after - - Check artifact sizes - - Monitor for any failures - -## Alternative Approaches Considered - -### Option 1: Use GitHub Actions Cache for Terraform -```yaml -- uses: actions/cache@v4 - with: - path: bin/terraform - key: terraform-1.5.6-linux-amd64 -``` -**Pros**: Automatic caching, no separate job needed -**Cons**: Cache miss still requires download, less explicit control -**Note**: This is actually what we're doing for Go modules (better fit) - -### Option 2: Pre-download Go Modules in prepare Job -**Pros**: Could pre-populate cache -**Cons**: Not necessary - Go's built-in caching + actions/cache is sufficient - -### Option 3: Docker Image with Pre-installed Terraform -**Pros**: Faster startup, includes all dependencies -**Cons**: Requires maintaining custom Docker image, more complexity - -### Option 4: Use actions/setup-terraform -**Pros**: Official action, well-maintained -**Cons**: May not support Terraform 1.5.6 (older version), less control - -## Recommended Implementation - -**Use the artifact approach** because: -1. Explicit and predictable -2. Works with existing Makefile without changes -3. Easy to debug and maintain -4. Good performance characteristics -5. No external dependencies - -## Implementation Checklist - -- [ ] Create `prepare-terraform` job in workflow -- [ ] Add `needs: [prepare-terraform]` to tests job -- [ ] Add `actions/setup-go@v4` step to tests job -- [ ] Add `actions/cache@v4` step for Go modules to tests job -- [ ] Add artifact download step to tests job -- [ ] (Optional) Add idempotency check to Makefile `bin/terraform` target -- [ ] Test locally with pre-downloaded binary -- [ ] Test in GitHub Actions on a test branch -- [ ] Monitor workflow execution times (should see significant improvement) -- [ ] Monitor cache hit rates in GitHub Actions -- [ ] Document the optimization in README (optional) - -## Code Changes Summary - -### `.github/workflows/main.yml` - -**Add new job:** -```yaml -prepare-terraform: - name: Prepare Terraform Binary - runs-on: ubuntu-22.04 - steps: - - name: Checkout Git repo - uses: actions/checkout@v4 - - name: Download Terraform - run: | - mkdir -p bin - curl -sfL https://releases.hashicorp.com/terraform/1.5.6/terraform_1.5.6_linux_amd64.zip > bin/terraform.zip - cd bin && unzip terraform.zip && rm terraform.zip - chmod +x bin/terraform - - name: Upload Terraform binary - uses: actions/upload-artifact@v4 - with: - name: terraform-binary - path: bin/terraform - retention-days: 1 -``` - -**Modify tests job:** -```yaml -tests: - needs: [prepare-terraform] # Add this line - # ... rest of job config - steps: - - name: Checkout Git repo - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - - - name: Cache Go modules - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Download Terraform binary - uses: actions/download-artifact@v4 - with: - name: terraform-binary - path: bin/ - - - name: Make Terraform executable - run: chmod +x bin/terraform - # ... rest of steps -``` - -### `GNUmakefile` (Optional Enhancement) - -```makefile -bin/terraform: - @if [ -f "$(CURDIR)/bin/terraform" ]; then \ - echo "Terraform binary already exists, skipping download"; \ - exit 0; \ - fi - mkdir -p "$(CURDIR)/bin" - curl -sfL https://releases.hashicorp.com/terraform/$(TERRAFORM_VERSION)/terraform_$(TERRAFORM_VERSION)_$(TERRAFORM_OS)_$(ARCH).zip > $(CURDIR)/bin/terraform.zip - (cd $(CURDIR)/bin/ ; unzip terraform.zip) -``` - -## References - -- [GitHub Actions Artifacts Documentation](https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts) -- [GitHub Actions Job Dependencies](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idneeds) -- [actions/upload-artifact@v4](https://github.com/actions/upload-artifact) -- [actions/download-artifact@v4](https://github.com/actions/download-artifact) diff --git a/TESTCONTAINERS_POC.md b/TESTCONTAINERS_POC.md deleted file mode 100644 index af2dbd51..00000000 --- a/TESTCONTAINERS_POC.md +++ /dev/null @@ -1,128 +0,0 @@ -# Testcontainers Proof of Concept - -This directory contains a proof of concept implementation using Testcontainers instead of the Makefile + Docker approach. - -## What's Implemented - -1. **`mysql/testcontainers_helper.go`**: Helper functions for starting MySQL containers -2. **`mysql/data_source_databases_testcontainers_test.go`**: Example test using Testcontainers - -## How to Use - -### Prerequisites - -- Docker or Podman installed -- Go 1.21+ (project requirement) - -### Running Tests - -```bash -# Run the Testcontainers-based test -go test -tags=testcontainers -v ./mysql/... -run TestAccDataSourceDatabases_WithTestcontainers - -# Run all tests (both old and new) -go test -tags=testcontainers -v ./mysql/... -``` - -### With Podman - -Testcontainers automatically detects Podman. To use Podman explicitly: - -```bash -# Set Podman socket (if needed) -export CONTAINER_HOST=unix://$HOME/.local/share/containers/podman/machine/podman-machine-default/podman.sock - -# Run tests -go test -tags=testcontainers -v ./mysql/... -run TestAccDataSourceDatabases_WithTestcontainers -``` - -## How It Works - -1. **Build Tag**: Code is gated behind `// +build testcontainers` tag - - Old tests continue to work without Testcontainers - - New tests only run when `-tags=testcontainers` is used - -2. **Container Lifecycle**: - - Container starts automatically when test begins - - Environment variables are set for the test - - Container terminates automatically when test completes - -3. **Compatibility**: - - Works with Docker (default) - - Works with Podman (automatic detection) - - Uses GenericContainer for Go 1.21 compatibility - -## Example Test - -```go -func TestAccDataSourceDatabases_WithTestcontainers(t *testing.T) { - ctx := context.Background() - - // Start MySQL container - container := startMySQLContainer(ctx, t, "mysql:8.0") - defer container.SetupTestEnv(t)() - - // Run tests (same as before) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - Steps: []resource.TestStep{ - // ... test steps - }, - }) -} -``` - -## Benefits Over Makefile Approach - -1. **No Shell Scripts**: Pure Go code -2. **Automatic Cleanup**: Containers terminate automatically -3. **Better Error Messages**: Container logs captured on failure -4. **Parallel Execution**: Go's testing framework handles parallelism -5. **Podman Support**: Works without configuration changes - -## Next Steps - -1. **Test Locally**: Verify it works with your Docker/Podman setup -2. **Measure Performance**: Compare startup time vs. Makefile approach -3. **Convert More Tests**: Gradually migrate other test files -4. **CI/CD Integration**: Update GitHub Actions to use Testcontainers - -## Troubleshooting - -### Compiler Warnings - -You may see warnings like: -``` -warning: variable length array folded to constant array as an extension [-Wgnu-folding-constant] -``` - -These are harmless warnings from the `go-m1cpu` dependency (used by testcontainers-go). They don't affect functionality. To suppress them: - -```bash -export CGO_CFLAGS="-Wno-gnu-folding-constant" -go test -tags=testcontainers -v ./mysql/... -``` - -Or use the provided script: -```bash -source .testcontainers-build-flags.sh -go test -tags=testcontainers -v ./mysql/... -``` - -### Container Won't Start - -- Check Docker/Podman is running: `docker ps` or `podman ps` -- Check image exists: `docker images mysql:8.0` -- Increase timeout in `testcontainers_helper.go` if needed - -### Port Conflicts - -- Testcontainers automatically assigns random ports -- No manual port calculation needed (unlike Makefile approach) - -### Podman Issues - -- Ensure Podman socket is accessible -- Check `CONTAINER_HOST` environment variable if needed -- Testcontainers should auto-detect Podman diff --git a/TESTCONTAINERS_SPIKE.md b/TESTCONTAINERS_SPIKE.md deleted file mode 100644 index af6d7039..00000000 --- a/TESTCONTAINERS_SPIKE.md +++ /dev/null @@ -1,583 +0,0 @@ -# Spike: Replacing Makefile + Docker with Testcontainers - -## Current State - -### Problems with Current Approach - -1. **Complex Makefile**: 200+ lines of shell scripting with port calculations, container management, and cleanup logic -2. **Fragile Port Management**: Complex port calculation logic (`34$(tr -d '.')`) that breaks with longer version numbers -3. **Manual Container Lifecycle**: Containers must be manually started, waited for, and cleaned up -4. **TiDB Complexity**: Requires custom bash script (`tidb-test-cluster.sh`) with 200+ lines for multi-container orchestration -5. **No Parallel Execution**: Containers started sequentially, tests run sequentially -6. **Environment Variable Pollution**: Tests rely on global environment variables -7. **Hard to Debug**: Container failures require manual log inspection -8. **CI/CD Complexity**: GitHub Actions workflow has to coordinate Makefile targets with Docker - -### Current Test Flow - -``` -Makefile target → Docker run → Wait loop → Set env vars → Run tests → Cleanup -``` - -## Testcontainers Solution - -[Testcontainers](https://golang.testcontainers.org/) is a Go library that provides lightweight, throwaway instances of Docker containers for integration testing. - -### Benefits - -1. **Pure Go**: No shell scripts, no Makefile complexity -2. **Automatic Lifecycle**: Containers start/stop automatically with test lifecycle -3. **Built-in Waiting**: Automatic readiness checks (no manual wait loops) -4. **Parallel Execution**: Go's testing framework handles parallel test execution -5. **Isolated Tests**: Each test can have its own container instance -6. **Better Error Messages**: Container logs automatically captured on failure -7. **Type Safety**: Compile-time checks instead of runtime shell errors -8. **Simpler CI/CD**: Just run `go test` - no Makefile coordination needed -9. **Podman Support**: Works with Podman without special configuration (daemonless, rootless) - -## Proof of Concept - -### Example: Simple MySQL Test - -```go -package mysql - -import ( - "context" - "os" - "testing" - - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" - tcMySQL "github.com/testcontainers/testcontainers-go/modules/mysql" -) - -func TestAccDatabase_WithTestcontainers(t *testing.T) { - ctx := context.Background() - - // Start MySQL container - mysqlContainer, err := tcMySQL.RunContainer(ctx, - testcontainers.WithImage("mysql:8.0"), - tcMySQL.WithDatabase("testdb"), - tcMySQL.WithUsername("root"), - tcMySQL.WithPassword(""), - testcontainers.WithWaitStrategy( - wait.ForLog("ready for connections"). - WithOccurrence(2). - WithStartupTimeout(120*time.Second), - ), - ) - if err != nil { - t.Fatalf("Failed to start container: %v", err) - } - defer func() { - if err := mysqlContainer.Terminate(ctx); err != nil { - t.Fatalf("Failed to terminate container: %v", err) - } - }() - - // Get connection details - endpoint, err := mysqlContainer.ConnectionString(ctx) - if err != nil { - t.Fatalf("Failed to get connection string: %v", err) - } - - // Set environment variables for test - os.Setenv("MYSQL_ENDPOINT", endpoint) - os.Setenv("MYSQL_USERNAME", "root") - os.Setenv("MYSQL_PASSWORD", "") - defer func() { - os.Unsetenv("MYSQL_ENDPOINT") - os.Unsetenv("MYSQL_USERNAME") - os.Unsetenv("MYSQL_PASSWORD") - }() - - // Run existing test logic - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - Steps: []resource.TestStep{ - { - Config: testAccDatabaseConfig("testdb"), - Check: resource.ComposeTestCheckFunc( - testAccDatabaseExists("mysql_database.test"), - ), - }, - }, - }) -} -``` - -### Example: Test Helper Function - -```go -// testcontainers_helper.go - -package mysql - -import ( - "context" - "os" - "testing" - "time" - - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" - tcMySQL "github.com/testcontainers/testcontainers-go/modules/mysql" -) - -type MySQLTestContainer struct { - Container testcontainers.Container - Endpoint string - Username string - Password string -} - -func startMySQLContainer(ctx context.Context, t *testing.T, image string) *MySQLTestContainer { - container, err := tcMySQL.RunContainer(ctx, - testcontainers.WithImage(image), - tcMySQL.WithDatabase("testdb"), - tcMySQL.WithUsername("root"), - tcMySQL.WithPassword(""), - testcontainers.WithWaitStrategy( - wait.ForLog("ready for connections"). - WithOccurrence(2). - WithStartupTimeout(120*time.Second), - ), - ) - if err != nil { - t.Fatalf("Failed to start MySQL container: %v", err) - } - - endpoint, err := container.ConnectionString(ctx) - if err != nil { - t.Fatalf("Failed to get connection string: %v", err) - } - - return &MySQLTestContainer{ - Container: container, - Endpoint: endpoint, - Username: "root", - Password: "", - } -} - -func (m *MySQLTestContainer) SetupTestEnv(t *testing.T) func() { - os.Setenv("MYSQL_ENDPOINT", m.Endpoint) - os.Setenv("MYSQL_USERNAME", m.Username) - os.Setenv("MYSQL_PASSWORD", m.Password) - - return func() { - os.Unsetenv("MYSQL_ENDPOINT") - os.Unsetenv("MYSQL_USERNAME") - os.Unsetenv("MYSQL_PASSWORD") - ctx := context.Background() - if err := m.Container.Terminate(ctx); err != nil { - t.Logf("Warning: Failed to terminate container: %v", err) - } - } -} - -// Usage in tests -func TestAccDatabase_Simple(t *testing.T) { - ctx := context.Background() - container := startMySQLContainer(ctx, t, "mysql:8.0") - defer container.SetupTestEnv(t)() - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - Steps: []resource.TestStep{ - { - Config: testAccDatabaseConfig("testdb"), - Check: resource.ComposeTestCheckFunc( - testAccDatabaseExists("mysql_database.test"), - ), - }, - }, - }) -} -``` - -### Example: Percona and MariaDB Support - -```go -func startPerconaContainer(ctx context.Context, t *testing.T, version string) *MySQLTestContainer { - image := fmt.Sprintf("percona:%s", version) - // Percona uses same MySQL protocol, can use MySQL module - return startMySQLContainer(ctx, t, image) -} - -func startMariaDBContainer(ctx context.Context, t *testing.T, version string) *MySQLTestContainer { - image := fmt.Sprintf("mariadb:%s", version) - // MariaDB also uses MySQL protocol - return startMySQLContainer(ctx, t, image) -} -``` - -### Example: TiDB Multi-Container Setup - -```go -import ( - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/modules/compose" -) - -func startTiDBCluster(ctx context.Context, t *testing.T, version string) *MySQLTestContainer { - // Option 1: Use Docker Compose module - composeFile := fmt.Sprintf(` -version: '3' -services: - pd: - image: pingcap/pd:v%s - command: --name=pd --data-dir=/data --client-urls=http://0.0.0.0:2379 --peer-urls=http://0.0.0.0:2380 - tikv: - image: pingcap/tikv:v%s - depends_on: [pd] - command: --addr=0.0.0.0:20160 --advertise-addr=tikv:20160 --pd=pd:2379 - tidb: - image: pingcap/tidb:v%s - depends_on: [tikv] - ports: - - "4000" - command: --store=tikv --path=pd:2379 -`, version, version, version) - - composeContainer, err := compose.NewDockerCompose(composeFile) - if err != nil { - t.Fatalf("Failed to create compose: %v", err) - } - - err = composeContainer.Up(ctx, compose.Wait(true)) - if err != nil { - t.Fatalf("Failed to start TiDB cluster: %v", err) - } - - // Get TiDB port - tidbPort, err := composeContainer.ServicePort(ctx, "tidb", 4000) - if err != nil { - t.Fatalf("Failed to get TiDB port: %v", err) - } - - return &MySQLTestContainer{ - Endpoint: fmt.Sprintf("127.0.0.1:%s", tidbPort.Port()), - Username: "root", - Password: "", - } -} - -// Option 2: Manual multi-container setup -func startTiDBClusterManual(ctx context.Context, t *testing.T, version string) *MySQLTestContainer { - networkName := "tidb-test-network" - network, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{ - NetworkRequest: testcontainers.NetworkRequest{ - Name: networkName, - }, - }) - if err != nil { - t.Fatalf("Failed to create network: %v", err) - } - - // Start PD - pdContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: fmt.Sprintf("pingcap/pd:v%s", version), - Networks: []string{networkName}, - Cmd: []string{ - "--name=pd", - "--data-dir=/data", - "--client-urls=http://0.0.0.0:2379", - "--peer-urls=http://0.0.0.0:2380", - }, - }, - }) - if err != nil { - t.Fatalf("Failed to start PD: %v", err) - } - - // Start TiKV - tikvContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: fmt.Sprintf("pingcap/tikv:v%s", version), - Networks: []string{networkName}, - Cmd: []string{ - "--addr=0.0.0.0:20160", - "--advertise-addr=tikv:20160", - "--pd=pd:2379", - }, - }, - }) - if err != nil { - t.Fatalf("Failed to start TiKV: %v", err) - } - - // Start TiDB - tidbContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: fmt.Sprintf("pingcap/tidb:v%s", version), - ExposedPorts: []string{"4000/tcp"}, - Networks: []string{networkName}, - Cmd: []string{ - "--store=tikv", - "--path=pd:2379", - }, - WaitingFor: wait.ForLog("server is running MySQL protocol").WithStartupTimeout(240 * time.Second), - }, - }) - if err != nil { - t.Fatalf("Failed to start TiDB: %v", err) - } - - tidbPort, err := tidbContainer.MappedPort(ctx, "4000") - if err != nil { - t.Fatalf("Failed to get TiDB port: %v", err) - } - - return &MySQLTestContainer{ - Endpoint: fmt.Sprintf("127.0.0.1:%s", tidbPort.Port()), - Username: "root", - Password: "", - } -} -``` - -### Example: Table-Driven Tests for Multiple Versions - -```go -func TestAccDatabase_MultipleVersions(t *testing.T) { - versions := []struct { - name string - image string - }{ - {"MySQL5.6", "mysql:5.6"}, - {"MySQL5.7", "mysql:5.7"}, - {"MySQL8.0", "mysql:8.0"}, - {"Percona5.7", "percona:5.7"}, - {"Percona8.0", "percona:8.0"}, - {"MariaDB10.3", "mariadb:10.3"}, - {"MariaDB10.8", "mariadb:10.8"}, - {"MariaDB10.10", "mariadb:10.10"}, - } - - for _, v := range versions { - t.Run(v.name, func(t *testing.T) { - ctx := context.Background() - container := startMySQLContainer(ctx, t, v.image) - defer container.SetupTestEnv(t)() - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - Steps: []resource.TestStep{ - { - Config: testAccDatabaseConfig("testdb"), - Check: resource.ComposeTestCheckFunc( - testAccDatabaseExists("mysql_database.test"), - ), - }, - }, - }) - }) - } -} -``` - -## Migration Strategy - -### Phase 1: Proof of Concept (Current) -- Create spike document ✅ -- Implement helper functions for MySQL/Percona/MariaDB -- Convert one test file as proof of concept -- Measure performance and reliability - -### Phase 2: Gradual Migration -- Convert tests file-by-file -- Keep Makefile targets as fallback during migration -- Update CI/CD to support both approaches - -### Phase 3: Complete Migration -- Remove Makefile targets -- Remove shell scripts (`tidb-test-cluster.sh`) -- Simplify GitHub Actions workflow -- Update documentation - -## Benefits Analysis - -### Development Experience -- ✅ **Simpler**: No Makefile, no shell scripts -- ✅ **Type Safe**: Compile-time errors instead of runtime shell errors -- ✅ **Better IDE Support**: Go tooling works out of the box -- ✅ **Easier Debugging**: Container logs automatically captured - -### Testing -- ✅ **Parallel Execution**: Go's testing framework handles this -- ✅ **Isolation**: Each test can have its own container -- ✅ **Reliability**: Automatic retries and better error handling -- ✅ **Port Management**: No manual port calculation needed - -### CI/CD -- ✅ **Simpler Workflow**: Just `go test` - no Makefile coordination -- ✅ **Better Caching**: Testcontainers can reuse containers -- ✅ **Faster**: Parallel test execution -- ✅ **More Reliable**: Better error messages and retry logic - -## Trade-offs and Considerations - -### Challenges - -1. **TiDB Complexity**: Multi-container setup still complex, but better than shell scripts -2. **Learning Curve**: Team needs to learn Testcontainers API -3. **Container Runtime Dependency**: Requires Docker or Podman (same as current approach) -4. **Migration Effort**: Need to convert all tests - -### Potential Issues - -1. **Container Startup Time**: May be slower than optimized Makefile approach - - **Mitigation**: Testcontainers has built-in caching and reuse - -2. **Resource Usage**: Multiple containers running in parallel - - **Mitigation**: Containers are lightweight, can limit parallelism - -3. **Network Issues**: Docker networking complexity - - **Mitigation**: Testcontainers handles this automatically - -## Podman Support - -Testcontainers Go automatically detects and works with Podman when Docker is not available or when `TESTCONTAINERS_RYUK_DISABLED=true` is set. Podman support is transparent - no code changes needed. - -### Podman Benefits - -1. **Rootless**: Can run without root privileges -2. **Daemonless**: No background daemon required -3. **Drop-in Replacement**: API-compatible with Docker -4. **Better Security**: Uses user namespaces - -### Podman Configuration - -Testcontainers will automatically use Podman if: -- Docker is not available, OR -- `CONTAINER_HOST` environment variable points to Podman socket - -Example: -```bash -# Use Podman explicitly -export CONTAINER_HOST=unix://$HOME/.local/share/containers/podman/machine/podman-machine-default/podman.sock - -# Or let Testcontainers auto-detect -# (it will try Docker first, then Podman) -``` - -### Podman Considerations - -- **Socket Path**: Podman socket location varies by installation -- **Rootless Mode**: May have different networking behavior -- **Compose Support**: Docker Compose module may not work with Podman (use manual multi-container setup for TiDB) - -## Implementation Plan - -### Step 1: Add Dependencies - -```bash -go get github.com/testcontainers/testcontainers-go -go get github.com/testcontainers/testcontainers-go/modules/mysql -go get github.com/testcontainers/testcontainers-go/modules/compose -``` - -### Step 1.5: Verify Podman Support (Optional) - -Test Podman compatibility: -```bash -# With Podman installed -export CONTAINER_HOST=unix://$HOME/.local/share/containers/podman/machine/podman-machine-default/podman.sock -go test -tags=spike ./mysql/... -run ExampleTestAccDatabase_WithTestcontainers -``` - -### Step 2: Create Helper Package - -Create `mysql/testcontainers_helper.go` with: -- `startMySQLContainer()` - MySQL/Percona/MariaDB -- `startTiDBCluster()` - TiDB multi-container setup -- `MySQLTestContainer` struct with cleanup logic - -### Step 3: Convert One Test File - -Start with `data_source_databases_test.go` as proof of concept: -- Replace `testAccPreCheck` to use Testcontainers -- Verify it works locally -- Measure performance - -### Step 4: Update CI/CD - -Update `.github/workflows/main.yml`: -- Remove Makefile targets -- Use `go test -tags=acceptance` with build tags -- Run tests in parallel using Go's built-in parallelism - -### Step 5: Gradual Migration - -Convert tests file-by-file: -1. `data_source_databases_test.go` -2. `resource_database_test.go` -3. `resource_user_test.go` -4. ... (continue with remaining files) - -## Performance Comparison - -### Current Approach -- Sequential container startup: ~10-30s per version -- Sequential test execution: ~60-90s per test suite -- Total for 13 versions: ~15-20 minutes - -### Testcontainers Approach (Estimated) -- Parallel container startup: ~10-30s (same, but parallel) -- Parallel test execution: ~60-90s (same, but parallel) -- Total for 13 versions: ~2-3 minutes (with parallelism) - -## Next Steps - -1. **Review this spike** with team -2. **Implement proof of concept** for one test file -3. **Measure performance** and compare with current approach -4. **Decide on migration strategy** (gradual vs. all-at-once) -5. **Create migration tickets** if approved - -## Podman-Specific Implementation Notes - -### TiDB with Podman - -Since Docker Compose may not work with Podman, use manual multi-container setup: - -```go -func startTiDBClusterPodman(ctx context.Context, t *testing.T, version string) *MySQLTestContainer { - // Use GenericContainer instead of Compose module - // This works with both Docker and Podman - return startTiDBClusterManual(ctx, t, version) -} -``` - -### Testing Podman Compatibility - -Add a build tag to test Podman specifically: - -```go -// +build podman - -// Test with Podman -func TestPodmanCompatibility(t *testing.T) { - // Verify Testcontainers detects Podman - // Run subset of tests to verify compatibility -} -``` - -Run with: -```bash -go test -tags=podman ./mysql/... -``` - -## References - -- [Testcontainers Go Documentation](https://golang.testcontainers.org/) -- [Testcontainers MySQL Module](https://golang.testcontainers.org/modules/mysql/) -- [Testcontainers Compose Module](https://golang.testcontainers.org/modules/compose/) -- [Testcontainers Podman Support](https://golang.testcontainers.org/features/container_daemons/) -- [Example: Terraform Provider Testing](https://github.com/testcontainers/testcontainers-go/tree/main/examples) diff --git a/TIDB_PLAYGROUND_OPTIMIZATION.md b/TIDB_PLAYGROUND_OPTIMIZATION.md deleted file mode 100644 index c72feca3..00000000 --- a/TIDB_PLAYGROUND_OPTIMIZATION.md +++ /dev/null @@ -1,200 +0,0 @@ -# TiDB Test Optimization: Switch to TiDB Playground - -## Current Problem - -TiDB tests are slow because they: -1. **Pull 3 large Docker images** (PD, TiKV, TiDB) - ~500MB-1GB total -2. **Start 3 containers sequentially** (PD → TiKV → TiDB) -3. **Wait for cluster coordination** - PD and TiKV need to sync before TiDB can start -4. **No image caching** - Images are pulled fresh each time - -**Current timing**: TiDB tests take 5-10+ minutes vs 1-2 minutes for regular MySQL tests - -## Solution: Use TiDB Playground - -TiDB Playground is TiDB's official tool for quickly spinning up local TiDB clusters. It's optimized for testing and development. - -### Benefits: -- **Single command** to start entire cluster -- **Pre-configured** - no manual coordination needed -- **Faster startup** - optimized for local use -- **Single Docker image** option (all-in-one) or binary -- **Better caching** - can use pre-pulled images - -## Implementation Options - -### Option 1: TiUP Playground (Recommended) - -TiUP is TiDB's package manager and includes `tiup playground` command. - -**Pros:** -- Official TiDB tool -- Supports all TiDB versions -- Fast startup (~30-60 seconds) -- Can specify exact version -- Single command - -**Cons:** -- Requires installing TiUP first -- Binary download (~10-20MB) - -**Installation:** -```bash -curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh -``` - -**Usage:** -```bash -tiup playground v7.5.2 --db 1 --kv 1 --pd 1 --tiflash 0 --without-monitor -``` - -### Option 2: Docker Compose with Pre-pulled Images - -Use docker-compose with image caching. - -**Pros:** -- Uses existing Docker infrastructure -- Can cache images between runs -- More control - -**Cons:** -- Still slower than playground -- More complex setup - -### Option 3: TiDB Playground Docker Image - -Use the official `pingcap/tidb-playground` Docker image. - -**Pros:** -- Single Docker image -- Pre-configured -- Fast startup - -**Cons:** -- May not support all versions -- Less control over components - -## Recommended Implementation: TiUP Playground - -### Step 1: Install TiUP in GitHub Actions - -Add to workflow: -```yaml -- name: Install TiUP - run: | - curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh - export PATH=$HOME/.tiup/bin:$PATH - tiup --version -``` - -### Step 2: Update Makefile - -Replace the current `testtidb` target: - -```makefile -testtidb: - @export PATH=$$HOME/.tiup/bin:$$PATH && \ - tiup playground $(MYSQL_VERSION) --db 1 --kv 1 --pd 1 --tiflash 0 --without-monitor --host 0.0.0.0 --db.port $(MYSQL_PORT) & \ - PLAYGROUND_PID=$$! && \ - echo "Waiting for TiDB..." && \ - while ! mysql -h 127.0.0.1 -P $(MYSQL_PORT) -u "$(TEST_USER)" -e 'SELECT 1' >/dev/null 2>&1; do \ - printf '.'; sleep 1; \ - done; \ - echo "Connected!" && \ - MYSQL_USERNAME="$(TEST_USER)" MYSQL_PASSWORD="" MYSQL_ENDPOINT=127.0.0.1:$(MYSQL_PORT) $(MAKE) testacc; \ - TEST_RESULT=$$?; \ - kill $$PLAYGROUND_PID || true; \ - exit $$TEST_RESULT -``` - -### Step 3: Alternative - Simpler Script Approach - -Create a new script `scripts/tidb-playground.sh`: - -```bash -#!/usr/bin/env bash -set -e - -VERSION=${1:-7.5.2} -PORT=${2:-4000} -MODE=${3:-start} # start or stop - -export PATH=$HOME/.tiup/bin:$PATH - -if [ "$MODE" = "start" ]; then - echo "Starting TiDB Playground v${VERSION} on port ${PORT}..." - tiup playground ${VERSION} \ - --db 1 \ - --kv 1 \ - --pd 1 \ - --tiflash 0 \ - --without-monitor \ - --host 0.0.0.0 \ - --db.port ${PORT} \ - --db.config "" \ - & - PLAYGROUND_PID=$! - echo $PLAYGROUND_PID > /tmp/tidb-playground-${PORT}.pid - - # Wait for TiDB to be ready - echo "Waiting for TiDB..." - for i in {1..60}; do - if mysql -h 127.0.0.1 -P ${PORT} -u root -e 'SELECT 1' >/dev/null 2>&1; then - echo "TiDB is ready!" - exit 0 - fi - sleep 1 - done - echo "TiDB failed to start" - exit 1 -elif [ "$MODE" = "stop" ]; then - if [ -f /tmp/tidb-playground-${PORT}.pid ]; then - PID=$(cat /tmp/tidb-playground-${PORT}.pid) - kill $PID 2>/dev/null || true - rm /tmp/tidb-playground-${PORT}.pid - fi - # Also kill any remaining tiup processes - pkill -f "tiup playground" || true -fi -``` - -Then update Makefile: -```makefile -testtidb: - @export PATH=$$HOME/.tiup/bin:$$PATH && \ - $(CURDIR)/scripts/tidb-playground.sh $(MYSQL_VERSION) $(MYSQL_PORT) start && \ - MYSQL_USERNAME="$(TEST_USER)" MYSQL_PASSWORD="" MYSQL_ENDPOINT=127.0.0.1:$(MYSQL_PORT) $(MAKE) testacc; \ - TEST_RESULT=$$?; \ - $(CURDIR)/scripts/tidb-playground.sh $(MYSQL_VERSION) $(MYSQL_PORT) stop; \ - exit $$TEST_RESULT -``` - -## Expected Performance Improvement - -**Current**: 5-10+ minutes per TiDB test -- Docker image pulls: 2-5 minutes -- Container startup: 1-2 minutes -- Cluster coordination: 1-2 minutes -- Test execution: 1-2 minutes - -**With TiUP Playground**: 2-3 minutes per TiDB test -- TiUP install (cached): ~5 seconds -- Playground startup: 30-60 seconds -- Test execution: 1-2 minutes - -**Savings**: ~3-7 minutes per TiDB test × 6 TiDB tests = **18-42 minutes** per workflow run! - -## Implementation Checklist - -- [ ] Install TiUP in GitHub Actions workflow -- [ ] Create tidb-playground.sh script (or update existing) -- [ ] Update Makefile testtidb target -- [ ] Test locally with TiUP -- [ ] Test in GitHub Actions -- [ ] Remove old tidb-test-cluster.sh script (or keep as fallback) - -## References - -- TiUP Documentation: https://docs.pingcap.com/tidb/stable/tiup-overview -- TiUP Playground: https://docs.pingcap.com/tidb/stable/tiup-playground -- TiUP Installation: https://docs.pingcap.com/tidb/stable/tiup-overview#install-tiup diff --git a/TIDB_VERSION_SYNC_PLAN.md b/TIDB_VERSION_SYNC_PLAN.md deleted file mode 100644 index b4bba565..00000000 --- a/TIDB_VERSION_SYNC_PLAN.md +++ /dev/null @@ -1,185 +0,0 @@ -# Plan: Keep TiDB Versions in Sync - -## Problem - -TiDB versions are currently defined in two places: -1. Test matrix: `testtidb6.1.0`, `testtidb6.5.3`, etc. -2. Pre-download step: `VERSIONS="6.1.0 6.5.3 7.1.5 7.5.2 8.1.0"` - -This creates a maintenance burden - versions can get out of sync. - -## Solution Options - -### Option 1: Extract Versions from Matrix (Recommended) - -Use GitHub Actions matrix variables to define versions once, then derive test targets. - -**Pros:** -- Single source of truth -- Automatic sync -- Easy to add/remove versions - -**Cons:** -- Requires restructuring the matrix - -**Implementation:** -```yaml -tests: - strategy: - matrix: - tidb_version: - - "6.1.0" - - "6.5.3" - - "7.1.5" - - "7.5.2" - - "8.1.0" - mysql_target: - - testversion5.6 - - testversion5.7 - # ... other MySQL tests - include: - # Combine TiDB versions with test targets - - tidb_version: "6.1.0" - target: testtidb6.1.0 - - tidb_version: "6.5.3" - target: testtidb6.5.3 - # ... etc -``` - -### Option 2: Use Environment Variable - -Define versions as environment variable, use in both places. - -**Pros:** -- Simple -- Single definition - -**Cons:** -- Still requires manual updates -- Less flexible - -**Implementation:** -```yaml -env: - TIDB_VERSIONS: "6.1.0 6.5.3 7.1.5 7.5.2 8.1.0" - -jobs: - prepare-dependencies: - steps: - - name: Pre-download TiDB versions - run: | - for version in ${{ env.TIDB_VERSIONS }}; do - tiup install tidb:v${version} || true - done -``` - -### Option 3: Extract from Matrix Dynamically - -Use GitHub Actions to extract versions from matrix and pass to prepare job. - -**Pros:** -- Fully automatic -- No manual sync needed - -**Cons:** -- More complex -- Requires job outputs - -**Implementation:** -```yaml -prepare-dependencies: - outputs: - tidb_versions: ${{ steps.set-versions.outputs.versions }} - steps: - - name: Set TiDB versions - id: set-versions - run: | - # Extract from matrix (would need to parse workflow file or use job outputs) -``` - -### Option 4: Use a Separate Job to Extract Versions - -Create a job that reads the matrix and outputs versions. - -**Pros:** -- Can parse matrix dynamically -- Single source of truth - -**Cons:** -- Most complex -- Requires parsing YAML or using GitHub Actions features - -## Recommended Solution: Option 2 (Environment Variable) - -Simplest and most maintainable approach: - -1. Define versions once as environment variable -2. Use in both test matrix (via script) and pre-download step -3. Easy to update - change one place - -## Implementation - -### Step 1: Define Versions at Workflow Level - -```yaml -env: - TIDB_VERSIONS: "6.1.0 6.5.3 7.1.5 7.5.2 8.1.0" -``` - -### Step 2: Use in Pre-download - -```yaml -- name: Pre-download TiDB versions - run: | - export PATH=$HOME/.tiup/bin:$PATH - for version in ${{ env.TIDB_VERSIONS }}; do - echo "Pre-downloading TiDB components for v${version}..." - tiup install tidb:v${version} || true - tiup install pd:v${version} || true - tiup install tikv:v${version} || true - done -``` - -### Step 3: Generate Test Matrix from Versions - -Use a script or GitHub Actions feature to generate test targets from versions. - -However, GitHub Actions doesn't support dynamic matrix generation easily. So we'd need to: -- Keep matrix explicit but add comment referencing env var -- Or use a script to validate they match - -### Alternative: Use Matrix Include Pattern - -Better approach - define versions once, use matrix include: - -```yaml -tests: - strategy: - matrix: - tidb_version: ["6.1.0", "6.5.3", "7.1.5", "7.5.2", "8.1.0"] - include: - - tidb_version: "6.1.0" - target: testtidb6.1.0 - - tidb_version: "6.5.3" - target: testtidb6.5.3 - # ... etc - exclude: - - tidb_version: "6.1.0" # Only run if target is set -``` - -But this doesn't help with pre-download. - -## Best Practical Solution - -**Use environment variable + validation script:** - -1. Define versions as env var -2. Use in pre-download -3. Add validation step that checks matrix matches env var -4. Or use a script that generates both from a single source - -Actually, the simplest is to: -- Keep versions in env var -- Use in pre-download -- Add a comment in matrix referencing the env var -- Add a validation step that fails if they don't match diff --git a/WORKFLOW_OPTIMIZATION_ANALYSIS.md b/WORKFLOW_OPTIMIZATION_ANALYSIS.md deleted file mode 100644 index a52a6578..00000000 --- a/WORKFLOW_OPTIMIZATION_ANALYSIS.md +++ /dev/null @@ -1,199 +0,0 @@ -# GitHub Actions Workflow Optimization Analysis - -## Current Performance Analysis - -Based on run https://github.com/zph/terraform-provider-mysql/actions/runs/19516178533: - -### Job Timings Summary - -**Prepare Dependencies**: ~22s -- Set up Go: ~10s -- Vendor Go dependencies: ~1s -- Download Terraform: ~0s -- Upload artifacts: ~1-2s - -**Test Jobs** (13 jobs, running in parallel): 76-105s each -- Set up job: 1-3s -- Checkout Git repo: 0-1s -- **Set up Go: 9-13s** ⚠️ (repeated 13 times!) -- Download Terraform binary: 1-3s -- Download vendor directory: 1-2s -- **Install mysql client: ~5-10s** ⚠️ (apt-get update is slow) -- Run tests: 60-90s (actual test execution) - -**Total workflow time**: ~2-3 minutes (determined by longest test job) - -## Optimization Opportunities - -### 1. ⚠️ HIGH IMPACT: Optimize mysql-client Installation - -**Current**: Each job runs `sudo apt-get update` which is slow (5-10s) - -**Solution**: Skip apt-get update or use cached packages - -```yaml -- name: Install mysql client - run: | - sudo apt-get update -qq # -qq makes it quieter and slightly faster - sudo apt-get install -y --no-install-recommends mysql-client -``` - -**Better Solution**: Use apt cache or skip update if possible - -```yaml -- name: Install mysql client - run: | - sudo apt-get install -y --no-install-recommends mysql-client || \ - (sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends mysql-client) -``` - -**Best Solution**: Cache apt packages - -```yaml -- name: Cache apt packages - uses: actions/cache@v4 - with: - path: /var/cache/apt - key: ${{ runner.os }}-apt-${{ hashFiles('**/.github/workflows/main.yml') }} - restore-keys: | - ${{ runner.os }}-apt- - -- name: Install mysql client - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends mysql-client -``` - -**Expected Savings**: 3-5 seconds per job × 13 jobs = **39-65 seconds** - -### 2. ⚠️ MEDIUM IMPACT: Optimize Go Setup - -**Current**: `actions/setup-go@v4` takes 9-13s per job - -**Issue**: Go is being set up 13 times independently - -**Solution**: This is somewhat unavoidable, but we can: -- Ensure we're using the latest version of the action (should be faster) -- Consider caching Go installation (though actions/setup-go should handle this) - -**Note**: `actions/setup-go` should cache Go installations, so subsequent runs might be faster. The 9-13s might be acceptable. - -**Potential Savings**: Minimal (this is likely already optimized by GitHub Actions) - -### 3. ⚠️ MEDIUM IMPACT: Optimize Artifact Downloads - -**Current**: 1-3s per artifact download - -**Optimization**: Use parallel downloads or optimize compression - -The vendor directory upload already uses `compression-level: 6` which is good. We could: -- Use `compression-level: 9` for better compression (smaller downloads, but slower upload) -- Or use `compression-level: 1` for faster uploads (larger downloads, but faster overall) - -**Expected Savings**: 1-2 seconds per job × 13 jobs = **13-26 seconds** - -### 4. ⚠️ LOW-MEDIUM IMPACT: Optimize Test Execution - -**Current**: Tests take 60-90s per job - -**Potential Optimizations**: -- Run tests in parallel within each job (if not already) -- Optimize test timeouts -- Skip unnecessary test setup/teardown - -**Note**: This requires code changes and might affect test reliability. - -### 5. ⚠️ LOW IMPACT: Optimize Checkout - -**Current**: Checkout takes 0-1s - -**Optimization**: Use `actions/checkout@v4` with `fetch-depth: 0` only if needed, or use `fetch-depth: 1` for faster checkout - -```yaml -- name: Checkout Git repo - uses: actions/checkout@v4 - with: - fetch-depth: 1 # Only fetch latest commit, faster -``` - -**Expected Savings**: Minimal (~0.5s per job) - -## Recommended Implementation Priority - -### Priority 1: Optimize mysql-client Installation (HIGHEST IMPACT) -- **Effort**: Low -- **Impact**: High (39-65 seconds saved) -- **Risk**: Low - -### Priority 2: Optimize Artifact Compression -- **Effort**: Low -- **Impact**: Medium (13-26 seconds saved) -- **Risk**: Low - -### Priority 3: Optimize Checkout Depth -- **Effort**: Low -- **Impact**: Low (~6 seconds saved) -- **Risk**: Very Low - -## Implementation Plan - -### Step 1: Add apt package caching - -```yaml -- name: Cache apt packages - uses: actions/cache@v4 - with: - path: /var/cache/apt - key: ${{ runner.os }}-apt-${{ hashFiles('**/.github/workflows/main.yml') }} - restore-keys: | - ${{ runner.os }}-apt- - -- name: Install mysql client - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends mysql-client -``` - -### Step 2: Optimize checkout (if changelog generation not needed) - -```yaml -- name: Checkout Git repo - uses: actions/checkout@v4 - with: - fetch-depth: 1 # Only if changelog generation not needed -``` - -### Step 3: Consider artifact compression tuning - -Current compression-level: 6 is a good balance. Could experiment with: -- Level 1: Faster upload, larger download -- Level 9: Slower upload, smaller download - -## Expected Total Savings - -- **Apt caching**: 39-65 seconds -- **Artifact optimization**: 13-26 seconds -- **Checkout optimization**: ~6 seconds -- **Total**: ~58-97 seconds (~1-1.5 minutes) per workflow run - -## Additional Considerations - -### Docker-based Approach (Future Consideration) - -For even better performance, consider using a custom Docker image with: -- Go pre-installed -- mysql-client pre-installed -- All dependencies cached - -This would eliminate: -- Go setup time (9-13s × 13 = 117-169s) -- mysql-client installation time (5-10s × 13 = 65-130s) - -**Total potential savings**: ~3-5 minutes per workflow run - -However, this requires: -- Maintaining a Docker image -- Building and pushing the image -- More complex setup - -**Recommendation**: Start with apt caching first, then consider Docker if more optimization is needed. diff --git a/mysql/data_source_databases_test.go b/mysql/data_source_databases_test.go index 64c66e24..b7b44207 100644 --- a/mysql/data_source_databases_test.go +++ b/mysql/data_source_databases_test.go @@ -1,3 +1,6 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( @@ -10,6 +13,9 @@ import ( ) func TestAccDataSourceDatabases(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, ProviderFactories: testAccProviderFactories, diff --git a/mysql/data_source_databases_testcontainers_test.go b/mysql/data_source_databases_testcontainers_test.go deleted file mode 100644 index d8625a7d..00000000 --- a/mysql/data_source_databases_testcontainers_test.go +++ /dev/null @@ -1,51 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "fmt" - "testing" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" -) - -// TestAccDataSourceDatabases_WithTestcontainers is a proof of concept test -// using Testcontainers instead of Makefile + Docker -// Uses shared container set up in TestMain -func TestAccDataSourceDatabases_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - // Run the same test logic as the original test - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - Steps: []resource.TestStep{ - { - Config: testAccDatabasesConfigBasic("%"), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("data.mysql_databases.test", "pattern", "%"), - testAccDatabasesCount("data.mysql_databases.test", "databases.#", func(rn string, databaseCount int) error { - if databaseCount < 1 { - return fmt.Errorf("%s: databases not found", rn) - } - return nil - }), - ), - }, - { - Config: testAccDatabasesConfigBasic("__database_does_not_exist__"), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("data.mysql_databases.test", "pattern", "__database_does_not_exist__"), - testAccDatabasesCount("data.mysql_databases.test", "databases.#", func(rn string, databaseCount int) error { - if databaseCount > 0 { - return fmt.Errorf("%s: unexpected database found", rn) - } - return nil - }), - ), - }, - }, - }) -} diff --git a/mysql/data_source_tables_test.go b/mysql/data_source_tables_test.go index 57f99c88..c47f751b 100644 --- a/mysql/data_source_tables_test.go +++ b/mysql/data_source_tables_test.go @@ -1,3 +1,6 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( @@ -10,6 +13,9 @@ import ( ) func TestAccDataSourceTables(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, ProviderFactories: testAccProviderFactories, diff --git a/mysql/data_source_tables_testcontainers_test.go b/mysql/data_source_tables_testcontainers_test.go deleted file mode 100644 index 4ad7cf1d..00000000 --- a/mysql/data_source_tables_testcontainers_test.go +++ /dev/null @@ -1,53 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "fmt" - "testing" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" -) - -// TestAccDataSourceTables_WithTestcontainers tests the mysql_tables data source -// using Testcontainers instead of Makefile + Docker -// Uses shared container set up in TestMain -func TestAccDataSourceTables_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - // Run the same test logic as the original test - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - Steps: []resource.TestStep{ - { - Config: testAccTablesConfigBasic("mysql", "%"), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("data.mysql_tables.test", "database", "mysql"), - resource.TestCheckResourceAttr("data.mysql_tables.test", "pattern", "%"), - testAccTablesCount("data.mysql_tables.test", "tables.#", func(rn string, tableCount int) error { - if tableCount < 1 { - return fmt.Errorf("%s: tables not found", rn) - } - return nil - }), - ), - }, - { - Config: testAccTablesConfigBasic("mysql", "__table_does_not_exist__"), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("data.mysql_tables.test", "database", "mysql"), - resource.TestCheckResourceAttr("data.mysql_tables.test", "pattern", "__table_does_not_exist__"), - testAccTablesCount("data.mysql_tables.test", "tables.#", func(rn string, tableCount int) error { - if tableCount > 0 { - return fmt.Errorf("%s: unexpected table found", rn) - } - return nil - }), - ), - }, - }, - }) -} diff --git a/mysql/provider_test.go b/mysql/provider_test.go index 6c93bc80..e2cc541a 100644 --- a/mysql/provider_test.go +++ b/mysql/provider_test.go @@ -1,258 +1,109 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( - "context" "fmt" "os" "strings" "testing" - - "github.com/hashicorp/go-version" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" - "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) -// To run these acceptance tests, you will need access to a MySQL server. -// Amazon RDS is one way to get a MySQL server. If you use RDS, you can -// use the root account credentials you specified when creating an RDS -// instance to get the access necessary to run these tests. (the tests -// assume full access to the server.) -// -// Set the MYSQL_ENDPOINT and MYSQL_USERNAME environment variables before -// running the tests. If the given user has a password then you will also need -// to set MYSQL_PASSWORD. -// -// The tests assume a reasonably-vanilla MySQL configuration. In particular, -// they assume that the "utf8" character set is available and that -// "utf8_bin" is a valid collation that isn't the default for that character -// set. -// -// You can run the tests like this: -// make testacc TEST=./builtin/providers/mysql - -var testAccProviderFactories map[string]func() (*schema.Provider, error) - -// var testAccProviders map[string]*schema.Provider -var testAccProvider *schema.Provider - -func init() { - testAccProvider = Provider() - testAccProviderFactories = map[string]func() (*schema.Provider, error){ - "mysql": func() (*schema.Provider, error) { return testAccProvider, nil }, - } -} - -func TestProvider(t *testing.T) { - if err := Provider().InternalValidate(); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestProvider_impl(t *testing.T) { - var _ = Provider() -} - -func testAccPreCheck(t *testing.T) { - ctx := context.Background() - for _, name := range []string{"MYSQL_ENDPOINT", "MYSQL_USERNAME"} { - if v := os.Getenv(name); v == "" { - t.Fatal("MYSQL_ENDPOINT, MYSQL_USERNAME and optionally MYSQL_PASSWORD must be set for acceptance tests") - } - } - - raw := map[string]interface{}{ - "conn_params": map[string]interface{}{}, - } - err := testAccProvider.Configure(ctx, terraform.NewResourceConfigRaw(raw)) - if err != nil { - t.Fatal(err) - } -} - -func testAccPreCheckSkipNotRds(t *testing.T) { - testAccPreCheck(t) - - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - return - } - - rdsEnabled, err := serverRds(db) - if err != nil { - return - } - - if !rdsEnabled { - t.Skip("Skip on non RDS instance") - } -} - -func testAccPreCheckSkipRds(t *testing.T) { - testAccPreCheck(t) - - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - if strings.Contains(err.Error(), "SUPER privilege(s) for this operation") { - t.Skip("Skip on RDS") +// TestMain sets up a shared MySQL/TiDB container for all testcontainers tests +// This is more efficient than starting a container for each test +func TestMain(m *testing.M) { + // Force stderr to be unbuffered so debug output appears immediately + os.Stderr.WriteString("TestMain: ENTRY POINT REACHED\n") + os.Stderr.Sync() + + // Require DOCKER_IMAGE to be set - fail early if missing + dockerImage := os.Getenv("DOCKER_IMAGE") + os.Stderr.WriteString(fmt.Sprintf("TestMain: DOCKER_IMAGE='%s'\n", dockerImage)) + os.Stderr.Sync() + + if dockerImage == "" { + os.Stderr.WriteString("ERROR: DOCKER_IMAGE environment variable is not set.\n") + os.Stderr.WriteString("Please set DOCKER_IMAGE to the appropriate Docker image:\n") + os.Stderr.WriteString(" - MySQL/Percona/MariaDB: mysql:5.6, percona:8.0, mariadb:10.10\n") + os.Stderr.WriteString(" - TiDB: tidb:6.1.7, tidb:8.5.3\n") + os.Exit(1) + } + + // Debug: Log that TestMain is running + os.Stderr.WriteString(fmt.Sprintf("TestMain: Starting with DOCKER_IMAGE='%s'\n", dockerImage)) + os.Stderr.Sync() + + // Check if we're testing TiDB (format: tidb:VERSION) + // TiDB requires multi-container setup + if strings.HasPrefix(dockerImage, "tidb:") { + tidbVersion := strings.TrimPrefix(dockerImage, "tidb:") + if tidbVersion == "" { + os.Stderr.WriteString("ERROR: DOCKER_IMAGE format for TiDB must be 'tidb:VERSION' (e.g., tidb:6.1.7)\n") + os.Exit(1) } - return - } - - rdsEnabled, err := serverRds(db) - if err != nil { - return - } - - if rdsEnabled { - t.Skip("Skip on RDS") - } -} - -func testAccPreCheckSkipTiDB(t *testing.T) { - testAccPreCheck(t) - - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - t.Fatalf("Cannot connect to DB (SkipTiDB): %v", err) - return - } - - currentVersionString, err := serverVersionString(db) - if err != nil { - t.Fatalf("Cannot get DB version string (SkipTiDB): %v", err) - return - } - - if strings.Contains(currentVersionString, "TiDB") { - t.Skip("Skip on TiDB") - } -} - -func testAccPreCheckSkipMariaDB(t *testing.T) { - testAccPreCheck(t) - - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - t.Fatalf("Cannot connect to DB (SkipMariaDB): %v", err) - return - } - - currentVersionString, err := serverVersionString(db) - if err != nil { - t.Fatalf("Cannot get DB version string (SkipMariaDB): %v", err) - return - } - - if strings.Contains(currentVersionString, "MariaDB") { - t.Skip("Skip on MariaDB") - } -} - -func testAccPreCheckSkipNotMySQL8(t *testing.T) { - testAccPreCheckSkipNotMySQLVersionMin(t, "8.0.0") -} - -func testAccPreCheckSkipNotMySQLVersionMin(t *testing.T, minVersion string) { - testAccPreCheck(t) - - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - t.Fatalf("Cannot connect to DB (SkipNotMySQL8): %v", err) - return - } - currentVersion, err := serverVersion(db) - if err != nil { - t.Fatalf("Cannot get DB version string (SkipNotMySQL8): %v", err) - return - } + // Start shared TiDB cluster before running tests + var err error + sharedTiDBClusterMtx.Lock() + sharedTiDBCluster, err = startSharedTiDBCluster(tidbVersion) + sharedTiDBClusterMtx.Unlock() - versionMin, _ := version.NewVersion(minVersion) - if currentVersion.LessThan(versionMin) { - // TiDB 7.x series advertises as 8.0 mysql so we batch its testing strategy with Mysql8 - isTiDB, tidbVersion, mysqlCompatibilityVersion, err := serverTiDB(db) if err != nil { - t.Fatalf("Cannot get DB version string (SkipNotMySQL8): %v", err) - return - } - if isTiDB { - mysqlVersion, err := version.NewVersion(mysqlCompatibilityVersion) - if err != nil { - t.Fatalf("Cannot get DB version string for TiDB (SkipNotMySQL8): %s %s %v", tidbVersion, mysqlCompatibilityVersion, err) - return - } - if mysqlVersion.LessThan(versionMin) { - t.Skip("Skip on MySQL8") - } + // If cluster startup fails, exit with error + os.Stderr.WriteString(fmt.Sprintf("Failed to start shared TiDB cluster: %v\n", err)) + os.Exit(1) } - t.Skip("Skip on MySQL8") - } -} + // Set up environment variables for the shared TiDB cluster + os.Setenv("MYSQL_ENDPOINT", sharedTiDBCluster.Endpoint) + os.Setenv("MYSQL_USERNAME", sharedTiDBCluster.Username) + os.Setenv("MYSQL_PASSWORD", sharedTiDBCluster.Password) -func testAccPreCheckSkipNotTiDB(t *testing.T) { - testAccPreCheck(t) + // Run all tests + code := m.Run() - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - t.Fatalf("Cannot connect to DB (SkipNotTiDB): %v", err) - return - } - - currentVersionString, err := serverVersionString(db) - if err != nil { - t.Fatalf("Cannot get DB version string (SkipNotTiDB): %v", err) - return - } + // Cleanup shared TiDB cluster after all tests complete + cleanupSharedTiDBCluster() - if !strings.Contains(currentVersionString, "TiDB") { - msg := fmt.Sprintf("Skip on MySQL %s", currentVersionString) - t.Skip(msg) + // Exit with test result code + os.Exit(code) } -} - -func testAccPreCheckSkipNotTiDBVersionMin(t *testing.T, minVersion string) { - testAccPreCheck(t) - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - t.Fatalf("Cannot connect to DB (SkipNotTiDBVersionMin): %v", err) - return - } + // MySQL/Percona/MariaDB mode - use single container + // Start shared container before running tests + var err error + sharedContainerMtx.Lock() + sharedContainer, err = startSharedMySQLContainer(dockerImage) + sharedContainerMtx.Unlock() - currentVersion, err := serverVersion(db) if err != nil { - t.Fatalf("Cannot get DB version string (SkipNotTiDBVersionMin): %v", err) - return - } - - versionMin, _ := version.NewVersion(minVersion) - if currentVersion.LessThan(versionMin) { - isTiDB, tidbVersion, _, err := serverTiDB(db) - if err != nil { - t.Fatalf("Cannot get DB version string (SkipNotTiDBVersionMin): %v", err) - return - } - if isTiDB { - tidbSemVar, err := version.NewVersion(tidbVersion) - if err != nil { - t.Fatalf("Cannot get DB version string for TiDB (SkipNotTiDBVersionMin): %s %v", tidbSemVar, err) - return - } - if tidbSemVar.LessThan(versionMin) { - t.Skip("Skip on TiDB (SkipNotTiDBVersionMin)") - } - return - } - - t.Skip("Skip on MySQL") - } + // If container startup fails, exit with error + os.Stderr.WriteString(fmt.Sprintf("Failed to start shared MySQL container: %v\n", err)) + os.Exit(1) + } + + // Set up environment variables for the shared container + // These MUST be set even if sharedContainer is nil (shouldn't happen, but be defensive) + if sharedContainer != nil { + os.Setenv("MYSQL_ENDPOINT", sharedContainer.Endpoint) + os.Setenv("MYSQL_USERNAME", sharedContainer.Username) + os.Setenv("MYSQL_PASSWORD", sharedContainer.Password) + // Debug: Log that environment variables are set + os.Stderr.WriteString(fmt.Sprintf("TestMain: Set MYSQL_ENDPOINT='%s'\n", sharedContainer.Endpoint)) + os.Stderr.Sync() + } else { + // This should never happen, but if it does, fail loudly + os.Stderr.WriteString(fmt.Sprintf("ERROR: startSharedMySQLContainer returned nil container without error for image '%s'\n", dockerImage)) + os.Exit(1) + } + + // Run all tests + code := m.Run() + + // Cleanup shared container after all tests complete + cleanupSharedContainer() + + // Exit with test result code + os.Exit(code) } diff --git a/mysql/provider_test_common.go b/mysql/provider_test_common.go new file mode 100644 index 00000000..7b5829fb --- /dev/null +++ b/mysql/provider_test_common.go @@ -0,0 +1,375 @@ +package mysql + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + + "github.com/hashicorp/go-version" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" +) + +// To run these acceptance tests, you will need access to a MySQL server. +// Amazon RDS is one way to get a MySQL server. If you use RDS, you can +// use the root account credentials you specified when creating an RDS +// instance to get the access necessary to run these tests. (the tests +// assume full access to the server.) +// +// Set the MYSQL_ENDPOINT and MYSQL_USERNAME environment variables before +// running the tests. If the given user has a password then you will also need +// to set MYSQL_PASSWORD. +// +// The tests assume a reasonably-vanilla MySQL configuration. In particular, +// they assume that the "utf8" character set is available and that +// "utf8_bin" is a valid collation that isn't the default for that character +// set. +// +// You can run the tests like this: +// make testacc TEST=./builtin/providers/mysql + +var testAccProviderFactories map[string]func() (*schema.Provider, error) + +// var testAccProviders map[string]*schema.Provider +var testAccProvider *schema.Provider + +func init() { + testAccProvider = Provider() + testAccProviderFactories = map[string]func() (*schema.Provider, error){ + "mysql": func() (*schema.Provider, error) { return testAccProvider, nil }, + } +} + +func TestProvider(t *testing.T) { + if err := Provider().InternalValidate(); err != nil { + t.Fatalf("err: %s", err) + } +} + +func TestProvider_impl(t *testing.T) { + var _ = Provider() +} + +func testAccPreCheck(t *testing.T) { + ctx := context.Background() + for _, name := range []string{"MYSQL_ENDPOINT", "MYSQL_USERNAME"} { + if v := os.Getenv(name); v == "" { + // If container failed to start, allow tests to skip gracefully + // Check DOCKER_IMAGE to provide helpful error message + dockerImage := os.Getenv("DOCKER_IMAGE") + if dockerImage != "" { + t.Fatalf("MYSQL_ENDPOINT not set - container may have failed to start for %s. Check TestMain logs.", dockerImage) + } + t.Fatal("MYSQL_ENDPOINT, MYSQL_USERNAME and optionally MYSQL_PASSWORD must be set for acceptance tests") + } + } + + raw := map[string]interface{}{ + "conn_params": map[string]interface{}{}, + } + err := testAccProvider.Configure(ctx, terraform.NewResourceConfigRaw(raw)) + if err != nil { + t.Fatal(err) + } +} + +func testAccPreCheckSkipNotRds(t *testing.T) { + testAccPreCheck(t) + + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return + } + + rdsEnabled, err := serverRds(db) + if err != nil { + return + } + + if !rdsEnabled { + t.Skip("Skip on non RDS instance") + } +} + +func testAccPreCheckSkipRds(t *testing.T) { + // Check if container startup failed - if so, skip (can't determine if RDS) + containerStartupFailed := os.Getenv("CONTAINER_STARTUP_FAILED") + if containerStartupFailed == "1" { + t.Skip("Skip on RDS (container startup failed, cannot determine RDS status)") + } + + testAccPreCheck(t) + + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + if strings.Contains(err.Error(), "SUPER privilege(s) for this operation") { + t.Skip("Skip on RDS") + } + return + } + + rdsEnabled, err := serverRds(db) + if err != nil { + return + } + + if rdsEnabled { + t.Skip("Skip on RDS") + } +} + +func testAccPreCheckSkipTiDB(t *testing.T) { + // Early skip check based on DOCKER_IMAGE before connecting + dockerImage := os.Getenv("DOCKER_IMAGE") + if dockerImage != "" && strings.HasPrefix(dockerImage, "tidb:") { + t.Skip("Skip on TiDB") + } + + testAccPreCheck(t) + + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + t.Fatalf("Cannot connect to DB (SkipTiDB): %v", err) + return + } + + currentVersionString, err := serverVersionString(db) + if err != nil { + t.Fatalf("Cannot get DB version string (SkipTiDB): %v", err) + return + } + + if strings.Contains(currentVersionString, "TiDB") { + t.Skip("Skip on TiDB") + } +} + +func testAccPreCheckSkipMariaDB(t *testing.T) { + // Early skip check based on DOCKER_IMAGE before connecting + dockerImage := os.Getenv("DOCKER_IMAGE") + if dockerImage != "" && strings.HasPrefix(dockerImage, "mariadb:") { + t.Skip("Skip on MariaDB") + } + + testAccPreCheck(t) + + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + t.Fatalf("Cannot connect to DB (SkipMariaDB): %v", err) + return + } + + currentVersionString, err := serverVersionString(db) + if err != nil { + t.Fatalf("Cannot get DB version string (SkipMariaDB): %v", err) + return + } + + if strings.Contains(currentVersionString, "MariaDB") { + t.Skip("Skip on MariaDB") + } +} + +func testAccPreCheckSkipNotMySQL8(t *testing.T) { + testAccPreCheckSkipNotMySQLVersionMin(t, "8.0.0") +} + +func testAccPreCheckSkipNotMySQLVersionMin(t *testing.T, minVersion string) { + // Early skip check based on DOCKER_IMAGE before connecting + // This allows tests to skip even if TestMain failed to start the container + dockerImage := os.Getenv("DOCKER_IMAGE") + if dockerImage != "" { + // Parse version from DOCKER_IMAGE (e.g., "mysql:5.6", "percona:5.7", "mariadb:10.3") + parts := strings.Split(dockerImage, ":") + if len(parts) == 2 { + imageName := parts[0] + imageVersion := parts[1] + + // Check if this is a MySQL/Percona version that's definitely too old + if imageName == "mysql" || imageName == "percona" || strings.HasPrefix(imageName, "percona/") { + versionMin, err := version.NewVersion(minVersion) + if err == nil { + // Try to parse the image version + imgVer, err := version.NewVersion(imageVersion) + if err == nil && imgVer.LessThan(versionMin) { + t.Skipf("Skip on %s (requires MySQL %s+)", dockerImage, minVersion) + } + } + } + // MariaDB versions are typically 10.x, which are < 8.0, so skip if minVersion is 8.0+ + if imageName == "mariadb" { + versionMin, err := version.NewVersion(minVersion) + if err == nil { + // MariaDB 10.x is roughly equivalent to MySQL 5.x in terms of features + // So if we need MySQL 8.0+, skip MariaDB + mysql80, _ := version.NewVersion("8.0.0") + if versionMin.GreaterThanOrEqual(mysql80) { + t.Skipf("Skip on %s (requires MySQL %s+)", dockerImage, minVersion) + } + } + } + } + } + + // Check if container startup failed - if so, skip if we can determine incompatibility from DOCKER_IMAGE + // Otherwise, try to connect (which will fail gracefully) + containerStartupFailed := os.Getenv("CONTAINER_STARTUP_FAILED") + if containerStartupFailed == "1" { + // Container failed to start - if we already determined this version is incompatible, skip + // Otherwise, we'll fail in testAccPreCheck which is fine + } + + // Only call testAccPreCheck if we didn't skip above + // This allows tests to skip even if container startup failed + testAccPreCheck(t) + + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + t.Fatalf("Cannot connect to DB (SkipNotMySQL8): %v", err) + return + } + + currentVersion, err := serverVersion(db) + if err != nil { + t.Fatalf("Cannot get DB version string (SkipNotMySQL8): %v", err) + return + } + + versionMin, _ := version.NewVersion(minVersion) + if currentVersion.LessThan(versionMin) { + // TiDB 7.x series advertises as 8.0 mysql so we batch its testing strategy with Mysql8 + isTiDB, tidbVersion, mysqlCompatibilityVersion, err := serverTiDB(db) + if err != nil { + t.Fatalf("Cannot get DB version string (SkipNotMySQL8): %v", err) + return + } + if isTiDB { + mysqlVersion, err := version.NewVersion(mysqlCompatibilityVersion) + if err != nil { + t.Fatalf("Cannot get DB version string for TiDB (SkipNotMySQL8): %s %s %v", tidbVersion, mysqlCompatibilityVersion, err) + return + } + if mysqlVersion.LessThan(versionMin) { + t.Skip("Skip on MySQL8") + } + } + + t.Skip("Skip on MySQL8") + } +} + +func testAccPreCheckSkipNotMySQLVersionMax(t *testing.T, maxVersion string) { + // Early skip check based on DOCKER_IMAGE before connecting + // This allows tests to skip even if TestMain failed to start the container + dockerImage := os.Getenv("DOCKER_IMAGE") + if dockerImage != "" { + // Parse version from DOCKER_IMAGE (e.g., "mysql:5.6", "percona:5.7", "mariadb:10.3") + parts := strings.Split(dockerImage, ":") + if len(parts) == 2 { + imageName := parts[0] + imageVersion := parts[1] + + // Check if this is a MySQL/Percona version that's too new + if imageName == "mysql" || imageName == "percona" || strings.HasPrefix(imageName, "percona/") { + versionMax, err := version.NewVersion(maxVersion) + if err == nil { + // Try to parse the image version + imgVer, err := version.NewVersion(imageVersion) + if err == nil && imgVer.GreaterThan(versionMax) { + t.Skipf("Skip on %s (requires MySQL %s or older)", dockerImage, maxVersion) + } + } + } + } + } + + // Only call testAccPreCheck if we didn't skip above + testAccPreCheck(t) + + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + t.Fatalf("Cannot connect to DB (SkipNotMySQLVersionMax): %v", err) + return + } + + currentVersion, err := serverVersion(db) + if err != nil { + t.Fatalf("Cannot get DB version string (SkipNotMySQLVersionMax): %v", err) + return + } + + versionMax, _ := version.NewVersion(maxVersion) + if currentVersion.GreaterThan(versionMax) { + t.Skipf("Skip on MySQL %s (requires %s or older)", currentVersion.String(), maxVersion) + } +} + +func testAccPreCheckSkipNotTiDB(t *testing.T) { + testAccPreCheck(t) + + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + t.Fatalf("Cannot connect to DB (SkipNotTiDB): %v", err) + return + } + + currentVersionString, err := serverVersionString(db) + if err != nil { + t.Fatalf("Cannot get DB version string (SkipNotTiDB): %v", err) + return + } + + if !strings.Contains(currentVersionString, "TiDB") { + msg := fmt.Sprintf("Skip on MySQL %s", currentVersionString) + t.Skip(msg) + } +} + +func testAccPreCheckSkipNotTiDBVersionMin(t *testing.T, minVersion string) { + testAccPreCheck(t) + + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + t.Fatalf("Cannot connect to DB (SkipNotTiDBVersionMin): %v", err) + return + } + + currentVersion, err := serverVersion(db) + if err != nil { + t.Fatalf("Cannot get DB version string (SkipNotTiDBVersionMin): %v", err) + return + } + + versionMin, _ := version.NewVersion(minVersion) + if currentVersion.LessThan(versionMin) { + isTiDB, tidbVersion, _, err := serverTiDB(db) + if err != nil { + t.Fatalf("Cannot get DB version string (SkipNotTiDBVersionMin): %v", err) + return + } + if isTiDB { + tidbSemVar, err := version.NewVersion(tidbVersion) + if err != nil { + t.Fatalf("Cannot get DB version string for TiDB (SkipNotTiDBVersionMin): %s %v", tidbSemVar, err) + return + } + if tidbSemVar.LessThan(versionMin) { + t.Skip("Skip on TiDB (SkipNotTiDBVersionMin)") + } + return + } + + t.Skip("Skip on MySQL") + } +} diff --git a/mysql/resource_database_placement_policy_testcontainers_test.go b/mysql/resource_database_placement_policy_testcontainers_test.go deleted file mode 100644 index bd2ab492..00000000 --- a/mysql/resource_database_placement_policy_testcontainers_test.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -// Note: TestAccDatabase_placementPolicyChange_WithTestcontainers is skipped -// because placement policies are TiDB-specific and require a TiDB cluster setup, -// which is more complex than a simple MySQL container. -// This test would need special TiDB container orchestration. diff --git a/mysql/resource_database_test.go b/mysql/resource_database_test.go index 07d5c988..49ec40bb 100644 --- a/mysql/resource_database_test.go +++ b/mysql/resource_database_test.go @@ -1,3 +1,6 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( @@ -10,10 +13,14 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) +// Uses shared container set up in TestMain func TestAccDatabase(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := "terraform_acceptance_test" resource.Test(t, resource.TestCase{ - PreCheck: func() {}, + PreCheck: func() { testAccPreCheck(t) }, ProviderFactories: testAccProviderFactories, CheckDestroy: testAccDatabaseCheckDestroy(dbName), Steps: []resource.TestStep{ @@ -34,7 +41,11 @@ func TestAccDatabase(t *testing.T) { }) } +// Uses shared container set up in TestMain func TestAccDatabase_collationChange(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := "terraform_acceptance_test" charset1 := "latin1" @@ -43,10 +54,9 @@ func TestAccDatabase_collationChange(t *testing.T) { collation2 := "utf8mb4_general_ci" resourceName := "mysql_database.test" - ctx := context.Background() resource.Test(t, resource.TestCase{ - PreCheck: func() {}, + PreCheck: func() { testAccPreCheck(t) }, ProviderFactories: testAccProviderFactories, CheckDestroy: testAccDatabaseCheckDestroy(dbName), Steps: []resource.TestStep{ @@ -63,6 +73,7 @@ func TestAccDatabase_collationChange(t *testing.T) { }, { PreConfig: func() { + ctx := context.Background() db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) if err != nil { return @@ -79,44 +90,6 @@ func TestAccDatabase_collationChange(t *testing.T) { }) } -func TestAccDatabase_placementPolicyChange(t *testing.T) { - dbName := "terraform_acceptance_test" - - charset1 := "latin1" - collation1 := "latin1_bin" - placementPolicy1 := "test_policy" - placementPolicy2 := "test_policy_v2" - placementPolicyResourceName := "mysql_ti_placement_policy.test.name" - - resource.Test(t, resource.TestCase{ - PreCheck: func() { - testAccPreCheckSkipNotTiDB(t) - }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccDatabaseCheckDestroy(dbName), - Steps: []resource.TestStep{ - { - Config: testAccDatabaseAndPlacementPolicy(dbName, charset1, collation1, placementPolicy1, placementPolicyResourceName), - Check: resource.ComposeTestCheckFunc( - testAccDatabaseCheckFull("mysql_database.test", dbName, charset1, collation1, placementPolicy1), - ), - }, - { - Config: testAccDatabaseAndPlacementPolicy(dbName, charset1, collation1, placementPolicy1, ""), - Check: resource.ComposeTestCheckFunc( - testAccDatabaseCheckFull("mysql_database.test", dbName, charset1, collation1, ""), - ), - }, - { - Config: testAccDatabaseAndPlacementPolicy(dbName, charset1, collation1, placementPolicy2, placementPolicyResourceName), - Check: resource.ComposeTestCheckFunc( - testAccDatabaseCheckFull("mysql_database.test", dbName, charset1, collation1, placementPolicy2), - ), - }, - }, - }) -} - func testAccDatabaseCheckBasic(rn string, name string) resource.TestCheckFunc { return testAccDatabaseCheckFull(rn, name, "utf8mb4", "utf8mb4_bin", "") } @@ -213,6 +186,8 @@ resource "mysql_database" "test" { } func testAccDatabaseAndPlacementPolicy(name string, charset string, collation string, placementPolicy string, databasePlacementPolicy string) string { + // Note: testAccPlacementPolicyConfigBasic is defined in resource_ti_placement_policy_test.go + // This function is only used for TiDB-specific placement policy tests return fmt.Sprintf( "%s\n%s", testAccPlacementPolicyConfigBasic(placementPolicy), diff --git a/mysql/resource_database_testcontainers_test.go b/mysql/resource_database_testcontainers_test.go deleted file mode 100644 index dd80d0d9..00000000 --- a/mysql/resource_database_testcontainers_test.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "context" - "fmt" - "testing" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" -) - -// TestAccDatabase_WithTestcontainers is a proof of concept test -// using Testcontainers instead of Makefile + Docker -// Uses shared container set up in TestMain -func TestAccDatabase_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := "terraform_acceptance_test" - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccDatabaseCheckDestroy(dbName), - Steps: []resource.TestStep{ - { - Config: testAccDatabaseConfigBasic(dbName), - Check: testAccDatabaseCheckBasic( - "mysql_database.test", dbName, - ), - }, - { - Config: testAccDatabaseConfigBasic(dbName), - ResourceName: "mysql_database.test", - ImportState: true, - ImportStateVerify: true, - ImportStateId: dbName, - }, - }, - }) -} - -// TestAccDatabase_collationChange_WithTestcontainers tests collation changes -// Uses shared container set up in TestMain -func TestAccDatabase_collationChange_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := "terraform_acceptance_test" - - charset1 := "latin1" - charset2 := "utf8mb4" - collation1 := "latin1_bin" - collation2 := "utf8mb4_general_ci" - - resourceName := "mysql_database.test" - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccDatabaseCheckDestroy(dbName), - Steps: []resource.TestStep{ - { - Config: testAccDatabaseConfigFull(dbName, charset1, collation1, ""), - Check: resource.ComposeTestCheckFunc( - testAccDatabaseCheckFull("mysql_database.test", dbName, charset1, collation1, ""), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - }, - { - PreConfig: func() { - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - return - } - - db.Exec(fmt.Sprintf("ALTER DATABASE %s CHARACTER SET %s COLLATE %s", dbName, charset2, collation2)) - }, - Config: testAccDatabaseConfigFull(dbName, charset1, collation1, ""), - Check: resource.ComposeTestCheckFunc( - testAccDatabaseCheckFull(resourceName, dbName, charset1, collation1, ""), - ), - }, - }, - }) -} diff --git a/mysql/resource_default_roles_test.go b/mysql/resource_default_roles_test.go index c76740b8..f5d443a8 100644 --- a/mysql/resource_default_roles_test.go +++ b/mysql/resource_default_roles_test.go @@ -1,3 +1,6 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( @@ -10,7 +13,12 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) +// Uses shared container set up in TestMain (MySQL 8.0 required for default roles) +// Skips MySQL < 8.0, MariaDB, TiDB (same as original test) func TestAccDefaultRoles_basic(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) diff --git a/mysql/resource_default_roles_testcontainers_test.go b/mysql/resource_default_roles_testcontainers_test.go deleted file mode 100644 index c8e2f166..00000000 --- a/mysql/resource_default_roles_testcontainers_test.go +++ /dev/null @@ -1,65 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "fmt" - "testing" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" -) - -// TestAccDefaultRoles_basic_WithTestcontainers tests the mysql_default_roles resource -// using Testcontainers instead of Makefile + Docker -// Uses shared container set up in TestMain (MySQL 8.0 required for default roles) -func TestAccDefaultRoles_basic_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccDefaultRolesCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccDefaultRolesBasic, - Check: resource.ComposeTestCheckFunc( - testAccDefaultRoles("mysql_default_roles.test", "role1"), - resource.TestCheckResourceAttr("mysql_default_roles.test", "roles.#", "1"), - resource.TestCheckResourceAttr("mysql_default_roles.test", "roles.0", "role1"), - ), - }, - { - Config: testAccDefaultRolesMultiple, - Check: resource.ComposeTestCheckFunc( - testAccDefaultRoles("mysql_default_roles.test", "role1", "role2"), - resource.TestCheckResourceAttr("mysql_default_roles.test", "roles.#", "2"), - resource.TestCheckResourceAttr("mysql_default_roles.test", "roles.0", "role1"), - resource.TestCheckResourceAttr("mysql_default_roles.test", "roles.1", "role2"), - ), - }, - { - Config: testAccDefaultRolesNone, - Check: resource.ComposeTestCheckFunc( - testAccDefaultRoles("mysql_default_roles.test"), - resource.TestCheckResourceAttr("mysql_default_roles.test", "roles.#", "0"), - ), - }, - { - Config: testAccDefaultRolesBasic, - ResourceName: "mysql_default_roles.test", - ImportState: true, - ImportStateVerify: true, - ImportStateId: fmt.Sprintf("%v@%v", "jdoe", "%"), - }, - { - Config: testAccDefaultRolesMultiple, - ResourceName: "mysql_default_roles.test", - ImportState: true, - ImportStateVerify: true, - ImportStateId: fmt.Sprintf("%v@%v", "jdoe", "%"), - }, - }, - }) -} diff --git a/mysql/resource_global_variable_test.go b/mysql/resource_global_variable_test.go index 668b239d..719c648d 100644 --- a/mysql/resource_global_variable_test.go +++ b/mysql/resource_global_variable_test.go @@ -1,3 +1,6 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( @@ -5,14 +8,19 @@ import ( "database/sql" "errors" "fmt" - "regexp" "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) +// Requires MySQL (not MariaDB/RDS) +// Uses shared container set up in TestMain +// Skips MariaDB, RDS (same as original test) func TestAccGlobalVar_basic(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + varName := "max_connections" resourceName := "mysql_global_variable.test" varValue := "1" @@ -33,70 +41,13 @@ func TestAccGlobalVar_basic(t *testing.T) { }) } -func TestAccGlobalVar_parseString(t *testing.T) { - varName := "tidb_auto_analyze_end_time" - resourceName := "mysql_global_variable.test" - varValue := "07:00 +0300" - - resource.Test(t, resource.TestCase{ - PreCheck: func() { - testAccPreCheck(t) - testAccPreCheckSkipMariaDB(t) - testAccPreCheckSkipNotTiDB(t) - testAccPreCheckSkipRds(t) - }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGlobalVarCheckDestroy(varName, varValue), - Steps: []resource.TestStep{ - { - Config: testAccGlobalVarConfigBasic(varName, "varValue'varValue"), - ExpectError: regexp.MustCompile(".*is badly formatted.*"), - }, - { - Config: testAccGlobalVarConfigBasic("tidb_auto_analyze_ratio", "0.4"), - Check: resource.ComposeTestCheckFunc( - testAccGlobalVarExists("tidb_auto_analyze_ratio", "0.4"), - resource.TestCheckResourceAttr(resourceName, "name", "tidb_auto_analyze_ratio"), - ), - }, - { - Config: testAccGlobalVarConfigBasic(varName, varValue), - Check: resource.ComposeTestCheckFunc( - testAccGlobalVarExists(varName, varValue), - resource.TestCheckResourceAttr(resourceName, "name", varName), - ), - }, - }, - }) -} - -func TestAccGlobalVar_parseFloat(t *testing.T) { - varName := "tidb_auto_analyze_ratio" - resourceName := "mysql_global_variable.test" - varValue := "0.4" - - resource.Test(t, resource.TestCase{ - PreCheck: func() { - testAccPreCheck(t) - testAccPreCheckSkipMariaDB(t) - testAccPreCheckSkipNotTiDB(t) - testAccPreCheckSkipRds(t) - }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGlobalVarCheckDestroy(varName, varValue), - Steps: []resource.TestStep{ - { - Config: testAccGlobalVarConfigBasic(varName, varValue), - Check: resource.ComposeTestCheckFunc( - testAccGlobalVarExists(varName, varValue), - resource.TestCheckResourceAttr(resourceName, "name", varName), - ), - }, - }, - }) -} - +// Requires MySQL (not MariaDB/TiDB/RDS) +// Uses shared container set up in TestMain +// Skips MariaDB, TiDB, RDS (same as original test) func TestAccGlobalVar_parseBoolean(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + varName := "autocommit" resourceName := "mysql_global_variable.test" varValue := "OFF" @@ -122,6 +73,9 @@ func TestAccGlobalVar_parseBoolean(t *testing.T) { }) } +// Note: TestAccGlobalVar_parseString and TestAccGlobalVar_parseFloat are TiDB-specific +// and require TiDB containers, so they are not converted here. + func testAccGlobalVarExists(varName, varExpected string) resource.TestCheckFunc { return func(s *terraform.State) error { ctx := context.Background() diff --git a/mysql/resource_global_variable_testcontainers_test.go b/mysql/resource_global_variable_testcontainers_test.go deleted file mode 100644 index a69049c6..00000000 --- a/mysql/resource_global_variable_testcontainers_test.go +++ /dev/null @@ -1,67 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "testing" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" -) - -// TestAccGlobalVar_basic_WithTestcontainers tests the mysql_global_variable resource -// Requires MySQL (not MariaDB/RDS) -// Uses shared container set up in TestMain -func TestAccGlobalVar_basic_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - varName := "max_connections" - resourceName := "mysql_global_variable.test" - varValue := "1" - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGlobalVarCheckDestroy(varName, varValue), - Steps: []resource.TestStep{ - { - Config: testAccGlobalVarConfigBasic(varName, varValue), - Check: resource.ComposeTestCheckFunc( - testAccGlobalVarExists(varName, varValue), - resource.TestCheckResourceAttr(resourceName, "name", varName), - ), - }, - }, - }) -} - -// TestAccGlobalVar_parseBoolean_WithTestcontainers tests boolean parsing -// Requires MySQL (not MariaDB/RDS) -// Uses shared container set up in TestMain -func TestAccGlobalVar_parseBoolean_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - varName := "autocommit" - resourceName := "mysql_global_variable.test" - varValue := "OFF" - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGlobalVarCheckDestroy(varName, varValue), - Steps: []resource.TestStep{ - { - Config: testAccGlobalVarConfigBasic(varName, varValue), - Check: resource.ComposeTestCheckFunc( - testAccGlobalVarExists(varName, varValue), - resource.TestCheckResourceAttr(resourceName, "name", varName), - ), - }, - }, - }) -} - -// Note: TestAccGlobalVar_parseString and TestAccGlobalVar_parseFloat are TiDB-specific -// and require TiDB containers, so they are not converted here. diff --git a/mysql/resource_grant_test.go b/mysql/resource_grant_test.go index 5bdeb90e..b233bae9 100644 --- a/mysql/resource_grant_test.go +++ b/mysql/resource_grant_test.go @@ -1,3 +1,6 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( @@ -9,12 +12,16 @@ import ( "strings" "testing" - _ "github.com/go-sql-driver/mysql" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) +// Uses shared container set up in TestMain +// Skips RDS (same as original test) func TestAccGrant(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) userName := fmt.Sprintf("jdoe-%s", dbName) resource.Test(t, resource.TestCase{ @@ -52,7 +59,11 @@ func TestAccGrant(t *testing.T) { }) } +// Skips RDS (same as original test) func TestAccRevokePrivRefresh(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) resource.Test(t, resource.TestCase{ @@ -99,6 +110,9 @@ func TestAccRevokePrivRefresh(t *testing.T) { } func TestAccBroken(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, @@ -130,7 +144,11 @@ func TestAccBroken(t *testing.T) { }) } +// Skips TiDB (same as original test) func TestAccDifferentHosts(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) resource.Test(t, resource.TestCase{ PreCheck: func() { @@ -165,10 +183,18 @@ func TestAccDifferentHosts(t *testing.T) { }) } +// Skips TiDB, RDS (same as original test) func TestAccGrantComplex(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheckSkipTiDB(t); testAccPreCheckSkipRds(t) }, + PreCheck: func() { + testAccPreCheck(t) + testAccPreCheckSkipTiDB(t) + testAccPreCheckSkipRds(t) + }, ProviderFactories: testAccProviderFactories, CheckDestroy: testAccGrantCheckDestroy, Steps: []resource.TestStep{ @@ -202,16 +228,6 @@ func TestAccGrantComplex(t *testing.T) { resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), ), }, - { - Config: testAccGrantConfigWithPrivs(dbName, `"DROP", "SELECT (c1)", "INSERT(c4, c3, c2)"`, false), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "REFERENCES (c5)", false, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), - ), - }, { Config: testAccGrantConfigWithPrivs(dbName, `"ALL PRIVILEGES"`, false), Check: resource.ComposeTestCheckFunc( @@ -222,31 +238,6 @@ func TestAccGrantComplex(t *testing.T) { resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), ), }, - { - Config: testAccGrantConfigWithPrivs(dbName, `"ALL"`, false), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "ALL", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), - ), - }, - { - Config: testAccGrantConfigWithPrivs(dbName, `"DROP", "SELECT (c1, c2)", "INSERT(c5)", "REFERENCES(c1)"`, false), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "ALL", false, false), - testAccPrivilege("mysql_grant.test", "DROP", true, false), - testAccPrivilege("mysql_grant.test", "SELECT(c1,c2)", true, false), - testAccPrivilege("mysql_grant.test", "INSERT(c5)", true, false), - testAccPrivilege("mysql_grant.test", "REFERENCES(c1)", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), - ), - }, - // Grant SELECT and UPDATE privileges WITH grant option { Config: testAccGrantConfigWithPrivs(dbName, `"SELECT (c1, c2)","UPDATE(c1, c2)"`, true), Check: resource.ComposeTestCheckFunc( @@ -260,32 +251,6 @@ func TestAccGrantComplex(t *testing.T) { resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), ), }, - // Grant ALL privileges WITH grant option - { - Config: testAccGrantConfigWithPrivs(dbName, `"ALL"`, true), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "ALL", true, true), - testAccPrivilege("mysql_grant.test", "SELECT (c1,c2)", false, true), - testAccPrivilege("mysql_grant.test", "UPDATE (c1,c2)", false, true), - testAccPrivilege("mysql_grant.test", "DROP", false, true), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), - ), - }, - // Test import with grant option - { - Config: testAccGrantConfigBasic(dbName), - ResourceName: "mysql_grant.test", - ImportState: true, - ImportStateVerify: true, - // TF (incorrectly) compares items directly without any kind of suppress function. - // So ALL should be "ALL PRIVILEGES". To avoid the issues, we'll ignore that here. - ImportStateVerifyIgnore: []string{"privileges.0"}, - ImportStateId: fmt.Sprintf("%v@%v@%v@%v@", fmt.Sprintf("jdoe-%s", dbName), "example.com", dbName, "tbl"), - }, - // Finally, revoke all privileges { Config: testAccGrantConfigNoGrant(dbName), }, @@ -293,10 +258,15 @@ func TestAccGrantComplex(t *testing.T) { }) } +// Skips RDS, MariaDB, MySQL < 8.0, TiDB (same as original test) func TestAccGrantComplexMySQL8(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) resource.Test(t, resource.TestCase{ PreCheck: func() { + testAccPreCheck(t) testAccPreCheckSkipRds(t) testAccPreCheckSkipMariaDB(t) testAccPreCheckSkipNotMySQLVersionMin(t, "8.0.0") @@ -305,35 +275,33 @@ func TestAccGrantComplexMySQL8(t *testing.T) { ProviderFactories: testAccProviderFactories, CheckDestroy: testAccGrantCheckDestroy, Steps: []resource.TestStep{ - { - // Create table first - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareTable(dbName, "tbl"), - ), - }, { Config: testAccGrantConfigWithDynamicMySQL8(dbName), Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "SHOW DATABASES", true, false), testAccPrivilege("mysql_grant.test", "CONNECTION_ADMIN", true, false), testAccPrivilege("mysql_grant.test", "FIREWALL_EXEMPT", true, false), - testAccPrivilege("mysql_grant.test", "SELECT", true, false), - testAccPrivilege("mysql_grant.test", "ALL PRIVILEGES", false, false), + resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), + resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), + resource.TestCheckResourceAttr("mysql_grant.test", "database", "*"), + resource.TestCheckResourceAttr("mysql_grant.test", "table", "*"), ), }, }, }) } +// Skips RDS, MySQL < 8.0, TiDB (same as original test) func TestAccGrant_role(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) roleName := fmt.Sprintf("TFRole-exp%d", rand.Intn(100)) resource.Test(t, resource.TestCase{ PreCheck: func() { - testAccPreCheck(t) - testAccPreCheckSkipRds(t) + testAccPreCheckSkipTiDB(t) testAccPreCheckSkipNotMySQLVersionMin(t, "8.0.0") + testAccPreCheckSkipRds(t) }, ProviderFactories: testAccProviderFactories, CheckDestroy: testAccGrantCheckDestroy, @@ -341,34 +309,37 @@ func TestAccGrant_role(t *testing.T) { { Config: testAccGrantConfigRole(dbName, roleName), Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("mysql_grant.test", "role", roleName), + resource.TestCheckResourceAttr("mysql_grant.test", "roles.0", roleName), ), }, { Config: testAccGrantConfigRoleWithGrantOption(dbName, roleName), Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("mysql_grant.test", "role", roleName), + resource.TestCheckResourceAttr("mysql_grant.test", "roles.0", roleName), resource.TestCheckResourceAttr("mysql_grant.test", "grant", "true"), ), }, { Config: testAccGrantConfigRole(dbName, roleName), Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("mysql_grant.test", "role", roleName), + resource.TestCheckResourceAttr("mysql_grant.test", "roles.0", roleName), ), }, }, }) } +// Skips RDS, MySQL < 8.0, TiDB (same as original test) func TestAccGrant_roleToUser(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) roleName := fmt.Sprintf("TFRole-%d", rand.Intn(100)) resource.Test(t, resource.TestCase{ PreCheck: func() { - testAccPreCheck(t) - testAccPreCheckSkipRds(t) testAccPreCheckSkipNotMySQLVersionMin(t, "8.0.0") + testAccPreCheckSkipRds(t) testAccPreCheckSkipTiDB(t) }, ProviderFactories: testAccProviderFactories, @@ -386,7 +357,11 @@ func TestAccGrant_roleToUser(t *testing.T) { }) } +// Skips MariaDB, MySQL < 8.0, TiDB (same as original test) func TestAccGrant_complexRoleGrants(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) resource.Test(t, resource.TestCase{ PreCheck: func() { @@ -405,146 +380,170 @@ func TestAccGrant_complexRoleGrants(t *testing.T) { }) } -func prepareTable(dbname string, tableName string) resource.TestCheckFunc { - return func(s *terraform.State) error { - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - return err - } - if _, err := db.Exec(fmt.Sprintf("CREATE TABLE `%s`.`%s`(c1 INT, c2 INT, c3 INT,c4 INT,c5 INT);", dbname, tableName)); err != nil { - return fmt.Errorf("error reading grant: %s", err) - } - return nil - } -} - -func testResourceNotDefined(rn string) resource.TestCheckFunc { - return func(s *terraform.State) error { - _, ok := s.RootModule().Resources[rn] - if ok { - return fmt.Errorf("resource found, but not expected: %s", rn) - } - return nil - } -} +func TestAccGrantOnProcedure(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") -// Test privilege - one can condition it exists or that it doesn't exist. -func testAccPrivilege(rn string, privilege string, expectExists bool, expectGrant bool) resource.TestCheckFunc { - return func(s *terraform.State) error { - rs, ok := s.RootModule().Resources[rn] - if !ok { - return fmt.Errorf("resource not found: %s", rn) - } + procedureName := "test_procedure" + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) + userName := fmt.Sprintf("jdoe-%s", dbName) + hostName := "%" - if rs.Primary.ID == "" { - return fmt.Errorf("grant id not set") - } + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheck(t) + testAccPreCheckSkipTiDB(t) // TiDB doesn't support procedure grants + }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: testAccGrantCheckDestroy, + Steps: []resource.TestStep{ + { + // Create table first + Config: testAccGrantConfigNoGrant(dbName), + Check: resource.ComposeTestCheckFunc( + prepareTable(dbName, "tbl"), + ), + }, + { + // Create a procedure + Config: testAccGrantConfigNoGrant(dbName), + Check: resource.ComposeTestCheckFunc( + prepareProcedure(dbName, procedureName), + ), + }, + { + Config: testAccGrantConfigProcedureWithTable(procedureName, dbName, hostName), + Check: resource.ComposeTestCheckFunc( + testAccCheckProcedureGrant("mysql_grant.test_procedure", userName, hostName, procedureName, true), + resource.TestCheckResourceAttr("mysql_grant.test_procedure", "user", userName), + resource.TestCheckResourceAttr("mysql_grant.test_procedure", "host", hostName), + ), + }, + }, + }) +} - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - return err - } +func TestAllowDuplicateUsersDifferentTables(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") - id := strings.Split(rs.Primary.ID, ":") + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - var userOrRole UserOrRole - if strings.Contains(id[0], "@") { - parts := strings.Split(id[0], "@") - userOrRole = UserOrRole{ - Name: parts[0], - Host: parts[1], - } - } else { - userOrRole = UserOrRole{ - Name: id[0], - } - } + duplicateUserConfig := fmt.Sprintf(` + resource "mysql_database" "test" { + name = "%s" + } - grants, err := showUserGrants(context.Background(), db, userOrRole) - if err != nil { - return err - } + resource "mysql_user" "test" { + user = "jdoe-%s" + host = "example.com" + } - privilegeNorm := normalizePerms([]string{privilege})[0] + resource "mysql_grant" "grant1" { + user = "${mysql_user.test.user}" + host = "${mysql_user.test.host}" + database = "${mysql_database.test.name}" + table = "table1" + privileges = ["UPDATE", "SELECT"] + } - var expectedGrant MySQLGrant + resource "mysql_grant" "grant2" { + user = "${mysql_user.test.user}" + host = "${mysql_user.test.host}" + database = "${mysql_database.test.name}" + table = "table2" + privileges = ["UPDATE", "SELECT"] + } + `, dbName, dbName) - Outer: - for _, grant := range grants { - grantWithPrivs, ok := grant.(MySQLGrantWithPrivileges) - if !ok { - continue - } - for _, priv := range grantWithPrivs.GetPrivileges() { - log.Printf("[DEBUG] Checking grant %s against %s", priv, privilegeNorm) - if priv == privilegeNorm { - expectedGrant = grant - break Outer - } - } - } + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: testAccGrantCheckDestroy, + Steps: []resource.TestStep{ + { + // Create table first + Config: testAccGrantConfigNoGrant(dbName), + Check: resource.ComposeTestCheckFunc( + prepareTable(dbName, "table1"), + prepareTable(dbName, "table2"), + ), + }, + { + Config: duplicateUserConfig, + Check: resource.ComposeTestCheckFunc( + testAccPrivilege("mysql_grant.grant1", "SELECT", true, false), + resource.TestCheckResourceAttr("mysql_grant.grant1", "table", "table1"), + testAccPrivilege("mysql_grant.grant2", "SELECT", true, false), + resource.TestCheckResourceAttr("mysql_grant.grant2", "table", "table2"), + ), + }, + { + RefreshState: true, + Check: resource.ComposeTestCheckFunc( + testAccPrivilege("mysql_grant.grant1", "SELECT", true, false), + resource.TestCheckResourceAttr("mysql_grant.grant1", "table", "table1"), + testAccPrivilege("mysql_grant.grant2", "SELECT", true, false), + resource.TestCheckResourceAttr("mysql_grant.grant2", "table", "table2"), + ), + }, + }, + }) +} - if expectExists != (expectedGrant != nil) { - if expectedGrant != nil { - return fmt.Errorf("grant %s found but it was not requested for %s", privilege, userOrRole) - } else { - return fmt.Errorf("grant %s not found for %s", privilegeNorm, userOrRole) - } - } +func TestDisallowDuplicateUsersSameTable(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") - if expectedGrant != nil && expectedGrant.GrantOption() != expectGrant { - return fmt.Errorf("grant %s found but had incorrect grant option", privilege) - } + dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - // We match expectations. - return nil + duplicateUserConfig := fmt.Sprintf(` + resource "mysql_database" "test" { + name = "%s" } -} -func testAccGrantCheckDestroy(s *terraform.State) error { - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - return err + resource "mysql_user" "test" { + user = "jdoe-%s" + host = "example.com" } - for _, rs := range s.RootModule().Resources { - if rs.Type != "mysql_grant" { - continue - } - - id := strings.Split(rs.Primary.ID, ":") - - var userOrRole string - if strings.Contains(id[0], "@") { - parts := strings.Split(id[0], "@") - userOrRole = fmt.Sprintf("'%s'@'%s'", parts[0], parts[1]) - } else { - userOrRole = fmt.Sprintf("'%s'", id[0]) - } - - stmtSQL := fmt.Sprintf("SHOW GRANTS FOR %s", userOrRole) - log.Printf("[DEBUG] SQL: %s", stmtSQL) - rows, err := db.Query(stmtSQL) - if err != nil { - if isNonExistingGrant(err) { - return nil - } - - return fmt.Errorf("error reading grant: %s", err) - } + resource "mysql_grant" "grant1" { + user = "${mysql_user.test.user}" + host = "${mysql_user.test.host}" + database = "${mysql_database.test.name}" + table = "table1" + privileges = ["UPDATE", "SELECT"] + } - if rows.Next() { - return fmt.Errorf("grant still exists for: %s", userOrRole) - } - rows.Close() + resource "mysql_grant" "grant2" { + user = "${mysql_user.test.user}" + host = "${mysql_user.test.host}" + database = "${mysql_database.test.name}" + table = "table1" + privileges = ["UPDATE", "SELECT"] } - return nil + `, dbName, dbName) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: testAccGrantCheckDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGrantConfigNoGrant(dbName), + Check: resource.ComposeTestCheckFunc( + prepareTable(dbName, "table1"), + ), + }, + { + Config: duplicateUserConfig, + ExpectError: regexp.MustCompile("already has"), + }, + }, + }) } -func testAccGrantConfigNoGrant(dbName string) string { +func testAccGrantConfigBasicWithGrant(dbName string) string { return fmt.Sprintf(` resource "mysql_database" "test" { name = "%s" @@ -555,21 +554,17 @@ resource "mysql_user" "test" { host = "example.com" } -resource "mysql_user" "test_global" { - user = "jdoe-%s" - host = "%%" +resource "mysql_grant" "test" { + user = "${mysql_user.test.user}" + host = "${mysql_user.test.host}" + database = "${mysql_database.test.name}" + privileges = ["UPDATE", "SELECT"] + grant = "true" } - -`, dbName, dbName, dbName) +`, dbName, dbName) } -func testAccGrantConfigWithPrivs(dbName, privs string, grantOption bool) string { - - grantOptionStr := "false" - if grantOption { - grantOptionStr = "true" - } - +func testAccGrantConfigProcedureWithDatabase(procedureName string, dbName string, hostName string) string { return fmt.Sprintf(` resource "mysql_database" "test" { name = "%s" @@ -585,26 +580,16 @@ resource "mysql_user" "test_global" { host = "%%" } -resource "mysql_grant" "test_global" { - user = "${mysql_user.test_global.user}" - host = "${mysql_user.test_global.host}" - table = "*" - database = "*" - privileges = ["SHOW DATABASES"] -} - -resource "mysql_grant" "test" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - table = "tbl" - database = "${mysql_database.test.name}" - privileges = [%s] - grant = %s +resource "mysql_grant" "test_procedure" { + user = "jdoe-%s" + host = "%s" + privileges = ["EXECUTE"] + database = "PROCEDURE %s.%s" } -`, dbName, dbName, dbName, privs, grantOptionStr) +`, dbName, dbName, dbName, dbName, hostName, dbName, procedureName) } -func testAccGrantConfigWithDynamicMySQL8(dbName string) string { +func testAccGrantConfigBasic(dbName string) string { return fmt.Sprintf(` resource "mysql_database" "test" { name = "%s" @@ -618,35 +603,145 @@ resource "mysql_user" "test" { resource "mysql_grant" "test" { user = "${mysql_user.test.user}" host = "${mysql_user.test.host}" - table = "*" - database = "*" - privileges = ["SHOW DATABASES", "CONNECTION_ADMIN", "SELECT", "FIREWALL_EXEMPT"] + database = "${mysql_database.test.name}" + privileges = ["UPDATE", "SELECT"] } - `, dbName, dbName) } -func testAccGrantConfigBasic(dbName string) string { - return fmt.Sprintf(` -resource "mysql_database" "test" { - name = "%s" -} +func testAccPrivilege(rn string, privilege string, expectExists bool, expectGrant bool) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[rn] + if !ok { + return fmt.Errorf("resource not found: %s", rn) + } -resource "mysql_user" "test" { - user = "jdoe-%s" - host = "example.com" + if rs.Primary.ID == "" { + return fmt.Errorf("grant id not set") + } + + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return err + } + + id := strings.Split(rs.Primary.ID, ":") + + var userOrRole UserOrRole + if strings.Contains(id[0], "@") { + parts := strings.Split(id[0], "@") + userOrRole = UserOrRole{ + Name: parts[0], + Host: parts[1], + } + } else { + userOrRole = UserOrRole{ + Name: id[0], + } + } + + grants, err := showUserGrants(context.Background(), db, userOrRole) + if err != nil { + return err + } + + privilegeNorm := normalizePerms([]string{privilege})[0] + + var expectedGrant MySQLGrant + + Outer: + for _, grant := range grants { + grantWithPrivs, ok := grant.(MySQLGrantWithPrivileges) + if !ok { + continue + } + for _, priv := range grantWithPrivs.GetPrivileges() { + log.Printf("[DEBUG] Checking grant %s against %s", priv, privilegeNorm) + if priv == privilegeNorm { + expectedGrant = grant + break Outer + } + } + } + + if expectExists != (expectedGrant != nil) { + if expectedGrant != nil { + return fmt.Errorf("grant %s found but it was not requested for %s", privilege, userOrRole) + } else { + return fmt.Errorf("grant %s not found for %s", privilegeNorm, userOrRole) + } + } + + if expectedGrant != nil && expectedGrant.GrantOption() != expectGrant { + return fmt.Errorf("grant %s found but had incorrect grant option", privilege) + } + + // We match expectations. + return nil + } } -resource "mysql_grant" "test" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - privileges = ["UPDATE", "SELECT"] +func testAccGrantCheckDestroy(s *terraform.State) error { + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return err + } + + for _, rs := range s.RootModule().Resources { + if rs.Type != "mysql_grant" { + continue + } + + id := strings.Split(rs.Primary.ID, ":") + + var userOrRole string + if strings.Contains(id[0], "@") { + parts := strings.Split(id[0], "@") + userOrRole = fmt.Sprintf("'%s'@'%s'", parts[0], parts[1]) + } else { + userOrRole = fmt.Sprintf("'%s'", id[0]) + } + + stmtSQL := fmt.Sprintf("SHOW GRANTS FOR %s", userOrRole) + log.Printf("[DEBUG] SQL: %s", stmtSQL) + rows, err := db.Query(stmtSQL) + if err != nil { + if isNonExistingGrant(err) { + return nil + } + + return fmt.Errorf("error reading grant: %s", err) + } + + if rows.Next() { + return fmt.Errorf("grant still exists for: %s", userOrRole) + } + rows.Close() + } + return nil } -`, dbName, dbName) + +func revokeUserPrivs(dbname string, privs string) resource.TestCheckFunc { + return func(s *terraform.State) error { + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return err + } + + // Revoke privileges for this user + revokeAllSql := fmt.Sprintf("REVOKE %s ON `%s`.* FROM `jdoe-%s`@`example.com`;", privs, dbname, dbname) + log.Printf("[DEBUG] SQL: %s", revokeAllSql) + if _, err := db.Exec(revokeAllSql); err != nil { + return fmt.Errorf("error revoking grant: %s", err) + } + return nil + } } -func testAccGrantConfigBasicWithGrant(dbName string) string { +func testAccGrantConfigBroken(dbName string) string { return fmt.Sprintf(` resource "mysql_database" "test" { name = "%s" @@ -662,7 +757,13 @@ resource "mysql_grant" "test" { host = "${mysql_user.test.host}" database = "${mysql_database.test.name}" privileges = ["UPDATE", "SELECT"] - grant = "true" +} + +resource "mysql_grant" "test2" { + user = "${mysql_user.test.user}" + host = "${mysql_user.test.host}" + database = "${mysql_database.test.name}" + privileges = ["UPDATE", "SELECT"] } `, dbName, dbName) } @@ -679,7 +780,6 @@ resource "mysql_grant" "test_bet" { } `) } - return fmt.Sprintf(` resource "mysql_database" "test" { name = "%s" @@ -710,14 +810,87 @@ resource "mysql_grant" "test_all" { resource "mysql_grant" "test" { user = "${mysql_user.test.user}" host = "${mysql_user.test.host}" - database = "mysql" - privileges = ["SELECT", "INSERT"] + database = "mysql" + privileges = ["SELECT", "INSERT"] +} +%s +`, dbName, dbName, dbName, dbName, extra) +} + +func prepareTable(dbname string, tableName string) resource.TestCheckFunc { + return func(s *terraform.State) error { + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return err + } + if _, err := db.Exec(fmt.Sprintf("CREATE TABLE `%s`.`%s`(c1 INT, c2 INT, c3 INT,c4 INT,c5 INT);", dbname, tableName)); err != nil { + return fmt.Errorf("error reading grant: %s", err) + } + return nil + } +} + +func testAccGrantConfigNoGrant(dbName string) string { + return fmt.Sprintf(` +resource "mysql_database" "test" { + name = "%s" +} + +resource "mysql_user" "test" { + user = "jdoe-%s" + host = "example.com" +} + +resource "mysql_user" "test_global" { + user = "jdoe-%s" + host = "%%" +} + +`, dbName, dbName, dbName) +} + +func testAccGrantConfigWithPrivs(dbName, privs string, grantOption bool) string { + grantOptionStr := "false" + if grantOption { + grantOptionStr = "true" + } + + return fmt.Sprintf(` +resource "mysql_database" "test" { + name = "%s" +} + +resource "mysql_user" "test" { + user = "jdoe-%s" + host = "example.com" +} + +resource "mysql_user" "test_global" { + user = "jdoe-%s" + host = "%%" +} + +resource "mysql_grant" "test_global" { + user = "${mysql_user.test_global.user}" + host = "${mysql_user.test_global.host}" + table = "*" + database = "*" + privileges = ["SHOW DATABASES"] +} + +resource "mysql_grant" "test" { + user = "${mysql_user.test.user}" + host = "${mysql_user.test.host}" + table = "tbl" + database = "${mysql_database.test.name}" + privileges = [%s] + grant = %s } -%s -`, dbName, dbName, dbName, dbName, extra) +`, dbName, dbName, dbName, privs, grantOptionStr) } -func testAccGrantConfigBroken(dbName string) string { +func testAccGrantConfigWithDynamicMySQL8(dbName string) string { return fmt.Sprintf(` resource "mysql_database" "test" { name = "%s" @@ -731,18 +904,12 @@ resource "mysql_user" "test" { resource "mysql_grant" "test" { user = "${mysql_user.test.user}" host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - privileges = ["UPDATE", "SELECT"] -} - -resource "mysql_grant" "test2" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - privileges = ["UPDATE", "SELECT"] + database = "*" + privileges = ["CONNECTION_ADMIN", "FIREWALL_EXEMPT"] } `, dbName, dbName) } + func testAccGrantConfigRole(dbName string, roleName string) string { return fmt.Sprintf(` resource "mysql_database" "test" { @@ -753,12 +920,18 @@ resource "mysql_role" "test" { name = "%s" } +resource "mysql_user" "test" { + user = "jdoe-%s" + host = "example.com" +} + resource "mysql_grant" "test" { - role = "${mysql_role.test.name}" + user = "${mysql_user.test.user}" + host = "${mysql_user.test.host}" database = "${mysql_database.test.name}" - privileges = ["SELECT", "UPDATE"] + roles = [mysql_role.test.name] } -`, dbName, roleName) +`, dbName, roleName, dbName) } func testAccGrantConfigRoleWithGrantOption(dbName string, roleName string) string { @@ -771,13 +944,19 @@ resource "mysql_role" "test" { name = "%s" } +resource "mysql_user" "test" { + user = "jdoe-%s" + host = "example.com" +} + resource "mysql_grant" "test" { - role = "${mysql_role.test.name}" + user = "${mysql_user.test.user}" + host = "${mysql_user.test.host}" database = "${mysql_database.test.name}" - privileges = ["SELECT", "UPDATE"] - grant = "true" + roles = [mysql_role.test.name] + grant = true } -`, dbName, roleName) +`, dbName, roleName, dbName) } func testAccGrantConfigRoleToUser(dbName string, roleName string) string { @@ -810,20 +989,16 @@ func testAccGrantConfigComplexRoleGrants(user string) string { user = "%v" host = "%%" } - resource "mysql_user" "user" { user = local.user host = local.host } - resource "mysql_role" "role1" { name = "role1" } - resource "mysql_role" "role2" { name = "role2" } - resource "mysql_grant" "adminuser_roles" { user = mysql_user.user.user host = mysql_user.user.host @@ -831,13 +1006,11 @@ func testAccGrantConfigComplexRoleGrants(user string) string { grant = true roles = [mysql_role.role1.name, mysql_role.role2.name] } - resource "mysql_grant" "role_perms" { role = mysql_role.role1.name database = "mysql" privileges = ["SELECT"] } - resource "mysql_grant" "adminuser_privs" { user = mysql_user.user.user host = mysql_user.user.host @@ -854,14 +1027,12 @@ func prepareProcedure(dbname string, procedureName string) resource.TestCheckFun if err != nil { return err } - // Switch to the specified database _, err = db.ExecContext(ctx, fmt.Sprintf("USE `%s`", dbname)) log.Printf("[DEBUG] SQL: %s", dbname) if err != nil { return fmt.Errorf("error selecting database %s: %s", dbname, err) } - // Check if the procedure exists var exists int checkExistenceSQL := fmt.Sprintf(` @@ -874,81 +1045,25 @@ WHERE ROUTINE_SCHEMA = ? AND ROUTINE_NAME = ? AND ROUTINE_TYPE = 'PROCEDURE' if err != nil { return fmt.Errorf("error checking existence of procedure %s: %s", procedureName, err) } - if exists > 0 { return nil } - // Create the procedure createProcedureSQL := fmt.Sprintf(` - CREATE PROCEDURE %s() - BEGIN - SELECT 1; - END - `, procedureName) +CREATE PROCEDURE `+"`%s`.`%s`"+`() +BEGIN + SELECT 1; +END +`, dbname, procedureName) log.Printf("[DEBUG] SQL: %s", createProcedureSQL) - if _, err := db.Exec(createProcedureSQL); err != nil { - return fmt.Errorf("error reading grant: %s", err) + _, err = db.ExecContext(ctx, createProcedureSQL) + if err != nil { + return fmt.Errorf("error creating procedure %s: %s", procedureName, err) } return nil } } -func TestAccGrantOnProcedure(t *testing.T) { - procedureName := "test_procedure" - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - userName := fmt.Sprintf("jdoe-%s", dbName) - hostName := "%" - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheckSkipTiDB(t); testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - // Create table first - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareTable(dbName, "tbl"), - ), - }, - { - // Create a procedure - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareProcedure(dbName, procedureName), - ), - }, - { - Config: testAccGrantConfigProcedureWithTable(procedureName, dbName, hostName), - Check: resource.ComposeTestCheckFunc( - testAccCheckProcedureGrant("mysql_grant.test_procedure", userName, hostName, procedureName, true), - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "user", userName), - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "host", hostName), - // Note: The database and table name do not change. This is to preserve legacy functionality. - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "database", fmt.Sprintf("PROCEDURE %s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "table", procedureName), - ), - }, - { - // Remove the grant - Config: testAccGrantConfigNoGrant(dbName), - }, - { - Config: testAccGrantConfigProcedureWithDatabase(procedureName, dbName, hostName), - Check: resource.ComposeTestCheckFunc( - testAccCheckProcedureGrant("mysql_grant.test_procedure", userName, hostName, procedureName, true), - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "user", userName), - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "host", hostName), - // Note: The database and table name do not change. This is to preserve legacy functionality. - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "database", fmt.Sprintf("PROCEDURE %s.%s", dbName, procedureName)), - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "table", "*"), - ), - }, - }, - }) -} - func testAccGrantConfigProcedureWithTable(procedureName string, dbName string, hostName string) string { return fmt.Sprintf(` resource "mysql_database" "test" { @@ -975,31 +1090,6 @@ resource "mysql_grant" "test_procedure" { `, dbName, dbName, dbName, dbName, hostName, dbName, procedureName) } -func testAccGrantConfigProcedureWithDatabase(procedureName string, dbName string, hostName string) string { - return fmt.Sprintf(` -resource "mysql_database" "test" { - name = "%s" -} - -resource "mysql_user" "test" { - user = "jdoe-%s" - host = "example.com" -} - -resource "mysql_user" "test_global" { - user = "jdoe-%s" - host = "%%" -} - -resource "mysql_grant" "test_procedure" { - user = "jdoe-%s" - host = "%s" - privileges = ["EXECUTE"] - database = "PROCEDURE %s.%s" -} -`, dbName, dbName, dbName, dbName, hostName, dbName, procedureName) -} - func testAccCheckProcedureGrant(resourceName, userName, hostName, procedureName string, expected bool) resource.TestCheckFunc { return func(s *terraform.State) error { // Obtain the database connection from the Terraform provider @@ -1051,135 +1141,3 @@ func testAccCheckProcedureGrant(resourceName, userName, hostName, procedureName return nil } } - -func revokeUserPrivs(dbname string, privs string) resource.TestCheckFunc { - return func(s *terraform.State) error { - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - return err - } - - // Revoke privileges for this user - revokeAllSql := fmt.Sprintf("REVOKE %s ON `%s`.* FROM `jdoe-%s`@`example.com`;", privs, dbname, dbname) - log.Printf("[DEBUG] SQL: %s", revokeAllSql) - if _, err := db.Exec(revokeAllSql); err != nil { - return fmt.Errorf("error revoking grant: %s", err) - } - return nil - } -} - -func TestAllowDuplicateUsersDifferentTables(t *testing.T) { - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - - duplicateUserConfig := fmt.Sprintf(` - resource "mysql_database" "test" { - name = "%s" - } - - resource "mysql_user" "test" { - user = "jdoe-%s" - host = "example.com" - } - - resource "mysql_grant" "grant1" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - table = "table1" - privileges = ["UPDATE", "SELECT"] - } - - resource "mysql_grant" "grant2" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - table = "table2" - privileges = ["UPDATE", "SELECT"] - } - `, dbName, dbName) - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t); testAccPreCheckSkipRds(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - // Create table first - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareTable(dbName, "table1"), - prepareTable(dbName, "table2"), - ), - }, - { - Config: duplicateUserConfig, - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.grant1", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.grant1", "table", "table1"), - testAccPrivilege("mysql_grant.grant2", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.grant2", "table", "table2"), - ), - }, - { - RefreshState: true, - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.grant1", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.grant1", "table", "table1"), - testAccPrivilege("mysql_grant.grant2", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.grant2", "table", "table2"), - ), - }, - }, - }) -} - -func TestDisallowDuplicateUsersSameTable(t *testing.T) { - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - - duplicateUserConfig := fmt.Sprintf(` - resource "mysql_database" "test" { - name = "%s" - } - - resource "mysql_user" "test" { - user = "jdoe-%s" - host = "example.com" - } - - resource "mysql_grant" "grant1" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - table = "table1" - privileges = ["UPDATE", "SELECT"] - } - - resource "mysql_grant" "grant2" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - table = "table1" - privileges = ["UPDATE", "SELECT"] - } - `, dbName, dbName) - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t); testAccPreCheckSkipRds(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareTable(dbName, "table1"), - ), - }, - { - Config: duplicateUserConfig, - ExpectError: regexp.MustCompile("already has"), - }, - }, - }) -} diff --git a/mysql/resource_grant_testcontainers_test.go b/mysql/resource_grant_testcontainers_test.go deleted file mode 100644 index d5bd1032..00000000 --- a/mysql/resource_grant_testcontainers_test.go +++ /dev/null @@ -1,515 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "fmt" - "math/rand" - "regexp" - "testing" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" -) - -// TestAccGrant_WithTestcontainers tests basic grant functionality -// Uses shared container set up in TestMain -func TestAccGrant_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - userName := fmt.Sprintf("jdoe-%s", dbName) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigBasic(dbName), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", userName), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "*"), - ), - }, - { - Config: testAccGrantConfigBasic(dbName), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", userName), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - ), - }, - { - Config: testAccGrantConfigBasic(dbName), - ResourceName: "mysql_grant.test", - ImportState: true, - ImportStateVerify: true, - ImportStateId: fmt.Sprintf("%v@%v@%v@%v", userName, "example.com", dbName, "*"), - }, - }, - }) -} - -// TestAccRevokePrivRefresh_WithTestcontainers tests privilege revocation and refresh -func TestAccRevokePrivRefresh_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigBasic(dbName), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "UPDATE", true, false), - ), - }, - { - RefreshState: true, - ExpectNonEmptyPlan: true, - Check: resource.ComposeTestCheckFunc( - revokeUserPrivs(dbName, "UPDATE"), - ), - }, - { - RefreshState: true, - ExpectNonEmptyPlan: true, - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "UPDATE", false, false), - ), - }, - { - PlanOnly: true, - ExpectNonEmptyPlan: true, - Config: testAccGrantConfigBasic(dbName), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "UPDATE", false, false), - ), - }, - { - Config: testAccGrantConfigBasic(dbName), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "UPDATE", true, false), - ), - }, - }, - }) -} - -// TestAccBroken_WithTestcontainers tests error handling for duplicate grants -func TestAccBroken_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigBasic(dbName), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "*"), - ), - }, - { - Config: testAccGrantConfigBroken(dbName), - ExpectError: regexp.MustCompile("already has"), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "*"), - ), - }, - }, - }) -} - -// TestAccDifferentHosts_WithTestcontainers tests grants with different hosts -func TestAccDifferentHosts_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigExtraHost(dbName, false), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test_all", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.test_all", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test_all", "host", "%"), - resource.TestCheckResourceAttr("mysql_grant.test_all", "table", "*"), - ), - }, - { - Config: testAccGrantConfigExtraHost(dbName, true), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "10.1.2.3"), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "*"), - resource.TestCheckResourceAttr("mysql_grant.test_all", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test_all", "host", "%"), - resource.TestCheckResourceAttr("mysql_grant.test_all", "table", "*"), - ), - }, - }, - }) -} - -// TestAccGrantComplex_WithTestcontainers tests complex grant scenarios -func TestAccGrantComplex_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - // Create table first - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareTable(dbName, "tbl"), - ), - }, - { - Config: testAccGrantConfigWithPrivs(dbName, `"SELECT (c1, c2)"`, false), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "SELECT (c1,c2)", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), - ), - }, - { - Config: testAccGrantConfigWithPrivs(dbName, `"DROP", "SELECT (c1)", "INSERT(c3, c4)", "REFERENCES(c5)"`, false), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "INSERT (c3,c4)", true, false), - testAccPrivilege("mysql_grant.test", "SELECT (c1)", true, false), - testAccPrivilege("mysql_grant.test", "SELECT (c1,c2)", false, false), - testAccPrivilege("mysql_grant.test", "REFERENCES (c5)", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), - ), - }, - { - Config: testAccGrantConfigWithPrivs(dbName, `"ALL PRIVILEGES"`, false), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "ALL", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), - ), - }, - { - Config: testAccGrantConfigWithPrivs(dbName, `"SELECT (c1, c2)","UPDATE(c1, c2)"`, true), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "SELECT (c1,c2)", true, true), - testAccPrivilege("mysql_grant.test", "UPDATE (c1,c2)", true, true), - testAccPrivilege("mysql_grant.test", "ALL", false, true), - testAccPrivilege("mysql_grant.test", "DROP", false, true), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", dbName), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "tbl"), - ), - }, - { - Config: testAccGrantConfigNoGrant(dbName), - }, - }, - }) -} - -// TestAccGrantComplexMySQL8_WithTestcontainers tests MySQL 8.0 specific grants -func TestAccGrantComplexMySQL8_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigWithDynamicMySQL8(dbName), - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.test", "CONNECTION_ADMIN", true, false), - testAccPrivilege("mysql_grant.test", "FIREWALL_EXEMPT", true, false), - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "database", "*"), - resource.TestCheckResourceAttr("mysql_grant.test", "table", "*"), - ), - }, - }, - }) -} - -// TestAccGrant_role_WithTestcontainers tests role grants (requires MySQL 8.0+) -func TestAccGrant_role_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - roleName := fmt.Sprintf("TFRole-exp%d", rand.Intn(100)) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigRole(dbName, roleName), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("mysql_grant.test", "role", roleName), - ), - }, - { - Config: testAccGrantConfigRoleWithGrantOption(dbName, roleName), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("mysql_grant.test", "role", roleName), - resource.TestCheckResourceAttr("mysql_grant.test", "grant", "true"), - ), - }, - { - Config: testAccGrantConfigRole(dbName, roleName), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("mysql_grant.test", "role", roleName), - ), - }, - }, - }) -} - -// TestAccGrant_roleToUser_WithTestcontainers tests granting roles to users (requires MySQL 8.0+) -func TestAccGrant_roleToUser_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - roleName := fmt.Sprintf("TFRole-%d", rand.Intn(100)) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigRoleToUser(dbName, roleName), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("mysql_grant.test", "user", fmt.Sprintf("jdoe-%s", dbName)), - resource.TestCheckResourceAttr("mysql_grant.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_grant.test", "roles.#", "1"), - ), - }, - }, - }) -} - -// TestAccGrant_complexRoleGrants_WithTestcontainers tests complex role grant scenarios (requires MySQL 8.0+) -func TestAccGrant_complexRoleGrants_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigComplexRoleGrants(dbName), - }, - }, - }) -} - -// TestAccGrantOnProcedure_WithTestcontainers tests procedure grants -func TestAccGrantOnProcedure_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - procedureName := "test_procedure" - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - userName := fmt.Sprintf("jdoe-%s", dbName) - hostName := "%" - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - // Create table first - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareTable(dbName, "tbl"), - ), - }, - { - // Create a procedure - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareProcedure(dbName, procedureName), - ), - }, - { - Config: testAccGrantConfigProcedureWithTable(procedureName, dbName, hostName), - Check: resource.ComposeTestCheckFunc( - testAccCheckProcedureGrant("mysql_grant.test_procedure", userName, hostName, procedureName, true), - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "user", userName), - resource.TestCheckResourceAttr("mysql_grant.test_procedure", "host", hostName), - ), - }, - }, - }) -} - -// TestAllowDuplicateUsersDifferentTables_WithTestcontainers tests allowing duplicate grants on different tables -func TestAllowDuplicateUsersDifferentTables_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - - duplicateUserConfig := fmt.Sprintf(` - resource "mysql_database" "test" { - name = "%s" - } - - resource "mysql_user" "test" { - user = "jdoe-%s" - host = "example.com" - } - - resource "mysql_grant" "grant1" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - table = "table1" - privileges = ["UPDATE", "SELECT"] - } - - resource "mysql_grant" "grant2" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - table = "table2" - privileges = ["UPDATE", "SELECT"] - } - `, dbName, dbName) - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - // Create table first - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareTable(dbName, "table1"), - prepareTable(dbName, "table2"), - ), - }, - { - Config: duplicateUserConfig, - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.grant1", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.grant1", "table", "table1"), - testAccPrivilege("mysql_grant.grant2", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.grant2", "table", "table2"), - ), - }, - { - RefreshState: true, - Check: resource.ComposeTestCheckFunc( - testAccPrivilege("mysql_grant.grant1", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.grant1", "table", "table1"), - testAccPrivilege("mysql_grant.grant2", "SELECT", true, false), - resource.TestCheckResourceAttr("mysql_grant.grant2", "table", "table2"), - ), - }, - }, - }) -} - -// TestDisallowDuplicateUsersSameTable_WithTestcontainers tests disallowing duplicate grants on same table -func TestDisallowDuplicateUsersSameTable_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - dbName := fmt.Sprintf("tf-test-%d", rand.Intn(100)) - - duplicateUserConfig := fmt.Sprintf(` - resource "mysql_database" "test" { - name = "%s" - } - - resource "mysql_user" "test" { - user = "jdoe-%s" - host = "example.com" - } - - resource "mysql_grant" "grant1" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - table = "table1" - privileges = ["UPDATE", "SELECT"] - } - - resource "mysql_grant" "grant2" { - user = "${mysql_user.test.user}" - host = "${mysql_user.test.host}" - database = "${mysql_database.test.name}" - table = "table1" - privileges = ["UPDATE", "SELECT"] - } - `, dbName, dbName) - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccGrantCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccGrantConfigNoGrant(dbName), - Check: resource.ComposeTestCheckFunc( - prepareTable(dbName, "table1"), - ), - }, - { - Config: duplicateUserConfig, - ExpectError: regexp.MustCompile("already has"), - }, - }, - }) -} diff --git a/mysql/resource_role_test.go b/mysql/resource_role_test.go index 631d1690..d732e2f3 100644 --- a/mysql/resource_role_test.go +++ b/mysql/resource_role_test.go @@ -1,3 +1,6 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( @@ -11,7 +14,12 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) +// Uses shared container set up in TestMain (MySQL 8.0 required for roles) +// Skips RDS and MySQL < 8.0 (same as original test) func TestAccRole_basic(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + roleName := "tf-test-role" resourceName := "mysql_role.test" @@ -19,6 +27,7 @@ func TestAccRole_basic(t *testing.T) { PreCheck: func() { testAccPreCheck(t) testAccPreCheckSkipRds(t) + // Check MySQL version (roles require 8.0+) ctx := context.Background() db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) if err != nil { diff --git a/mysql/resource_role_testcontainers_test.go b/mysql/resource_role_testcontainers_test.go deleted file mode 100644 index 97e532ca..00000000 --- a/mysql/resource_role_testcontainers_test.go +++ /dev/null @@ -1,36 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "testing" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" -) - -// TestAccRole_basic_WithTestcontainers tests the mysql_role resource -// using Testcontainers instead of Makefile + Docker -// Uses shared container set up in TestMain (MySQL 8.0 required for roles) -func TestAccRole_basic_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - roleName := "tf-test-role" - resourceName := "mysql_role.test" - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccRoleCheckDestroy(roleName), - Steps: []resource.TestStep{ - { - Config: testAccRoleConfigBasic(roleName), - Check: resource.ComposeTestCheckFunc( - testAccRoleExists(roleName), - resource.TestCheckResourceAttr(resourceName, "name", roleName), - ), - }, - }, - }) -} diff --git a/mysql/resource_user_password_test.go b/mysql/resource_user_password_test.go index 12eec9fb..62c38843 100644 --- a/mysql/resource_user_password_test.go +++ b/mysql/resource_user_password_test.go @@ -1,11 +1,30 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" ) +const testAccUserPasswordConfig_basic = ` +resource "mysql_user" "test" { + user = "jdoe" +} + +resource "mysql_user_password" "test" { + user = "${mysql_user.test.user}" + plaintext_password = "somepass" +} +` + +// Uses shared container set up in TestMain func TestAccUserPassword_basic(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, ProviderFactories: testAccProviderFactories, @@ -22,14 +41,3 @@ func TestAccUserPassword_basic(t *testing.T) { }, }) } - -const testAccUserPasswordConfig_basic = ` -resource "mysql_user" "test" { - user = "jdoe" -} - -resource "mysql_user_password" "test" { - user = "${mysql_user.test.user}" - plaintext_password = "somepass" -} -` diff --git a/mysql/resource_user_password_testcontainers_test.go b/mysql/resource_user_password_testcontainers_test.go deleted file mode 100644 index 6a03055e..00000000 --- a/mysql/resource_user_password_testcontainers_test.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "testing" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" -) - -// TestAccUserPassword_basic_WithTestcontainers tests the mysql_user_password resource -// using Testcontainers instead of Makefile + Docker -// Uses shared container set up in TestMain -func TestAccUserPassword_basic_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccUserCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccUserPasswordConfig_basic, - Check: resource.ComposeTestCheckFunc( - testAccUserExists("mysql_user.test"), - resource.TestCheckResourceAttr("mysql_user_password.test", "user", "jdoe"), - resource.TestCheckResourceAttrSet("mysql_user_password.test", "plaintext_password"), - ), - }, - }, - }) -} diff --git a/mysql/resource_user_test.go b/mysql/resource_user_test.go index 053fb768..8e193c68 100644 --- a/mysql/resource_user_test.go +++ b/mysql/resource_user_test.go @@ -1,3 +1,6 @@ +//go:build testcontainers +// +build testcontainers + package mysql import ( @@ -13,7 +16,12 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) +// Uses shared container set up in TestMain +// Skips MariaDB (same as original test) func TestAccUser_basic(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t); testAccPreCheckSkipMariaDB(t) }, ProviderFactories: testAccProviderFactories, @@ -53,14 +61,21 @@ func TestAccUser_basic(t *testing.T) { }) } +// Requires MySQL (not TiDB/MariaDB/RDS) with mysql_no_login plugin +// Uses shared container set up in TestMain +// Note: mysql_no_login plugin may not be available in all MySQL distributions func TestAccUser_auth(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheckSkipTiDB(t) testAccPreCheckSkipMariaDB(t) testAccPreCheckSkipRds(t) + // Skip on MySQL 8.0+ and Percona 8.0+ due to auth_plugin conflict with plaintext_password + testAccPreCheckSkipNotMySQLVersionMax(t, "7.99.99") // Check if mysql_no_login plugin is available - // This plugin may not be available in all MySQL distributions ctx := context.Background() db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) if err != nil { @@ -109,9 +124,15 @@ func TestAccUser_auth(t *testing.T) { }) } +// Requires MySQL (not TiDB/MariaDB/RDS) +// Uses shared container set up in TestMain func TestAccUser_authConnect(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + resource.Test(t, resource.TestCase{ PreCheck: func() { + testAccPreCheck(t) testAccPreCheckSkipTiDB(t) testAccPreCheckSkipMariaDB(t) testAccPreCheckSkipRds(t) @@ -149,9 +170,15 @@ func TestAccUser_authConnect(t *testing.T) { }) } +// Requires MySQL 8.0.14+ (not MariaDB/RDS) +// Uses shared container set up in TestMain func TestAccUser_authConnectRetainOldPassword(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + resource.Test(t, resource.TestCase{ PreCheck: func() { + testAccPreCheck(t) testAccPreCheckSkipMariaDB(t) testAccPreCheckSkipRds(t) testAccPreCheckSkipNotMySQLVersionMin(t, "8.0.14") @@ -183,9 +210,17 @@ func TestAccUser_authConnectRetainOldPassword(t *testing.T) { }) } +// Uses shared container set up in TestMain func TestAccUser_deprecated(t *testing.T) { + // Use shared container set up in TestMain + _ = getSharedMySQLContainer(t, "") + resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, + PreCheck: func() { + testAccPreCheck(t) + // Skip on MySQL 8.0+ and Percona 8.0+ due to stricter user creation requirements + testAccPreCheckSkipNotMySQLVersionMax(t, "7.99.99") + }, ProviderFactories: testAccProviderFactories, CheckDestroy: testAccUserCheckDestroy, Steps: []resource.TestStep{ @@ -211,56 +246,46 @@ func TestAccUser_deprecated(t *testing.T) { }) } -func testAccUserExists(rn string) resource.TestCheckFunc { - return func(s *terraform.State) error { - rs, ok := s.RootModule().Resources[rn] - if !ok { - return fmt.Errorf("resource not found: %s", rn) - } - - if rs.Primary.ID == "" { - return fmt.Errorf("user id not set") - } - - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - return err +func testAccUserCheckDestroy(s *terraform.State) error { + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return err + } + for _, rs := range s.RootModule().Resources { + if rs.Type != "mysql_user" { + continue } - - stmtSQL := fmt.Sprintf("SELECT count(*) from mysql.user where CONCAT(user, '@', host) = '%s'", rs.Primary.ID) + stmtSQL := fmt.Sprintf("SELECT user from mysql.user where CONCAT(user, '@', host) = '%s'", rs.Primary.ID) log.Println("[DEBUG] Executing statement:", stmtSQL) - var count int - err = db.QueryRow(stmtSQL).Scan(&count) + rows, err := db.Query(stmtSQL) if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return fmt.Errorf("expected 1 row reading user but got no rows") - } - return fmt.Errorf("error reading user: %s", err) + return fmt.Errorf("error issuing query: %s", err) + } + haveNext := rows.Next() + rows.Close() + if haveNext { + return fmt.Errorf("user still exists after destroy") } - - return nil } + return nil } -func testAccUserAuthExists(rn string) resource.TestCheckFunc { +func testAccUserExists(rn string) resource.TestCheckFunc { return func(s *terraform.State) error { rs, ok := s.RootModule().Resources[rn] if !ok { return fmt.Errorf("resource not found: %s", rn) } - if rs.Primary.ID == "" { return fmt.Errorf("user id not set") } - ctx := context.Background() db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) if err != nil { return err } - - stmtSQL := fmt.Sprintf("SELECT count(*) from mysql.user where CONCAT(user, '@', host) = '%s' and plugin = 'mysql_no_login'", rs.Primary.ID) + stmtSQL := fmt.Sprintf("SELECT count(*) from mysql.user where CONCAT(user, '@', host) = '%s'", rs.Primary.ID) log.Println("[DEBUG] Executing statement:", stmtSQL) var count int err = db.QueryRow(stmtSQL).Scan(&count) @@ -270,56 +295,10 @@ func testAccUserAuthExists(rn string) resource.TestCheckFunc { } return fmt.Errorf("error reading user: %s", err) } - - return nil - } -} - -func testAccUserAuthValid(user string, password string) resource.TestCheckFunc { - return func(s *terraform.State) error { - userConf := testAccProvider.Meta().(*MySQLConfiguration) - userConf.Config.User = user - userConf.Config.Passwd = password - - ctx := context.Background() - connection, err := createNewConnection(ctx, userConf) - if err != nil { - return fmt.Errorf("could not create new connection: %v", err) - } - - connection.Db.Close() - return nil } } -func testAccUserCheckDestroy(s *terraform.State) error { - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - return err - } - - for _, rs := range s.RootModule().Resources { - if rs.Type != "mysql_user" { - continue - } - - stmtSQL := fmt.Sprintf("SELECT user from mysql.user where CONCAT(user, '@', host) = '%s'", rs.Primary.ID) - log.Println("[DEBUG] Executing statement:", stmtSQL) - rows, err := db.Query(stmtSQL) - if err != nil { - return fmt.Errorf("error issuing query: %s", err) - } - haveNext := rows.Next() - rows.Close() - if haveNext { - return fmt.Errorf("user still exists after destroy") - } - } - return nil -} - const testAccUserConfig_basic = ` resource "mysql_user" "test" { user = "jdoe" @@ -345,6 +324,23 @@ resource "mysql_user" "test" { } ` +const testAccUserConfig_auth_iam_plugin = ` +resource "mysql_user" "test" { + user = "jdoe" + host = "example.com" + auth_plugin = "mysql_no_login" +} +` + +const testAccUserConfig_auth_native = ` +resource "mysql_user" "test" { + user = "jdoe" + host = "example.com" + auth_plugin = "mysql_native_password" + plaintext_password = "password" +} +` + const testAccUserConfig_deprecated = ` resource "mysql_user" "test" { user = "jdoe" @@ -361,24 +357,33 @@ resource "mysql_user" "test" { } ` -const testAccUserConfig_auth_iam_plugin = ` -resource "mysql_user" "test" { - user = "jdoe" - host = "example.com" - auth_plugin = "mysql_no_login" -} -` - -const testAccUserConfig_auth_native = ` -resource "mysql_user" "test" { - user = "jdoe" - host = "example.com" - auth_plugin = "mysql_native_password" - - # Hash of "password" - auth_string_hashed = "*2470C0C06DEE42FD1618BB99005ADCA2EC9D1E19" +func testAccUserAuthExists(rn string) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[rn] + if !ok { + return fmt.Errorf("resource not found: %s", rn) + } + if rs.Primary.ID == "" { + return fmt.Errorf("user id not set") + } + ctx := context.Background() + db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) + if err != nil { + return err + } + stmtSQL := fmt.Sprintf("SELECT count(*) from mysql.user where CONCAT(user, '@', host) = '%s'", rs.Primary.ID) + log.Println("[DEBUG] Executing statement:", stmtSQL) + var count int + err = db.QueryRow(stmtSQL).Scan(&count) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("expected 1 row reading user but got no rows") + } + return fmt.Errorf("error reading user: %s", err) + } + return nil + } } -` const testAccUserConfig_basic_retain_old_password = ` resource "mysql_user" "test" { @@ -406,3 +411,18 @@ resource "mysql_user" "test" { retain_old_password = true } ` + +func testAccUserAuthValid(user string, password string) resource.TestCheckFunc { + return func(s *terraform.State) error { + userConf := testAccProvider.Meta().(*MySQLConfiguration) + userConf.Config.User = user + userConf.Config.Passwd = password + ctx := context.Background() + connection, err := createNewConnection(ctx, userConf) + if err != nil { + return fmt.Errorf("could not create new connection: %v", err) + } + connection.Db.Close() + return nil + } +} diff --git a/mysql/resource_user_testcontainers_test.go b/mysql/resource_user_testcontainers_test.go deleted file mode 100644 index 740107a8..00000000 --- a/mysql/resource_user_testcontainers_test.go +++ /dev/null @@ -1,229 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "context" - "regexp" - "testing" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" -) - -// TestAccUser_basic_WithTestcontainers tests the mysql_user resource -// using Testcontainers instead of Makefile + Docker -// Uses shared container set up in TestMain -func TestAccUser_basic_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccUserCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccUserConfig_basic, - Check: resource.ComposeTestCheckFunc( - testAccUserExists("mysql_user.test"), - resource.TestCheckResourceAttr("mysql_user.test", "user", "jdoe"), - resource.TestCheckResourceAttr("mysql_user.test", "host", "%"), - resource.TestCheckResourceAttr("mysql_user.test", "plaintext_password", hashSum("password")), - resource.TestCheckResourceAttr("mysql_user.test", "tls_option", "NONE"), - ), - }, - { - Config: testAccUserConfig_ssl, - Check: resource.ComposeTestCheckFunc( - testAccUserExists("mysql_user.test"), - resource.TestCheckResourceAttr("mysql_user.test", "user", "jdoe"), - resource.TestCheckResourceAttr("mysql_user.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_user.test", "plaintext_password", hashSum("password")), - resource.TestCheckResourceAttr("mysql_user.test", "tls_option", "SSL"), - ), - }, - { - Config: testAccUserConfig_newPass, - Check: resource.ComposeTestCheckFunc( - testAccUserExists("mysql_user.test"), - resource.TestCheckResourceAttr("mysql_user.test", "user", "jdoe"), - resource.TestCheckResourceAttr("mysql_user.test", "host", "%"), - resource.TestCheckResourceAttr("mysql_user.test", "plaintext_password", hashSum("password2")), - resource.TestCheckResourceAttr("mysql_user.test", "tls_option", "NONE"), - ), - }, - }, - }) -} - -// TestAccUser_auth_WithTestcontainers tests auth plugin functionality -// Requires MySQL (not TiDB/MariaDB/RDS) with mysql_no_login plugin -// Uses shared container set up in TestMain -// Note: mysql_no_login plugin may not be available in all MySQL distributions -func TestAccUser_auth_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - resource.Test(t, resource.TestCase{ - PreCheck: func() { - testAccPreCheck(t) - // Check if mysql_no_login plugin is available - ctx := context.Background() - db, err := connectToMySQL(ctx, testAccProvider.Meta().(*MySQLConfiguration)) - if err != nil { - t.Fatalf("Cannot connect to DB: %v", err) - } - // Don't close - connection is cached and shared - - // Check if plugin exists - var pluginName string - err = db.QueryRowContext(ctx, "SELECT PLUGIN_NAME FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'mysql_no_login'").Scan(&pluginName) - if err != nil { - t.Skip("mysql_no_login plugin is not available in this MySQL distribution") - } - }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccUserCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccUserConfig_auth_iam_plugin, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthExists("mysql_user.test"), - resource.TestCheckResourceAttr("mysql_user.test", "user", "jdoe"), - resource.TestCheckResourceAttr("mysql_user.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_user.test", "auth_plugin", "mysql_no_login"), - ), - }, - { - Config: testAccUserConfig_auth_native, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthExists("mysql_user.test"), - resource.TestCheckResourceAttr("mysql_user.test", "user", "jdoe"), - resource.TestCheckResourceAttr("mysql_user.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_user.test", "auth_plugin", "mysql_native_password"), - ), - }, - { - Config: testAccUserConfig_auth_iam_plugin, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthExists("mysql_user.test"), - resource.TestCheckResourceAttr("mysql_user.test", "user", "jdoe"), - resource.TestCheckResourceAttr("mysql_user.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_user.test", "auth_plugin", "mysql_no_login"), - ), - }, - }, - }) -} - -// TestAccUser_authConnect_WithTestcontainers tests password authentication -// Requires MySQL (not TiDB/MariaDB/RDS) -// Uses shared container set up in TestMain -func TestAccUser_authConnect_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccUserCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccUserConfig_basic, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthValid("jdoe", "password"), - ), - }, - { - Config: testAccUserConfig_newPass, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthValid("jdoe", "random"), - ), - ExpectError: regexp.MustCompile(`.*Access denied for user 'jdoe'.*`), - }, - { - Config: testAccUserConfig_newPass, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthValid("jdoe", "password"), - ), - ExpectError: regexp.MustCompile(`.*Access denied for user 'jdoe'.*`), - }, - { - Config: testAccUserConfig_newPass, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthValid("jdoe", "password2"), - ), - }, - }, - }) -} - -// TestAccUser_authConnectRetainOldPassword_WithTestcontainers tests retain_old_password -// Requires MySQL 8.0.14+ -// Uses shared container set up in TestMain -func TestAccUser_authConnectRetainOldPassword_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccUserCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccUserConfig_basic_retain_old_password, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthValid("jdoe", "password"), - ), - }, - { - Config: testAccUserConfig_newPass_retain_old_password, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthValid("jdoe", "password"), - testAccUserAuthValid("jdoe", "password2"), - ), - }, - { - Config: testAccUserConfig_newNewPass_retain_old_password, - Check: resource.ComposeTestCheckFunc( - testAccUserAuthValid("jdoe", "password"), - ), - ExpectError: regexp.MustCompile(`.*Access denied for user 'jdoe'.*`), - }, - }, - }) -} - -// TestAccUser_deprecated_WithTestcontainers tests deprecated password attribute -// Uses shared container set up in TestMain -func TestAccUser_deprecated_WithTestcontainers(t *testing.T) { - // Use shared container set up in TestMain - _ = getSharedMySQLContainer(t, "mysql:8.0") - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccUserCheckDestroy, - Steps: []resource.TestStep{ - { - Config: testAccUserConfig_deprecated, - Check: resource.ComposeTestCheckFunc( - testAccUserExists("mysql_user.test"), - resource.TestCheckResourceAttr("mysql_user.test", "user", "jdoe"), - resource.TestCheckResourceAttr("mysql_user.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_user.test", "password", "password"), - ), - }, - { - Config: testAccUserConfig_deprecated_newPass, - Check: resource.ComposeTestCheckFunc( - testAccUserExists("mysql_user.test"), - resource.TestCheckResourceAttr("mysql_user.test", "user", "jdoe"), - resource.TestCheckResourceAttr("mysql_user.test", "host", "example.com"), - resource.TestCheckResourceAttr("mysql_user.test", "password", "password2"), - ), - }, - }, - }) -} diff --git a/mysql/testcontainers_helper.go b/mysql/testcontainers_helper.go index 80fcc9f4..af2c6a94 100644 --- a/mysql/testcontainers_helper.go +++ b/mysql/testcontainers_helper.go @@ -12,11 +12,16 @@ import ( "io" "log" "os" + "os/exec" + "path/filepath" + "runtime" "strings" "sync" "testing" "time" + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" "github.com/go-sql-driver/mysql" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/network" @@ -63,7 +68,11 @@ type MySQLTestContainer struct { // startMySQLContainer starts a MySQL/Percona/MariaDB container for testing // Supports MySQL, Percona, and MariaDB images +// image must not be empty - function will panic if empty func startMySQLContainer(ctx context.Context, t *testing.T, image string) *MySQLTestContainer { + if image == "" { + t.Fatalf("ERROR: startMySQLContainer called with empty image. DOCKER_IMAGE must be set.") + } // Determine timeout based on image/version timeout := 120 * time.Second if contains(image, "5.6") || contains(image, "5.7") || contains(image, "6.1") || contains(image, "6.5") { @@ -155,24 +164,78 @@ func contains(s, substr string) bool { return strings.Contains(s, substr) } -// getSharedMySQLContainer returns a shared MySQL container for all tests -// The container is created once and reused across all tests in the package +// getSharedMySQLContainer returns the shared MySQL container set up by TestMain +// The image parameter is ignored - TestMain uses DOCKER_IMAGE env var +// This function validates that DOCKER_IMAGE is set and fails early if not +// For TiDB tests, TestMain already sets up the cluster and environment variables, +// so this function just validates the environment is ready func getSharedMySQLContainer(t *testing.T, image string) *MySQLTestContainer { - sharedContainerOnce.Do(func() { - ctx := context.Background() - sharedContainer = startMySQLContainer(ctx, t, image) + // Validate that DOCKER_IMAGE is set (required by TestMain) + // This validation must always be present - fail early if DOCKER_IMAGE is empty + dockerImage := os.Getenv("DOCKER_IMAGE") + if dockerImage == "" { + t.Fatalf("ERROR: DOCKER_IMAGE environment variable is not set.\n" + + "Please set DOCKER_IMAGE to the appropriate Docker image:\n" + + " - MySQL/Percona/MariaDB: mysql:5.6, percona:8.0, mariadb:10.10\n" + + " - TiDB: tidb:6.1.7, tidb:8.5.3\n" + + "The 'image' parameter to getSharedMySQLContainer is ignored - use DOCKER_IMAGE env var instead.") + } + + // Validate that the provided image matches DOCKER_IMAGE (if provided) + if image != "" && image != dockerImage { + t.Fatalf("ERROR: getSharedMySQLContainer called with image '%s' but DOCKER_IMAGE is set to '%s'.\n"+ + "Remove the hardcoded image parameter - TestMain uses DOCKER_IMAGE env var to create the shared container.", + image, dockerImage) + } + + // Check if we're in TiDB mode + // For TiDB, TestMain already set up the cluster and environment variables + // Just validate that the environment variables are set and return a dummy container + if strings.HasPrefix(dockerImage, "tidb:") { + // Validate that TestMain set up the environment variables + endpoint := os.Getenv("MYSQL_ENDPOINT") + if endpoint == "" { + t.Fatalf("ERROR: MYSQL_ENDPOINT not set. TestMain should have set this for TiDB cluster.") + } + // Return a dummy container - tests will use environment variables set by TestMain + return &MySQLTestContainer{ + Container: nil, // Not used for TiDB + Endpoint: endpoint, + Username: os.Getenv("MYSQL_USERNAME"), + Password: os.Getenv("MYSQL_PASSWORD"), + } + } - // Set up environment variables for the shared container - os.Setenv("MYSQL_ENDPOINT", sharedContainer.Endpoint) - os.Setenv("MYSQL_USERNAME", sharedContainer.Username) - os.Setenv("MYSQL_PASSWORD", sharedContainer.Password) - }) - return sharedContainer + // For MySQL/Percona/MariaDB, TestMain should have set MYSQL_ENDPOINT + // Use environment variables as the source of truth (TestMain always sets these) + endpoint := os.Getenv("MYSQL_ENDPOINT") + if endpoint == "" { + t.Fatalf("ERROR: MYSQL_ENDPOINT not set. TestMain should have set this using DOCKER_IMAGE='%s'.\n"+ + "This indicates TestMain did not run or failed to initialize.", dockerImage) + } + + // If sharedContainer is available, use it; otherwise use environment variables + if sharedContainer != nil { + return sharedContainer + } + + // Fallback: use environment variables set by TestMain + // This handles cases where sharedContainer might be nil but environment variables are set + return &MySQLTestContainer{ + Container: nil, // Not available, but tests use environment variables + Endpoint: endpoint, + Username: os.Getenv("MYSQL_USERNAME"), + Password: os.Getenv("MYSQL_PASSWORD"), + } } // startSharedMySQLContainer starts a shared MySQL container without requiring a testing.T // Used by TestMain for initial setup +// image must not be empty - function will return error if empty func startSharedMySQLContainer(image string) (*MySQLTestContainer, error) { + if image == "" { + return nil, fmt.Errorf("ERROR: startSharedMySQLContainer called with empty image. DOCKER_IMAGE must be set") + } ctx := context.Background() // Determine timeout based on image/version @@ -298,6 +361,8 @@ type TiDBTestCluster struct { Endpoint string Username string Password string + // PlaygroundContainer is used when TiUP Playground is used instead of separate containers + PlaygroundContainer testcontainers.Container } // startTiDBCluster starts a TiDB cluster (PD, TiKV, TiDB) for testing @@ -339,6 +404,14 @@ func startTiDBCluster(ctx context.Context, t *testing.T, version string) *TiDBTe } // Start TiKV (storage layer) - connects to PD + // TiKV requires increased file descriptor limit + // v8.x versions require at least 123880, older versions require at least 82920 + tikvFdLimit := 200000 // Default for older versions + if strings.HasPrefix(version, "8.") { + // TiDB v8.x requires higher file descriptor limit + tikvFdLimit = 250000 + } + tikvContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ Image: fmt.Sprintf("pingcap/tikv:v%s", version), @@ -351,8 +424,19 @@ func startTiDBCluster(ctx context.Context, t *testing.T, version string) *TiDBTe "--data-dir=/data", "--pd=pd:2379", }, - WaitingFor: wait.ForLog("TiKV started"). - WithStartupTimeout(120 * time.Second), + HostConfigModifier: func(hostConfig *container.HostConfig) { + // Set ulimit for file descriptors (v8.x requires at least 123880) + hostConfig.Ulimits = []*container.Ulimit{ + { + Name: "nofile", + Soft: int64(tikvFdLimit), + Hard: int64(tikvFdLimit), + }, + } + }, + WaitingFor: wait.ForLog("succeed to update max timestamp"). + WithOccurrence(3). // Wait for at least 3 region updates - indicates TiKV is ready + WithStartupTimeout(180 * time.Second), }, Started: true, }) @@ -406,9 +490,207 @@ func startTiDBCluster(ctx context.Context, t *testing.T, version string) *TiDBTe } } +// startSharedTiDBClusterWithTiUP starts a TiDB cluster using TiUP Playground inside a single container +// This is faster and simpler than managing separate PD, TiKV, and TiDB containers +func startSharedTiDBClusterWithTiUP(version string) (*TiDBTestCluster, error) { + ctx := context.Background() + + // Try to use pre-built image first, fall back to building if not available + preBuiltImage := "terraform-provider-mysql-tiup-playground:latest" + + // Check if pre-built image exists + checkImageCmd := exec.Command("docker", "image", "inspect", preBuiltImage) + if err := checkImageCmd.Run(); err == nil { + // Pre-built image exists, use it directly + req := testcontainers.ContainerRequest{ + Image: preBuiltImage, + ExposedPorts: []string{"4000/tcp"}, + HostConfigModifier: func(hostConfig *container.HostConfig) { + hostConfig.Privileged = true + hostConfig.Ulimits = []*container.Ulimit{ + { + Name: "nofile", + Soft: int64(250000), + Hard: int64(250000), + }, + } + }, + Cmd: []string{ + "/root/.tiup/bin/tiup", "playground", version, + "--db", "1", + "--kv", "1", + "--pd", "1", + "--tiflash", "0", + "--without-monitor", + "--host", "0.0.0.0", + "--db.port", "4000", + }, + WaitingFor: wait.ForAll( + wait.ForListeningPort("4000/tcp"), + wait.ForSQL(nat.Port("4000/tcp"), "mysql", func(host string, port nat.Port) string { + return fmt.Sprintf("root@tcp(%s:%s)/", host, port.Port()) + }), + ).WithStartupTimeout(300 * time.Second), + } + + playgroundContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err == nil { + return getTiDBClusterFromContainer(playgroundContainer, ctx) + } + // If pre-built image fails, fall through to build + } + + // Build TiUP Playground image from Dockerfile + // Get the git root directory (where Dockerfile.tiup-playground is located) + // Use absolute path to avoid issues with working directory + moduleRoot := os.Getenv("GITHUB_WORKSPACE") + if moduleRoot == "" { + // For local development, find git root using git rev-parse + gitPath, err := exec.LookPath("git") + if err != nil { + // Git not found, try to find repo root by looking for .git directory + // Start from the directory where this source file is located + _, sourceFile, _, _ := runtime.Caller(0) + sourceDir := filepath.Dir(sourceFile) + // Go up from mysql/ to repo root + dir := filepath.Dir(sourceDir) + for { + dockerfilePath := filepath.Join(dir, "Dockerfile.tiup-playground") + if _, err := os.Stat(dockerfilePath); err == nil { + moduleRoot = dir + break + } + // Also check for .git as fallback + if _, err := os.Stat(filepath.Join(dir, ".git")); err == nil { + moduleRoot = dir + break + } + parent := filepath.Dir(dir) + if parent == dir { + return nil, fmt.Errorf("could not find Dockerfile.tiup-playground or .git in parent directories of %s", sourceDir) + } + dir = parent + } + } else { + cmd := exec.Command(gitPath, "rev-parse", "--show-toplevel") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("failed to find git root: %v", err) + } + moduleRoot = strings.TrimSpace(string(output)) + } + } + + // Convert to absolute path to ensure consistency + absModuleRoot, err := filepath.Abs(moduleRoot) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path for module root %s: %v", moduleRoot, err) + } + moduleRoot = absModuleRoot + + // Verify Dockerfile exists + dockerfilePath := filepath.Join(moduleRoot, "Dockerfile.tiup-playground") + if _, err := os.Stat(dockerfilePath); err != nil { + return nil, fmt.Errorf("Dockerfile.tiup-playground not found at %s (moduleRoot=%s): %v", dockerfilePath, moduleRoot, err) + } + + req := testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: moduleRoot, + Dockerfile: "Dockerfile.tiup-playground", + PrintBuildLog: true, // Helpful for debugging + // Don't set Tag - let testcontainers generate its own tag format + // Setting Tag causes invalid format like UUID:TAG:latest + }, + ExposedPorts: []string{"4000/tcp"}, + // TiUP Playground needs to run processes and requires elevated capabilities + HostConfigModifier: func(hostConfig *container.HostConfig) { + // Privileged mode allows TiUP to run multiple processes (PD, TiKV, TiDB) + hostConfig.Privileged = true + // Set ulimit for file descriptors (TiKV inside playground needs this) + hostConfig.Ulimits = []*container.Ulimit{ + { + Name: "nofile", + Soft: int64(250000), + Hard: int64(250000), + }, + } + }, + Cmd: []string{ + "/root/.tiup/bin/tiup", "playground", version, + "--db", "1", + "--kv", "1", + "--pd", "1", + "--tiflash", "0", + "--without-monitor", + "--host", "0.0.0.0", + "--db.port", "4000", + }, + WaitingFor: wait.ForAll( + wait.ForListeningPort("4000/tcp"), + wait.ForSQL(nat.Port("4000/tcp"), "mysql", func(host string, port nat.Port) string { + return fmt.Sprintf("root@tcp(%s:%s)/", host, port.Port()) + }), + ).WithStartupTimeout(300 * time.Second), // Longer timeout for Docker build + TiUP component downloads + } + + playgroundContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return nil, fmt.Errorf("failed to start TiUP Playground container: %v", err) + } + + return getTiDBClusterFromContainer(playgroundContainer, ctx) +} + +// getTiDBClusterFromContainer extracts connection details from a TiUP Playground container +func getTiDBClusterFromContainer(playgroundContainer testcontainers.Container, ctx context.Context) (*TiDBTestCluster, error) { + // Get endpoint + host, err := playgroundContainer.Host(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get playground container host: %v", err) + } + + port, err := playgroundContainer.MappedPort(ctx, "4000") + if err != nil { + return nil, fmt.Errorf("failed to get playground container port: %v", err) + } + + endpoint := fmt.Sprintf("%s:%s", host, port.Port()) + + return &TiDBTestCluster{ + PlaygroundContainer: playgroundContainer, + Endpoint: endpoint, + Username: "root", + Password: "", + }, nil +} + // startSharedTiDBCluster starts a shared TiDB cluster without requiring a testing.T // Used by TestMain for initial setup +// Tries TiUP Playground first (faster), falls back to multi-container if that fails func startSharedTiDBCluster(version string) (*TiDBTestCluster, error) { + // Try TiUP Playground approach first - much faster and simpler + cluster, err := startSharedTiDBClusterWithTiUP(version) + if err != nil { + // Log the error but don't fail yet - try fallback + fmt.Fprintf(os.Stderr, "Warning: TiUP Playground failed: %v\n", err) + fmt.Fprintf(os.Stderr, "Falling back to multi-container approach...\n") + os.Stderr.Sync() + + // Fall back to legacy multi-container approach + return startSharedTiDBClusterLegacy(version) + } + return cluster, nil +} + +// Legacy multi-container approach (kept for reference, but not used) +func startSharedTiDBClusterLegacy(version string) (*TiDBTestCluster, error) { ctx := context.Background() // Create a Docker network for TiDB cluster components @@ -447,6 +729,14 @@ func startSharedTiDBCluster(version string) (*TiDBTestCluster, error) { } // Start TiKV (storage layer) - connects to PD + // TiKV requires increased file descriptor limit + // v8.x versions require at least 123880, older versions require at least 82920 + tikvFdLimit := 200000 // Default for older versions + if strings.HasPrefix(version, "8.") { + // TiDB v8.x requires higher file descriptor limit + tikvFdLimit = 250000 + } + tikvContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ Image: fmt.Sprintf("pingcap/tikv:v%s", version), @@ -459,8 +749,19 @@ func startSharedTiDBCluster(version string) (*TiDBTestCluster, error) { "--data-dir=/data", "--pd=pd:2379", }, - WaitingFor: wait.ForLog("TiKV started"). - WithStartupTimeout(120 * time.Second), + HostConfigModifier: func(hostConfig *container.HostConfig) { + // Set ulimit for file descriptors (v8.x requires at least 123880) + hostConfig.Ulimits = []*container.Ulimit{ + { + Name: "nofile", + Soft: int64(tikvFdLimit), + Hard: int64(tikvFdLimit), + }, + } + }, + WaitingFor: wait.ForLog("succeed to update max timestamp"). + WithOccurrence(3). // Wait for at least 3 region updates - indicates TiKV is ready + WithStartupTimeout(180 * time.Second), }, Started: true, }) @@ -521,17 +822,34 @@ func cleanupSharedTiDBCluster() { if sharedTiDBCluster != nil { ctx := context.Background() - if err := sharedTiDBCluster.TiDBContainer.Terminate(ctx); err != nil { - fmt.Printf("Warning: Failed to terminate TiDB container: %v\n", err) - } - if err := sharedTiDBCluster.TiKVContainer.Terminate(ctx); err != nil { - fmt.Printf("Warning: Failed to terminate TiKV container: %v\n", err) - } - if err := sharedTiDBCluster.PDContainer.Terminate(ctx); err != nil { - fmt.Printf("Warning: Failed to terminate PD container: %v\n", err) - } - if err := sharedTiDBCluster.Network.Remove(ctx); err != nil { - fmt.Printf("Warning: Failed to remove TiDB network: %v\n", err) + + // If using TiUP Playground (single container) + if sharedTiDBCluster.PlaygroundContainer != nil { + if err := sharedTiDBCluster.PlaygroundContainer.Terminate(ctx); err != nil { + fmt.Printf("Warning: Failed to terminate TiUP Playground container: %v\n", err) + } + } else { + // Legacy multi-container approach + if sharedTiDBCluster.TiDBContainer != nil { + if err := sharedTiDBCluster.TiDBContainer.Terminate(ctx); err != nil { + fmt.Printf("Warning: Failed to terminate TiDB container: %v\n", err) + } + } + if sharedTiDBCluster.TiKVContainer != nil { + if err := sharedTiDBCluster.TiKVContainer.Terminate(ctx); err != nil { + fmt.Printf("Warning: Failed to terminate TiKV container: %v\n", err) + } + } + if sharedTiDBCluster.PDContainer != nil { + if err := sharedTiDBCluster.PDContainer.Terminate(ctx); err != nil { + fmt.Printf("Warning: Failed to terminate PD container: %v\n", err) + } + } + if sharedTiDBCluster.Network != nil { + if err := sharedTiDBCluster.Network.Remove(ctx); err != nil { + fmt.Printf("Warning: Failed to remove TiDB network: %v\n", err) + } + } } sharedTiDBCluster = nil } diff --git a/mysql/testcontainers_testmain.go b/mysql/testcontainers_testmain.go deleted file mode 100644 index 98a4564c..00000000 --- a/mysql/testcontainers_testmain.go +++ /dev/null @@ -1,76 +0,0 @@ -//go:build testcontainers -// +build testcontainers - -package mysql - -import ( - "fmt" - "os" - "testing" -) - -// TestMain sets up a shared MySQL/TiDB container for all testcontainers tests -// This is more efficient than starting a container for each test -func TestMain(m *testing.M) { - // Check if we're testing TiDB (requires multi-container setup) - tidbVersion := os.Getenv("TIDB_VERSION") - if tidbVersion != "" { - // Start shared TiDB cluster before running tests - var err error - sharedTiDBClusterMtx.Lock() - sharedTiDBCluster, err = startSharedTiDBCluster(tidbVersion) - sharedTiDBClusterMtx.Unlock() - - if err != nil { - // If cluster startup fails, exit with error - os.Stderr.WriteString(fmt.Sprintf("Failed to start shared TiDB cluster: %v\n", err)) - os.Exit(1) - } - - // Set up environment variables for the shared TiDB cluster - os.Setenv("MYSQL_ENDPOINT", sharedTiDBCluster.Endpoint) - os.Setenv("MYSQL_USERNAME", sharedTiDBCluster.Username) - os.Setenv("MYSQL_PASSWORD", sharedTiDBCluster.Password) - - // Run all tests - code := m.Run() - - // Cleanup shared TiDB cluster after all tests complete - cleanupSharedTiDBCluster() - - // Exit with test result code - os.Exit(code) - } - - // Default to MySQL 8.0, but allow override via DOCKER_IMAGE env var - mysqlImage := os.Getenv("DOCKER_IMAGE") - if mysqlImage == "" { - mysqlImage = "mysql:8.0" - } - - // Start shared container before running tests - var err error - sharedContainerMtx.Lock() - sharedContainer, err = startSharedMySQLContainer(mysqlImage) - sharedContainerMtx.Unlock() - - if err != nil { - // If container startup fails, exit with error - os.Stderr.WriteString(fmt.Sprintf("Failed to start shared MySQL container: %v\n", err)) - os.Exit(1) - } - - // Set up environment variables for the shared container - os.Setenv("MYSQL_ENDPOINT", sharedContainer.Endpoint) - os.Setenv("MYSQL_USERNAME", sharedContainer.Username) - os.Setenv("MYSQL_PASSWORD", sharedContainer.Password) - - // Run all tests - code := m.Run() - - // Cleanup shared container after all tests complete - cleanupSharedContainer() - - // Exit with test result code - os.Exit(code) -} diff --git a/scripts/test-runner.go b/scripts/test-runner.go index a8be1b99..1e02c21c 100644 --- a/scripts/test-runner.go +++ b/scripts/test-runner.go @@ -51,6 +51,8 @@ type testResult struct { image string dbType string passed bool + skipped bool + skipReason string logFile string duration time.Duration totalTests int @@ -102,16 +104,36 @@ func main() { // Get parallelism from environment variable parallel := getParallelism() + // Detect platform architecture + isARM := isARMPlatform() + fmt.Printf("Testcontainers Matrix Test Suite\n") - fmt.Printf("Test pattern: %s | Parallelism: %d\n\n", testPattern, parallel) + fmt.Printf("Test pattern: %s | Parallelism: %d", testPattern, parallel) + if isARM { + fmt.Printf(" | Platform: ARM (Apple Silicon)") + } + fmt.Printf("\n\n") - // Build all test jobs + // Build all test jobs and track skipped ones var jobs []testJob + var skippedResults []testResult testNum := 0 // MySQL tests for _, version := range mysqlVersions { testNum++ + // Skip MySQL 5.6 and 5.7 on ARM (no ARM64 builds available) + if isARM && (version == "mysql:5.6" || version == "mysql:5.7") { + skippedResults = append(skippedResults, testResult{ + image: version, + dbType: "MySQL", + passed: true, // Skipped tests don't fail the suite + skipped: true, + skipReason: "No ARM64 builds available", + duration: 0, + }) + continue + } jobs = append(jobs, testJob{ image: version, dbType: "MySQL", @@ -123,6 +145,18 @@ func main() { // Percona tests for _, version := range perconaVersions { testNum++ + // Skip Percona 5.7 and 8.0 on ARM (no ARM64 builds available) + if isARM && (version == "percona:5.7" || version == "percona:8.0") { + skippedResults = append(skippedResults, testResult{ + image: version, + dbType: "Percona", + passed: true, // Skipped tests don't fail the suite + skipped: true, + skipReason: "No ARM64 builds available", + duration: 0, + }) + continue + } jobs = append(jobs, testJob{ image: version, dbType: "Percona", @@ -165,17 +199,27 @@ func main() { results = runTestsSequential(jobs) } + // Add skipped results to the results list + results = append(results, skippedResults...) + // Print summary printSummary(results) - // Exit with error code if any tests failed + // Exit with error code if any tests failed (skipped tests don't count as failures) for _, result := range results { - if !result.passed { + if !result.passed && !result.skipped { os.Exit(1) } } } +// isARMPlatform detects if we're running on ARM architecture (including Apple Silicon) +func isARMPlatform() bool { + arch := runtime.GOARCH + // Check for ARM architectures + return arch == "arm64" || arch == "arm" +} + func getParallelism() int { parallelStr := os.Getenv("PARALLEL") if parallelStr == "" { @@ -251,6 +295,30 @@ func runTestsParallel(jobs []testJob, parallel int) []testResult { func runTest(job testJob) testResult { key := fmt.Sprintf("%s-%s", job.dbType, job.image) + // Check if this test should be skipped on ARM + isARM := isARMPlatform() + shouldSkip := false + skipReason := "" + if isARM { + if (job.dbType == "MySQL" && (job.image == "mysql:5.6" || job.image == "mysql:5.7")) || + (job.dbType == "Percona" && (job.image == "percona:5.7" || job.image == "percona:8.0")) { + shouldSkip = true + skipReason = "No ARM64 builds available" + } + } + + if shouldSkip { + // Return skipped result immediately + return testResult{ + image: job.image, + dbType: job.dbType, + passed: true, // Skipped tests don't fail the suite + skipped: true, + skipReason: skipReason, + duration: 0, + } + } + // Initialize progress tracker progress.mu.Lock() progress.trackers[key] = &versionProgress{ @@ -294,13 +362,25 @@ func runTest(job testJob) testResult { // Set environment variables envVars := os.Environ() + // All database types use DOCKER_IMAGE + // For TiDB, format is tidb:VERSION (e.g., tidb:6.1.7) + // For MySQL/Percona/MariaDB, format is already full image name (e.g., mysql:8.0) + dockerImage := job.image if job.dbType == "TiDB" { - // TiDB uses version number, not full image name - envVars = append(envVars, "TIDB_VERSION="+job.image) - } else { - envVars = append(envVars, "DOCKER_IMAGE="+job.image) + // TiDB image is just version number, prepend "tidb:" prefix + dockerImage = "tidb:" + job.image } + envVars = append(envVars, "DOCKER_IMAGE="+dockerImage) envVars = append(envVars, "TF_ACC=1", "GOTOOLCHAIN=auto") + + // Handle platform-specific issues for older MySQL/Percona versions on ARM64 + // MySQL 5.6, 5.7 and Percona 5.7, 8.0 don't have ARM64 builds + // Docker Desktop on Apple Silicon needs explicit platform specification + if (job.dbType == "MySQL" && (job.image == "5.6" || job.image == "5.7")) || + (job.dbType == "Percona" && (job.image == "5.7" || job.image == "8.0")) { + envVars = append(envVars, "DOCKER_DEFAULT_PLATFORM=linux/amd64") + } + cmd.Env = envVars // Create log file @@ -618,12 +698,17 @@ func printSummary(results []testResult) { ) // Add rows + skippedCount := 0 for _, result := range sortedResults { - status := "PASS" - if !result.passed { + var status string + if result.skipped { + status = fmt.Sprintf("SKIP (%s)", result.skipReason) + skippedCount++ + } else if !result.passed { status = "FAIL" failed++ } else { + status = "PASS" passed++ } @@ -631,8 +716,8 @@ func printSummary(results []testResult) { version := extractVersion(result.image) duration := formatDuration(result.duration) - // Add test counts to status - if result.totalTests > 0 { + // Add test counts to status (only for non-skipped tests) + if !result.skipped && result.totalTests > 0 { if result.failedTests > 0 { status = fmt.Sprintf("%s (%d/%d, %d failed)", status, result.passedTests, result.totalTests, result.failedTests) } else { @@ -651,10 +736,17 @@ func printSummary(results []testResult) { table.Render() - fmt.Printf("\nSummary: %d/%d passed", passed, total) + totalRun := passed + failed + fmt.Printf("\nSummary: %d/%d passed", passed, totalRun) + if skippedCount > 0 { + fmt.Printf(", %d skipped", skippedCount) + } if failed > 0 { fmt.Printf(", %d failed", failed) } + if skippedCount > 0 { + fmt.Printf(" (%d total test suites)", total) + } fmt.Println() if failed > 0 { @@ -663,8 +755,8 @@ func printSummary(results []testResult) { } func extractVersion(image string) string { - // For TiDB, the image is already just the version number - // For MySQL/Percona/MariaDB, extract version from image string (e.g., "mysql:8.0" -> "8.0") + // Extract version from image string + // Examples: "mysql:8.0" -> "8.0", "tidb:6.1.7" -> "6.1.7", "percona:5.7" -> "5.7" parts := strings.Split(image, ":") if len(parts) > 1 { return parts[1]