feat(test): adding support for backwards compatibility testing (#1912)

Author: chatton

.github/workflows/e2e-fork.yml

@@ -24,8 +24,8 @@ jobs:
 
   e2e:
     env:
-      SIMD_TAG: latest
-      SIMD_IMAGE: ibc-go-simd-e2e
+      CHAIN_A_SIMD_TAG: latest
+      CHAIN_A_SIMD_IMAGE: ibc-go-simd-e2e
     if: ${{ github.event.pull_request.head.repo.fork || github.actor == 'dependabot[bot]' || github.event_name == 'workflow_dispatch' }}
     needs:
       - build-test-matrix
@@ -36,7 +36,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Docker Build
-        run: docker build . -t "${SIMD_IMAGE}:${SIMD_TAG}"
+        run: docker build . -t "${CHAIN_A_SIMD_IMAGE}:${CHAIN_A_SIMD_TAG}"
       - name: Setup Go
         uses: actions/setup-go@v3
         with:

.github/workflows/e2e-manual.yaml

@@ -0,0 +1,94 @@
+name: Manual E2E
+on:
+  # when https://github.com/community/community/discussions/11795 is resolved
+  # we will be able to dynamically build up the list of valid inputs.
+  # for now this needs to be manual.
+  workflow_dispatch:
+    inputs:
+      test-entry-point:
+        description: 'Test entry point'
+        required: true
+        type: choice
+        options:
+          - TestFeeMiddlewareTestSuite
+      chain-a-image:
+        description: 'The image to use for chain A'
+        required: true
+        type: string
+        default: "ghcr.io/cosmos/ibc-go-simd-e2e"
+      chain-a-tag:
+        description: 'The tag to use for chain A'
+        required: true
+        type: choice
+        default: main
+        options:
+          - main
+          - v4.0.0-rc3
+          - v3.0.0
+          - v2.2.0
+          - v2.1.0
+          - v2.0.0
+      chain-b-image:
+        description: 'The image to use for chain B'
+        required: true
+        type: string
+        default: "ghcr.io/cosmos/ibc-go-simd"
+      chain-b-tag:
+        default: v4.0.0-rc3
+        description: 'The tag to use for chain B'
+        required: true
+        type: choice
+        options:
+          - v4.0.0-rc3
+          - v3.0.0
+          - v2.2.0
+          - v2.1.0
+          - v2.0.0
+      relayer-tag:
+        description: 'The tag to use for the relayer'
+        required: true
+        default: "v2.0.0-rc2"
+        type: string
+
+
+jobs:
+  # dynamically build a matrix of test/test suite pairs to run
+  build-test-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+      - id: set-matrix
+        run: echo "::set-output name=matrix::$(go run cmd/build_test_matrix/main.go)"
+        env:
+          TEST_ENTRYPOINT: "${{ github.event.inputs.test-entry-point }}"
+
+
+
+  e2e-manual:
+    runs-on: ubuntu-latest
+    needs:
+      - build-test-matrix
+    env:
+      CHAIN_A_SIMD_TAG: "${{ github.event.inputs.chain-a-tag }}"
+      CHAIN_A_SIMD_IMAGE: "${{ github.event.inputs.chain-a-image }}"
+      CHAIN_B_SIMD_TAG: "${{ github.event.inputs.chain-b-tag }}"
+      CHAIN_B_SIMD_IMAGE: "${{ github.event.inputs.chain-b-image }}"
+      # see images here https://github.com/cosmos/relayer/pkgs/container/relayer/versions
+      RLY_TAG: "${{ github.event.inputs.relayer-tag }}"
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.build-test-matrix.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+      - name: Run e2e Test
+        run: |
+          cd e2e
+          make e2e-test suite=${{ matrix.suite }} test=${{ matrix.test }}

.github/workflows/e2e.yaml

@@ -81,8 +81,8 @@ jobs:
       - determine-image-tag
       - docker-build
     env:
-      SIMD_TAG: ${{ needs.determine-image-tag.outputs.simd-tag }}
-      SIMD_IMAGE: ghcr.io/cosmos/ibc-go-simd-e2e
+      CHAIN_A_SIMD_TAG: ${{ needs.determine-image-tag.outputs.simd-tag }}
+      CHAIN_A_SIMD_IMAGE: ghcr.io/cosmos/ibc-go-simd-e2e
       # see images here https://github.com/cosmos/relayer/pkgs/container/relayer/versions
       RLY_TAG: "v2.0.0-rc2"
     strategy:

cmd/build_test_matrix/main.go

@@ -16,6 +16,7 @@ const (
 	testNamePrefix     = "Test"
 	testFileNameSuffix = "_test.go"
 	e2eTestDirectory   = "e2e"
+	testEntryPointEnv  = "TEST_ENTRYPOINT"
 )
 
 // GithubActionTestMatrix represents
@@ -29,7 +30,7 @@ type TestSuitePair struct {
 }
 
 func main() {
-	githubActionMatrix, err := getGithubActionMatrixForTests(e2eTestDirectory)
+	githubActionMatrix, err := getGithubActionMatrixForTests(e2eTestDirectory, getTestFunctionToRun())
 	if err != nil {
 		fmt.Printf("error generating github action json: %s", err)
 		os.Exit(1)
@@ -43,10 +44,20 @@ func main() {
 	fmt.Println(string(ghBytes))
 }
 
+// getTestFunctionToRun returns the specified test function to run if present, otherwise
+// it returns an empty string which will result in running all test suites.
+func getTestFunctionToRun() string {
+	testSuite, ok := os.LookupEnv(testEntryPointEnv)
+	if !ok {
+		return ""
+	}
+	return testSuite
+}
+
 // getGithubActionMatrixForTests returns a json string representing the contents that should go in the matrix
 // field in a github action workflow. This string can be used with `fromJSON(str)` to dynamically build
 // the workflow matrix to include all E2E tests under the e2eRootDirectory directory.
-func getGithubActionMatrixForTests(e2eRootDirectory string) (GithubActionTestMatrix, error) {
+func getGithubActionMatrixForTests(e2eRootDirectory, suite string) (GithubActionTestMatrix, error) {
 	testSuiteMapping := map[string][]string{}
 	fset := token.NewFileSet()
 	err := filepath.Walk(e2eRootDirectory, func(path string, info fs.FileInfo, err error) error {
@@ -69,7 +80,9 @@ func getGithubActionMatrixForTests(e2eRootDirectory string) (GithubActionTestMat
 			return fmt.Errorf("failed extracting test suite name and test cases: %s", err)
 		}
 
-		testSuiteMapping[suiteNameForFile] = testCases
+		if suite == "" || suiteNameForFile == suite {
+			testSuiteMapping[suiteNameForFile] = testCases
+		}
 
 		return nil
 	})

cmd/build_test_matrix/main_test.go

@@ -19,15 +19,15 @@ const (
 func TestGetGithubActionMatrixForTests(t *testing.T) {
 	t.Run("empty dir does not fail", func(t *testing.T) {
 		testingDir := t.TempDir()
-		_, err := getGithubActionMatrixForTests(testingDir)
+		_, err := getGithubActionMatrixForTests(testingDir, "")
 		assert.NoError(t, err)
 	})
 
 	t.Run("only test functions are picked up", func(t *testing.T) {
 		testingDir := t.TempDir()
 		createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, goTestFileNameOne)
 
-		gh, err := getGithubActionMatrixForTests(testingDir)
+		gh, err := getGithubActionMatrixForTests(testingDir, "")
 		assert.NoError(t, err)
 
 		expected := GithubActionTestMatrix{
@@ -50,7 +50,7 @@ func TestGetGithubActionMatrixForTests(t *testing.T) {
 		createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, goTestFileNameOne)
 		createFileWithTestSuiteAndTests(t, "TransferTestSuite", "TestC", "TestD", testingDir, goTestFileNameTwo)
 
-		gh, err := getGithubActionMatrixForTests(testingDir)
+		gh, err := getGithubActionMatrixForTests(testingDir, "")
 		assert.NoError(t, err)
 
 		expected := GithubActionTestMatrix{
@@ -81,7 +81,7 @@ func TestGetGithubActionMatrixForTests(t *testing.T) {
 		testingDir := t.TempDir()
 		createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, nonTestFile)
 
-		gh, err := getGithubActionMatrixForTests(testingDir)
+		gh, err := getGithubActionMatrixForTests(testingDir, "")
 		assert.NoError(t, err)
 		assert.Empty(t, gh.Include)
 	})
@@ -105,7 +105,7 @@ type FeeMiddlewareTestSuite struct {}
 		err := os.WriteFile(path.Join(testingDir, goTestFileNameOne), []byte(fileWithTwoSuites), os.FileMode(777))
 		assert.NoError(t, err)
 
-		_, err = getGithubActionMatrixForTests(testingDir)
+		_, err = getGithubActionMatrixForTests(testingDir, "")
 		assert.Error(t, err)
 	})
 }

e2e/README.md

@@ -57,8 +57,14 @@ Every time changes are pushed to a branch or to `main`, a new `simd` image is bu
 #### Example Command:
 
 ```sh
-export SIMD_IMAGE="ghcr.io/cosmos/ibc-go-simd-e2e"
-export SIMD_TAG="pr-1650"
+export CHAIN_A_SIMD_IMAGE="ghcr.io/cosmos/ibc-go-simd-e2e"
+export CHAIN_A_SIMD_TAG="pr-1650"
+
+# We can also specify different values for the chains if needed.
+# they will default to the same as chain a.
+# export CHAIN_B_SIMD_IMAGE="ghcr.io/cosmos/ibc-go-simd-e2e"
+# export CHAIN_B_SIMD_TAG="pr-1650"
+
 export RLY_TAG="v2.0.0-rc2"
 make e2e-test suite=FeeMiddlewareTestSuite test=TestMultiMsg_MsgPayPacketFeeSingleSender
 ```

e2e/scripts/run-e2e.sh

@@ -4,8 +4,8 @@ set -euo pipefail
 
 SUITE="${1}"
 TEST="${2}"
-export SIMD_TAG="${SIMD_TAG:-latest}"
-export SIMD_IMAGE="${SIMD_IMAGE:-ibc-go-simd-e2e}"
+export SIMD_TAG="${CHAIN_A_SIMD_TAG:-latest}"
+export SIMD_IMAGE="${CHAIN_A_SIMD_IMAGE:-ibc-go-simd-e2e}"
 
 # In CI, the docker images will be built separately.
 # context for building the image is one directory up.

e2e/testconfig/testconfig.go

@@ -8,42 +8,71 @@ import (
 )
 
 const (
-	DefaultSimdImage = "ghcr.io/cosmos/ibc-go-simd-e2e"
-	SimdImageEnv     = "SIMD_IMAGE"
-	SimdTagEnv       = "SIMD_TAG"
-	GoRelayerTag     = "RLY_TAG"
+	// ChainASimdImageEnv specifies the image that Chain A will use.
+	ChainASimdImageEnv = "CHAIN_A_SIMD_IMAGE"
+	// ChainASimdTagEnv specifies the tag that Chain A will use.
+	ChainASimdTagEnv = "CHAIN_A_SIMD_TAG"
+	// ChainBSimdImageEnv specifies the image that Chain B will use. If unspecified
+	// the value will default to the same value as Chain A.
+	ChainBSimdImageEnv = "CHAIN_B_SIMD_IMAGE"
+	// ChainBSimdTagEnv specifies the tag that Chain B will use. If unspecified
+	// the value will default to the same value as Chain A.
+	ChainBSimdTagEnv = "CHAIN_B_SIMD_TAG"
+	GoRelayerTagEnv  = "RLY_TAG"
 
-	defaultRlyTag = "main"
+	defaultSimdImage = "ghcr.io/cosmos/ibc-go-simd-e2e"
+	defaultRlyTag    = "main"
 )
 
 // TestConfig holds various fields used in the E2E tests.
 type TestConfig struct {
-	SimdImage string
-	SimdTag   string
-	RlyTag    string
+	ChainAConfig ChainConfig
+	ChainBConfig ChainConfig
+	RlyTag       string
+}
+
+type ChainConfig struct {
+	Image string
+	Tag   string
 }
 
 // FromEnv returns a TestConfig constructed from environment variables.
 func FromEnv() TestConfig {
-	simdImage, ok := os.LookupEnv(SimdImageEnv)
+	chainASimdImage, ok := os.LookupEnv(ChainASimdImageEnv)
+	if !ok {
+		chainASimdImage = defaultSimdImage
+	}
+
+	chainASimdTag, ok := os.LookupEnv(ChainASimdTagEnv)
 	if !ok {
-		simdImage = DefaultSimdImage
+		panic(fmt.Sprintf("must specify simd version for test with environment variable [%s]", ChainASimdTagEnv))
 	}
 
-	simdTag, ok := os.LookupEnv(SimdTagEnv)
+	chainBSimdImage, ok := os.LookupEnv(ChainBSimdImageEnv)
 	if !ok {
-		panic(fmt.Sprintf("must specify simd version for test with environment variable [%s]", SimdTagEnv))
+		chainBSimdImage = chainASimdImage
 	}
 
-	rlyTag, ok := os.LookupEnv(GoRelayerTag)
+	chainBSimdTag, ok := os.LookupEnv(ChainBSimdTagEnv)
+	if !ok {
+		chainBSimdTag = chainASimdTag
+	}
+
+	rlyTag, ok := os.LookupEnv(GoRelayerTagEnv)
 	if !ok {
 		rlyTag = defaultRlyTag
 	}
 
 	return TestConfig{
-		SimdImage: simdImage,
-		SimdTag:   simdTag,
-		RlyTag:    rlyTag,
+		ChainAConfig: ChainConfig{
+			Image: chainASimdImage,
+			Tag:   chainASimdTag,
+		},
+		ChainBConfig: ChainConfig{
+			Image: chainBSimdImage,
+			Tag:   chainBSimdTag,
+		},
+		RlyTag: rlyTag,
 	}
 }
 
@@ -62,24 +91,24 @@ type ChainOptionConfiguration func(options *ChainOptions)
 // These options can be configured by passing configuration functions to E2ETestSuite.GetChains.
 func DefaultChainOptions() ChainOptions {
 	tc := FromEnv()
-	chainACfg := newDefaultSimappConfig(tc, "simapp-a", "chain-a", "atoma")
-	chainBCfg := newDefaultSimappConfig(tc, "simapp-b", "chain-b", "atomb")
+	chainACfg := newDefaultSimappConfig(tc.ChainAConfig, "simapp-a", "chain-a", "atoma")
+	chainBCfg := newDefaultSimappConfig(tc.ChainBConfig, "simapp-b", "chain-b", "atomb")
 	return ChainOptions{
 		ChainAConfig: &chainACfg,
 		ChainBConfig: &chainBCfg,
 	}
 }
 
 // newDefaultSimappConfig creates an ibc configuration for simd.
-func newDefaultSimappConfig(tc TestConfig, name, chainID, denom string) ibc.ChainConfig {
+func newDefaultSimappConfig(cc ChainConfig, name, chainID, denom string) ibc.ChainConfig {
 	return ibc.ChainConfig{
 		Type:    "cosmos",
 		Name:    name,
 		ChainID: chainID,
 		Images: []ibc.DockerImage{
 			{
-				Repository: tc.SimdImage,
-				Version:    tc.SimdTag,
+				Repository: cc.Image,
+				Version:    cc.Tag,
 			},
 		},
 		Bin:            "simd",