feat: add ignoreFiles option to suppress file warnings in test directories

Adds a new TestSuite configuration option `ignoreFiles` that accepts
file patterns (e.g., "*.md", "README*") to ignore when collecting test
step files. Files matching these patterns are silently skipped without
generating warnings, allowing documentation and other non-test files to
coexist in test case directories.

Fixes #650.

Signed-off-by: Martin André <[email protected]>

Author: mandre

docs/testing/reference.md

@@ -13,6 +13,10 @@ kindContainers:
 testDirs:
 - tests/e2e/
 timeout: 120
+ignoreFiles:
+- "*.md"
+- "README*"
+- ".gitignore"
 ```
 
 Supported settings:
@@ -39,6 +43,7 @@ reportGranularity | string           | What granularity to report failures at. O
 reportName        | string           | The name of report to create. This field is not used unless reportFormat is set.         | "kuttl-test"
 namespace         | string           | The namespace to use for tests. This namespace will be created if it does not exist and removed if it was created (unless `skipDelete` is set). If no namespace is set, one will be auto-generated. |
 suppress          | list of strings  | Suppresses log collection of the specified types. Currently only `events` is supported.  |
+ignoreFiles       | list of strings  | File patterns (e.g., `*.md`, `README*`) to ignore when collecting test steps. Files matching these patterns will not generate warnings about not matching the expected test file pattern. | []
 
 ## TestStep
 

internal/harness/harness.go

@@ -88,6 +88,7 @@ func (h *Harness) LoadTests(dir string) ([]*testcase.Case, error) {
 			testcase.WithNamespace(h.TestSuite.Namespace),
 			testcase.WithTimeout(timeout),
 			testcase.WithLogSuppressions(h.TestSuite.Suppress),
+			testcase.WithIgnoreFiles(h.TestSuite.IgnoreFiles),
 			testcase.WithRunLabels(h.RunLabels),
 			testcase.WithClients(h.Client, h.DiscoveryClient),
 			testcase.WithTemplateVars(h.TemplateVars)))

internal/testcase/case.go

@@ -71,6 +71,13 @@ func WithLogSuppressions(suppressions []string) CaseOption {
 	}
 }
 
+// WithIgnoreFiles sets the list of file patterns to ignore.
+func WithIgnoreFiles(patterns []string) CaseOption {
+	return func(c *Case) {
+		c.ignoreFiles = patterns
+	}
+}
+
 // WithRunLabels sets the run labels.
 func WithRunLabels(runLabels labels.Set) CaseOption {
 	return func(c *Case) {
@@ -118,6 +125,8 @@ type Case struct {
 	logger testutils.Logger
 	// List of log types which should be suppressed.
 	suppressions []string
+	// List of file patterns to ignore when collecting test steps.
+	ignoreFiles []string
 	// Caution: the Vars element of this struct may be shared with other Case objects.
 	templateEnv template.Env
 }
@@ -347,7 +356,7 @@ func (c *Case) getEagerClients() ([]clientWithKubeConfig, error) {
 
 // LoadTestSteps loads all the test steps for a test case.
 func (c *Case) LoadTestSteps() error {
-	testStepFiles, err := files.CollectTestStepFiles(c.dir, c.logger)
+	testStepFiles, err := files.CollectTestStepFiles(c.dir, c.logger, c.ignoreFiles)
 	if err != nil {
 		return err
 	}

internal/utils/files/files.go

@@ -10,7 +10,7 @@ import (
 
 // CollectTestStepFiles collects a map of test steps and their associated files
 // from a directory.
-func CollectTestStepFiles(dir string, logger testutils.Logger) (map[int64][]string, error) {
+func CollectTestStepFiles(dir string, logger testutils.Logger, ignorePatterns []string) (map[int64][]string, error) {
 	testStepFiles := map[int64][]string{}
 
 	files, err := os.ReadDir(dir)
@@ -19,6 +19,23 @@ func CollectTestStepFiles(dir string, logger testutils.Logger) (map[int64][]stri
 	}
 
 	for _, file := range files {
+		// Check if file matches any ignore pattern
+		shouldIgnore := false
+		for _, pattern := range ignorePatterns {
+			matched, err := filepath.Match(pattern, file.Name())
+			if err != nil {
+				logger.Logf("Invalid ignore pattern %q: %v", pattern, err)
+				continue
+			}
+			if matched {
+				shouldIgnore = true
+				break
+			}
+		}
+		if shouldIgnore {
+			continue
+		}
+
 		f := kfile.Parse(file.Name())
 		if f.Type == kfile.TypeUnknown {
 			logger.Logf("Ignoring %q: %v.", file.Name(), f.Error)

internal/utils/files/files_test.go

@@ -48,9 +48,30 @@ func TestCollectTestStepFiles(t *testing.T) {
 		},
 	} {
 		t.Run(tt.path, func(t *testing.T) {
-			testStepFiles, err := CollectTestStepFiles(tt.path, testutils.NewTestLogger(t, tt.path))
+			testStepFiles, err := CollectTestStepFiles(tt.path, testutils.NewTestLogger(t, tt.path), nil)
 			require.NoError(t, err)
 			assert.Equal(t, tt.expected, testStepFiles)
 		})
 	}
 }
+
+func TestCollectTestStepFilesWithIgnorePatterns(t *testing.T) {
+	// Test with ignore patterns - should silently skip README.md
+	ignorePatterns := []string{"README*", "*.md"}
+	logger := testutils.NewTestLogger(t, "test_data/with-overrides-with-ignore")
+	testStepFiles, err := CollectTestStepFiles("test_data/with-overrides", logger, ignorePatterns)
+	require.NoError(t, err)
+
+	// Verify we still get the expected test step files
+	assert.NotEmpty(t, testStepFiles)
+	assert.Contains(t, testStepFiles, int64(0))
+	assert.Contains(t, testStepFiles, int64(1))
+	assert.Contains(t, testStepFiles, int64(2))
+	assert.Contains(t, testStepFiles, int64(3))
+
+	// Test that multiple patterns work
+	multiPatterns := []string{"*.md", "*.txt", "*.rst"}
+	testStepFilesMulti, err := CollectTestStepFiles("test_data/with-overrides", logger, multiPatterns)
+	require.NoError(t, err)
+	assert.Equal(t, testStepFiles, testStepFilesMulti)
+}

pkg/apis/testharness/v1beta1/test_types.go

@@ -97,6 +97,10 @@ type TestSuite struct {
 	// Suppress is used to suppress logs
 	Suppress []string `json:"suppress"`
 
+	// IgnoreFiles is a list of file patterns (e.g., "*.md", "README*") to ignore when collecting test steps.
+	// Files matching these patterns will not generate warnings about not matching the expected test file pattern.
+	IgnoreFiles []string `json:"ignoreFiles"`
+
 	Config *RestConfig `json:"config,omitempty"`
 }