diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 0fe1b35..02a5a8f 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -90,6 +90,7 @@ blocks: - test/e2e/change_in_semaphore_commit_range.rb - test/e2e/change_in_simple.rb - test/e2e/change_in_with_default_branch.rb + - test/e2e/change_in_shallow_merge_commit.rb - test/e2e/change_in_java_vs_javascript_clash.rb - test/e2e/change_in_large_commit_diff.rb - test/e2e/change_in_large_commit_diff_on_default_branch.rb diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..865cc5a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,26 @@ +# Repository Guidelines + +## Project Structure & Module Organization +Semaphore Pipeline Compiler (SPC) is a Go CLI. CLI entrypoint lives in `cmd/cli/main.go`. Core packages sit under `pkg/` (e.g., `pkg/commands`, `pkg/pipelines`, `pkg/when`) and share test doubles in `pkg/.../*_test.go`. Shared YAML schemas are in `schemas/` and feed generated models in `pkg/pipelines/models.go`. The `test/` tree stores fixtures (`test/fixtures/...`) and Ruby helpers for e2e runs, while compiled artifacts are written to `build/`. + +For quick triage or deeper architectural context, consult `DOCUMENTATION.md`—it maps CLI entry points, pipeline passes, and common task recipes. + +## Build, Test, and Development Commands +- `make setup` installs Go module dependencies. +- `make build` or `go build ./...` creates `build/cli`. +- `make dev.run.change-in` rebuilds then exercises the sample pipeline in `test/fixtures/hello.yml`. +- `make lint` runs Revive with `lint.toml`. +- `make test` executes `gotestsum --format short-verbose`. +- `make gen.pipeline.models` regenerates models after updating `schemas/v1.0.yml`. + +## Coding Style & Naming Conventions +Format Go code with `gofmt` (tabs for indentation, trailing newlines) and keep imports curated via `goimports` if available. Revive rules in `lint.toml` enforce receiver naming, error handling, and comment quality—run lint before pushing. Package names stay lowercase, files use snake_case, and exported identifiers should have doc comments connecting them to CLI behaviour. + +## Testing Guidelines +Unit tests live alongside code as `*_test.go` and rely on `gotest.tools/v3` assertions. Favor table-driven cases and mirror production package names. Use fixtures from `test/fixtures/` to cover YAML edge cases. Run `make test` locally before opening a PR. For manual regression checks, run `make dev.run.change-in` or target the Ruby harness with `make e2e TEST=test/e2e.rb`. + +## Commit & Pull Request Guidelines +Recent history (`git log`) shows Conventional-style prefixes (`fix:`, `feat:`, `chore:`) for clarity—continue that pattern and keep subjects under 72 characters. Reference issues or PRs with `(#id)` when applicable. Pull requests should summarise intent, list validation steps (`make test`, `make lint`), attach relevant pipeline outputs or screenshots, and link to Semaphore change requests. + +## Security & Release Tips +Security checks rely on the internal toolbox; run `make check.static` and `make check.deps` when touching dependencies or shell execution paths. Version bumps are automated: use `make tag.patch|minor|major` from a clean main branch, which triggers GoReleaser via Semaphore. Never commit secrets; configuration belongs in environment variables or the security toolbox. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md new file mode 100644 index 0000000..962f8b0 --- /dev/null +++ b/DOCUMENTATION.md @@ -0,0 +1,44 @@ +# SPC Architecture Cheat Sheet + +## Mission & Entry Points +This repository builds the Semaphore Pipeline Compiler (`spc`), a Go CLI that normalizes pipeline YAML before Semaphore runs it. The binary lives under `cmd/cli/main.go` and bootstraps the Cobra command tree in `pkg/cli`. First-level commands include `spc compile`, `spc evaluate change-in`, and `spc list-diff`, each orchestrating pipeline transforms or Git queries. + +## Pipeline Processing Flow +`pkg/pipelines` wraps the raw YAML (loaded through `LoadFromFile`) in a `Pipeline` struct backed by `github.com/Jeffail/gabs`. The default compilation flow called from `spc compile` performs three passes: +1. **Commands file expansion** via `ExtractCommandsFromCommandsFiles` which scans YAML for `commands_file` references and uses `pkg/commands.File` to inline shell commands (supporting relative or repo-absolute paths). +2. **Template evaluation** with `EvaluateTemplates`, which walks the document, discovers `{{ expression }}` placeholders, and delegates substitution to `pkg/templates` utilities while logging progress through `pkg/consolelogger`. +3. **`change_in` evaluation** through `EvaluateChangeIns`, reusing the `pkg/when` parser binary (`whencli`) plus helpers under `pkg/when/changein` to decide whether blocks/promotions should remain. +The resulting structure can be emitted back to JSON (`ToJSON`) or YAML (`ToYAML`). + +## Supporting Packages +- `pkg/cli`: Cobra command definitions, flag helpers (`util.go`), and shared error handling that exits on known `pkg/logs` errors. +- `pkg/logs`: central logging wiring for compiler output files; carries typed errors such as `ErrorChangeInMissingBranch`. +- `pkg/git`: wrappers over Git commands (`diff_set.go`, `git.go`) to compute commit ranges and run `git fetch` / `git diff --name-only`. Used by list-diff and the change_in evaluator. +- `pkg/environment`: adapters around Semaphore environment variables with local fallbacks (e.g., resolves current branch, repo slug, commit range). +- `pkg/templates`, `pkg/when`: pure-Go helpers to detect and evaluate template expressions and `when` syntax; the latter shells out to the external parser. +- `pkg/consolelogger`: indentation-aware stdout writer that gives numbered logs during template and change_in passes. + +## Data & Schemas +Shared pipeline schema definitions sit in `schemas/` (currently `v1.0.yml`). Regenerate the strongly typed Go model in `pkg/pipelines/models.go` with `make gen.pipeline.models`, which converts YAML to JSON (Ruby helper) and pipes it through `schema-generate`. + +## Tests & Fixtures +Unit tests live alongside code (`*_test.go`) and rely on `gotest.tools/v3`. YAML fixtures are under `test/fixtures/`; Ruby-driven end-to-end tests reside in `test/e2e` with a harness in `test/e2e.rb`. Run `make test` for Go coverage, `make dev.run.change-in` to compile the sample hello pipeline, and `make e2e TEST=test/e2e.rb` for smoke testing. + +## Build & Release Tooling +Key Make targets: +- `make setup`: pull Go module dependencies. +- `make build`: compile `build/cli`. +- `make lint`: run Revive using `lint.toml`. +- `make check.static`/`make check.deps`: invoke the security toolbox Docker image. +- `make tag.(patch|minor|major)`: automate semantic version bumps and push release tags (Semaphore + GoReleaser handle artifacts). + +## Common Task Recipes +- **Add a CLI command**: create a file in `pkg/cli`, attach it inside an `init()` function, and expose the handler from existing packages or a new `pkg/...` domain module. +- **Inject a new pipeline transform**: extend `pkg/pipelines.Pipeline` with a method, call it from `compileCmd` in the desired sequence, and provide fixtures under `test/fixtures/` with table-driven tests. +- **Debug change_in issues**: ensure `when` binary is on `$PATH` (`checkWhenInstalled` exits if missing), run `spc list-diff` to verify commit range inputs, and inspect logs written via `--logs`. + +## Operational Notes +- All repo paths resolve relative to the Git root; absolute `commands_file` values are treated as `/path/from/repo/root`. +- Exit behaviour differs for known semantic errors: `ErrorChangeInMissingBranch` and `ErrorInvalidWhenExpression` exit gracefully, everything else panics to highlight unexpected states. +- The CLI assumes Go 1.20+ (see `go.mod`) and uses modules exclusively; no vendoring is committed. +- Keep an eye on `pkg/consolelogger` output when adding evaluators—the logs form part of the artefacts consumed by Semaphore users. diff --git a/pkg/git/git.go b/pkg/git/git.go index c44ad99..79266a9 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -10,23 +10,19 @@ import ( consolelogger "github.com/semaphoreci/spc/pkg/consolelogger" ) -// // Fetching branches from Git remotes has a non-trivial performance impact. // In this structure we store already fetched branches. // If the branch was already fetched, the Fetch action will be a noop. // // Results of fetch are only memorized if there are no errors while fetching. -// var fetchedBranches map[string]string -// // Running and listing diffs has a non-trivial performance impact. // In this structure we store already evaluated git diff outputs. // If the diff is already evaluated for a commitRange range, the Diff action // will be noop. // // Diff results are only memorized if there are no errors. -// var evaluatedDiffs map[string][]string func init() { @@ -89,7 +85,7 @@ const InitialDeepenBy = 100 func unshallow(commitRange string) error { for i := 0; i < MaxUnshallowIterations; i++ { - if canResolveCommitRnage(commitRange) { + if canResolveCommitRange(commitRange) { return nil } @@ -116,7 +112,50 @@ func deepen(numberOfCommits int) error { return err } -func canResolveCommitRnage(commitRange string) bool { +func canResolveCommitRange(commitRange string) bool { + if needsMergeBase(commitRange) && !mergeBaseAvailable(commitRange) { + return false + } + + return diffIsResolvable(commitRange) +} + +func needsMergeBase(commitRange string) bool { + return strings.Contains(commitRange, threeDots) +} + +func mergeBaseAvailable(commitRange string) bool { + base, head, ok := splitThreeDotRange(commitRange) + if !ok { + return false + } + + output, err := run("merge-base", base, head) + if err != nil { + consolelogger.Info(output) + return false + } + + return strings.TrimSpace(output) != "" +} + +func splitThreeDotRange(commitRange string) (string, string, bool) { + parts := strings.Split(commitRange, threeDots) + if len(parts) != 2 { + return "", "", false + } + + base := strings.TrimSpace(parts[0]) + head := strings.TrimSpace(parts[1]) + + if base == "" || head == "" { + return "", "", false + } + + return base, head, true +} + +func diffIsResolvable(commitRange string) bool { output, err := run("diff", "--shortstat", commitRange) if err != nil { consolelogger.Info(output) diff --git a/test/e2e/change_in_shallow_merge_commit.rb b/test/e2e/change_in_shallow_merge_commit.rb new file mode 100644 index 0000000..6801dd0 --- /dev/null +++ b/test/e2e/change_in_shallow_merge_commit.rb @@ -0,0 +1,77 @@ +# rubocop:disable all + +require_relative "../e2e" +require 'yaml' + +pipeline = %{ +version: v1.0 +name: Test +agent: + machine: + type: e1-standard-2 + +blocks: + - name: ChangeIn + run: + when: "change_in('/lib')" + task: + jobs: + - name: Hello + commands: + - echo "Hello World" +} + +origin = TestRepoForChangeIn.setup() + +origin.add_file('.semaphore/semaphore.yml', pipeline) +origin.commit!("Bootstrap") + +origin.add_file("lib/base.txt", "base") +origin.commit!("Base change on master") + +commits_to_cherry_pick = 200 + +origin.run("git branch enterprise") + +commits_to_cherry_pick.times do |index| + origin.add_file("lib/master_history_#{index}.txt", "master #{index}") + origin.commit!("Master history #{index}") +end + +origin.switch_branch("enterprise") +origin.run("git cherry-pick master~#{commits_to_cherry_pick}..master") +origin.run("git merge master --strategy ours --no-edit") + +repo = origin.clone_local_copy(branch: "enterprise", depth: 1, single_branch: true) + +repo.run("git checkout --detach") + +repo.run(%{ + export SEMAPHORE_GIT_SHA=$(git rev-parse HEAD) + export SEMAPHORE_GIT_REF_TYPE=pull-request + export SEMAPHORE_GIT_BRANCH=master + export SEMAPHORE_GIT_PR_BRANCH=enterprise + + #{spc} compile \ + --input .semaphore/semaphore.yml \ + --output /tmp/output.yml \ + --logs /tmp/logs.yml +}) + +assert_eq(YAML.load_file('/tmp/output.yml'), YAML.load(%{ +version: v1.0 +name: Test +agent: + machine: + type: e1-standard-2 + +blocks: + - name: ChangeIn + run: + when: "false" + task: + jobs: + - name: Hello + commands: + - echo "Hello World" +})) diff --git a/test/e2e_utils/test_repo_for_change_in.rb b/test/e2e_utils/test_repo_for_change_in.rb index d8c4432..563fa3f 100644 --- a/test/e2e_utils/test_repo_for_change_in.rb +++ b/test/e2e_utils/test_repo_for_change_in.rb @@ -89,12 +89,14 @@ def commit!(message) def clone_local_copy(options = {}) clone_path = "/tmp/test-repo" + branch = options[:branch] || "master" + depth = options.fetch(:depth, 10) + single_branch = options[:single_branch] ? "--single-branch" : "" + depth_flag = depth ? "--depth #{depth}" : "" system "rm -rf #{clone_path}" - system "git clone file:///#{@path} --depth 10 --branch #{options[:branch]} #{clone_path}" + system "git clone file:///#{@path} #{depth_flag} #{single_branch} --branch #{branch} #{clone_path}" - repo = TestRepoForChangeIn.new(clone_path) - - repo + TestRepoForChangeIn.new(clone_path) end end