diff --git a/README.md b/README.md index 8f042cb..ba9d848 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,21 @@ One of the following is required. step to fetch metadata about the lock is necessary before a `put` step can check the existence of the lock. +* `check_unclaimed`: If set, we will check for an existing unclaimed lock in the pool and + wait until it becomes claimed. + + * If there is an existing lock in unclaimed state: wait until lock is unclaimed + * If there is an existing lock in claimed state: no-op + * If no lock exists: fail + + The purpose is to simply block until a given lock in a pool is moved from an + unclaimed state to a claimed state. This functionality allows us to build + dependencies between disparate pipelines without the need to `acquire` locks. + + Note: the lock must be present to perform a check. In other words, a `get` + step to fetch metadata about the lock is necessary before a `put` step can + check the existence of the lock. + ## Example Concourse Configuration The following example pipeline models acquiring, passing through, and releasing diff --git a/cmd/out/main.go b/cmd/out/main.go index 0efefcc..be3a7e5 100644 --- a/cmd/out/main.go +++ b/cmd/out/main.go @@ -110,6 +110,15 @@ func main() { } } + if request.Params.CheckUnclaimed != "" { + lock = request.Params.CheckUnclaimed + lockPath := filepath.Join(sourceDir, request.Params.CheckUnclaimed) + lock, version, err = lockPool.CheckUnclaimedLock(lockPath) + if err != nil { + fatal("checking unclaimed lock", err) + } + } + err = json.NewEncoder(os.Stdout).Encode(out.OutResponse{ Version: version, Metadata: []out.MetadataPair{ diff --git a/go.mod b/go.mod index da5c6f0..2fd27ca 100644 --- a/go.mod +++ b/go.mod @@ -13,10 +13,15 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect + github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 // indirect go.uber.org/automaxprocs v1.6.0 // indirect + golang.org/x/mod v0.26.0 // indirect golang.org/x/net v0.42.0 // indirect + golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect golang.org/x/tools v0.35.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +tool github.com/maxbrunsfeld/counterfeiter/v6 diff --git a/go.sum b/go.sum index c6f632d..edf3f91 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU= +github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= @@ -22,12 +24,18 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= +github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= @@ -37,7 +45,7 @@ golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/integration/out_test.go b/integration/out_test.go index 6110475..d393457 100644 --- a/integration/out_test.go +++ b/integration/out_test.go @@ -132,7 +132,7 @@ func itWorksWithBranch(branchName string) { It("complains about it", func() { errorMessages := string(session.Err.Contents()) - Ω(errorMessages).Should(ContainSubstring("invalid payload (missing acquire, release, remove, claim, add, add_claimed, update, or check)")) + Ω(errorMessages).Should(ContainSubstring("invalid payload (missing acquire, release, remove, claim, add, add_claimed, update, check, or check_unclaimed")) }) }) }) diff --git a/out/fakes/fake_lock_handler.go b/out/fakes/fake_lock_handler.go index 617037c..ddabc22 100644 --- a/out/fakes/fake_lock_handler.go +++ b/out/fakes/fake_lock_handler.go @@ -48,6 +48,19 @@ type FakeLockHandler struct { result1 string result2 error } + CheckUnclaimedLockStub func(string) (string, error) + checkUnclaimedLockMutex sync.RWMutex + checkUnclaimedLockArgsForCall []struct { + arg1 string + } + checkUnclaimedLockReturns struct { + result1 string + result2 error + } + checkUnclaimedLockReturnsOnCall map[int]struct { + result1 string + result2 error + } ClaimLockStub func(string) (string, error) claimLockMutex sync.RWMutex claimLockArgsForCall []struct { @@ -330,6 +343,70 @@ func (fake *FakeLockHandler) CheckLockReturnsOnCall(i int, result1 string, resul }{result1, result2} } +func (fake *FakeLockHandler) CheckUnclaimedLock(arg1 string) (string, error) { + fake.checkUnclaimedLockMutex.Lock() + ret, specificReturn := fake.checkUnclaimedLockReturnsOnCall[len(fake.checkUnclaimedLockArgsForCall)] + fake.checkUnclaimedLockArgsForCall = append(fake.checkUnclaimedLockArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.CheckUnclaimedLockStub + fakeReturns := fake.checkUnclaimedLockReturns + fake.recordInvocation("CheckUnclaimedLock", []interface{}{arg1}) + fake.checkUnclaimedLockMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeLockHandler) CheckUnclaimedLockCallCount() int { + fake.checkUnclaimedLockMutex.RLock() + defer fake.checkUnclaimedLockMutex.RUnlock() + return len(fake.checkUnclaimedLockArgsForCall) +} + +func (fake *FakeLockHandler) CheckUnclaimedLockCalls(stub func(string) (string, error)) { + fake.checkUnclaimedLockMutex.Lock() + defer fake.checkUnclaimedLockMutex.Unlock() + fake.CheckUnclaimedLockStub = stub +} + +func (fake *FakeLockHandler) CheckUnclaimedLockArgsForCall(i int) string { + fake.checkUnclaimedLockMutex.RLock() + defer fake.checkUnclaimedLockMutex.RUnlock() + argsForCall := fake.checkUnclaimedLockArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLockHandler) CheckUnclaimedLockReturns(result1 string, result2 error) { + fake.checkUnclaimedLockMutex.Lock() + defer fake.checkUnclaimedLockMutex.Unlock() + fake.CheckUnclaimedLockStub = nil + fake.checkUnclaimedLockReturns = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeLockHandler) CheckUnclaimedLockReturnsOnCall(i int, result1 string, result2 error) { + fake.checkUnclaimedLockMutex.Lock() + defer fake.checkUnclaimedLockMutex.Unlock() + fake.CheckUnclaimedLockStub = nil + if fake.checkUnclaimedLockReturnsOnCall == nil { + fake.checkUnclaimedLockReturnsOnCall = make(map[int]struct { + result1 string + result2 error + }) + } + fake.checkUnclaimedLockReturnsOnCall[i] = struct { + result1 string + result2 error + }{result1, result2} +} + func (fake *FakeLockHandler) ClaimLock(arg1 string) (string, error) { fake.claimLockMutex.Lock() ret, specificReturn := fake.claimLockReturnsOnCall[len(fake.claimLockArgsForCall)] @@ -766,6 +843,8 @@ func (fake *FakeLockHandler) Invocations() map[string][][]interface{} { defer fake.broadcastLockPoolMutex.RUnlock() fake.checkLockMutex.RLock() defer fake.checkLockMutex.RUnlock() + fake.checkUnclaimedLockMutex.RLock() + defer fake.checkUnclaimedLockMutex.RUnlock() fake.claimLockMutex.RLock() defer fake.claimLockMutex.RUnlock() fake.grabAvailableLockMutex.RLock() diff --git a/out/git_lock_handler.go b/out/git_lock_handler.go index 2b13e75..cf92404 100644 --- a/out/git_lock_handler.go +++ b/out/git_lock_handler.go @@ -214,6 +214,28 @@ func (glh *GitLockHandler) CheckLock(lockName string) (string, error) { return string(ref), nil } +func (glh *GitLockHandler) CheckUnclaimedLock(lockName string) (string, error) { + glh.checkOnly = true + + // Wait if unclaimed + _, err := os.ReadFile(filepath.Join(glh.dir, glh.Source.Pool, "unclaimed", lockName)) + if err == nil { + return "", ErrLockActive + } + + _, err = glh.git("pull", "origin", glh.Source.Branch) + if err != nil { + return "", err + } + + ref, err := glh.git("rev-parse", "HEAD") + if err != nil { + return "", err + } + + return string(ref), nil +} + func (glh *GitLockHandler) Setup() error { var err error diff --git a/out/lock_pool.go b/out/lock_pool.go index 6a92ed7..406ef1a 100644 --- a/out/lock_pool.go +++ b/out/lock_pool.go @@ -29,7 +29,9 @@ func NewLockPool(source Source, output io.Writer) LockPool { return lockPool } -//go:generate counterfeiter . LockHandler +//go:generate go tool counterfeiter -generate + +//counterfeiter:generate -o ./fakes . LockHandler type LockHandler interface { GrabAvailableLock() (lock string, version string, err error) UnclaimLock(lock string) (version string, err error) @@ -38,6 +40,7 @@ type LockHandler interface { ClaimLock(lock string) (version string, err error) UpdateLock(lock string, contents []byte) (version string, err error) CheckLock(lock string) (version string, err error) + CheckUnclaimedLock(lock string) (version string, err error) Setup() error BroadcastLockPool() ([]byte, error) @@ -304,6 +307,43 @@ func (lp *LockPool) CheckLock(inDir string) (string, Version, error) { }, nil } +func (lp *LockPool) CheckUnclaimedLock(inDir string) (string, Version, error) { + nameFileContents, err := os.ReadFile(filepath.Join(inDir, "name")) + if err != nil { + return "", Version{}, fmt.Errorf("could not read the file name of your lock: %s", err) + } + lockName := strings.TrimSpace(string(nameFileContents)) + + fmt.Fprintf(lp.Output, "checking lock: %s in pool: %s\n", lockName, lp.Source.Pool) + fmt.Fprintf(lp.Output, "waiting for lock to become claimed\n") + + var ref string + + err = lp.performRobustAction(func() (bool, error) { + var err error + ref, err = lp.LockHandler.CheckUnclaimedLock(lockName) + + if err == ErrLockActive { + fmt.Fprint(lp.Output, ".") + return true, nil + } + + if err != nil { + fmt.Fprintf(lp.Output, "failed to check the lock: %s! (err: %s) retrying...\n", lockName, err) + return true, nil + } + return false, nil + }) + + if err != nil { + return "", Version{}, err + } + + return lockName, Version{ + Ref: strings.TrimSpace(ref), + }, nil +} + func (lp *LockPool) performRobustAction(action func() (bool, error)) error { err := lp.LockHandler.Setup() if err != nil { diff --git a/out/lock_pool_test.go b/out/lock_pool_test.go index 73d6f7c..cd585b4 100644 --- a/out/lock_pool_test.go +++ b/out/lock_pool_test.go @@ -927,4 +927,72 @@ var _ = Describe("Lock Pool", func() { }) }) }) + + Context("Checking an unclaimed lock", func() { + var lockDir string + + BeforeEach(func() { + var err error + lockDir, err = os.MkdirTemp("", "lock-dir") + Ω(err).ShouldNot(HaveOccurred()) + + }) + + AfterEach(func() { + err := os.RemoveAll(lockDir) + Ω(err).ShouldNot(HaveOccurred()) + }) + + Context("when a name file doesn't exist", func() { + It("returns an error", func() { + _, _, err := lockPool.CheckUnclaimedLock(lockDir) + Ω(err).Should(HaveOccurred()) + }) + }) + + Context("when a name file does exist", func() { + BeforeEach(func() { + err := os.WriteFile(filepath.Join(lockDir, "name"), []byte("some-lock"), 0755) + Ω(err).ShouldNot(HaveOccurred()) + }) + + Context("when setup fails", func() { + BeforeEach(func() { + fakeLockHandler.SetupReturns(errors.New("some-error")) + }) + + It("returns an error", func() { + _, _, err := lockPool.CheckUnclaimedLock(lockDir) + Ω(err).Should(HaveOccurred()) + }) + }) + + Context("when setup succeeds", func() { + Context("when the lock is claimed", func() { + BeforeEach(func() { + fakeLockHandler.CheckUnclaimedLockReturns("some-ref", nil) + }) + + It("bypasses broadcasting to the lock pool", func() { + _, _, err := lockPool.CheckUnclaimedLock(lockDir) + Ω(err).ShouldNot(HaveOccurred()) + + Ω(fakeLockHandler.BroadcastLockPoolCallCount()).Should(Equal(1)) + }) + + Context("when broadcasting succeeds", func() { + It("returns the lockname, and a version", func() { + lockName, version, err := lockPool.CheckUnclaimedLock(lockDir) + + Ω(err).ShouldNot(HaveOccurred()) + Ω(lockName).Should(Equal("some-lock")) + Ω(version).Should(Equal(out.Version{ + Ref: "some-ref", + })) + }) + }) + }) + }) + }) + }) }) diff --git a/out/out_request.go b/out/out_request.go index e0339b4..0bd9408 100644 --- a/out/out_request.go +++ b/out/out_request.go @@ -47,14 +47,15 @@ func (s *Source) UnmarshalJSON(b []byte) error { } type OutParams struct { - Release string `json:"release"` - Acquire bool `json:"acquire"` - Add string `json:"add"` - AddClaimed string `json:"add_claimed"` - Remove string `json:"remove"` - Claim string `json:"claim"` - Update string `json:"update"` - Check string `json:"check"` + Release string `json:"release"` + Acquire bool `json:"acquire"` + Add string `json:"add"` + AddClaimed string `json:"add_claimed"` + Remove string `json:"remove"` + Claim string `json:"claim"` + Update string `json:"update"` + Check string `json:"check"` + CheckUnclaimed string `json:"check_unclaimed"` } func (request OutRequest) Validate() []string { @@ -79,8 +80,9 @@ func (request OutRequest) Validate() []string { request.Params.Remove == "" && request.Params.Claim == "" && request.Params.Update == "" && - request.Params.Check == "" { - errorMessages = append(errorMessages, "invalid payload (missing acquire, release, remove, claim, add, add_claimed, update, or check)") + request.Params.Check == "" && + request.Params.CheckUnclaimed == "" { + errorMessages = append(errorMessages, "invalid payload (missing acquire, release, remove, claim, add, add_claimed, update, check, or check_unclaimed)") } return errorMessages