Add (glob), (only-globs), and (except-globs) for fine-grained caching via include/exclude glob patterns

.gitignore

@@ -1,4 +1,5 @@
 # binaries are the path to darkness
+*.tar
 *.tgz
 *.zip
 *.tar

bass/bass.bass

@@ -10,38 +10,19 @@
 (defn checkout [sha]
   (git:github/vito/bass/sha/ sha))
 
-(provide [subgit git-submodules]
-  ; produces a fresh checkout of the last commit that affected the given paths
-  (defn subgit [src & paths]
-    (git:checkout (remote src) (latest-commit src paths)))
-
-  (defn git-out [repo thunk]
-    (-> thunk
-        (with-image (linux/alpine/git))
-        (with-dir repo)
-        (read :raw)
-        next
-        trim))
-
-  (defn remote [src]
-    (git-out src ($ git remote get-url origin)))
-
-  (defn latest-commit [src paths]
-    (git-out src ($ git log -n1 "--pretty=format:%H" -- & $paths)))
-
-  (defn git-submodules [src]
-    (map (fn [[_ path]] (string->dir path))
-      (-> ($ git config --file src/.gitmodules --get-regexp path)
-          (with-image (linux/alpine/git))
-          (read :unix-table)
-          take-all))))
+(defn git-submodules [src]
+  (map (fn [[_ path]] (string->dir path))
+       (-> ($ git config --file src/.gitmodules --get-regexp path)
+           (with-image (linux/alpine/git))
+           (read :unix-table)
+           take-all)))
 
 (provide [deps deps+go]
   ; monolithic image containing dependencies for building and testing
   (defn deps [src]
     (oci-load
       (nix:result
-        (cd (subgit src ./nix/ ./flake.nix ./flake.lock ./default.nix)
+        (cd (glob src ./nix/ ./flake.nix ./flake.lock ./default.nix)
           ($ nix build ".#depsImage"))
         ./deps-image.tar)
       {:os "linux"}))
@@ -73,19 +54,32 @@
 
   (defn with-deps-and-shims [thunk src]
     (-> thunk
-        (with-mount src ./)
+        (with-mount
+          (glob src
+                ./**/*.go
+                ./**/go.mod
+                ./**/go.sum
+                ./std/*.bass
+                ./pkg/bass/testdata/**/*
+                ./pkg/runtimes/testdata/**/*
+                ./pkg/lsp/testdata/**/*
+                ./Makefile)
+          ./)
         (with-mount (make-shims src) ./pkg/runtimes/bin/)
         with-go-cache
         (with-image (deps+go src))))
 
   ; returns a thunk with the make targets built into the output directory, as
   ; an overlay of src
   (defn make-shims [src]
-    (let [submodules (git-submodules src)]
-      (-> (from (deps+go src)
-            (cd (subgit src ./pkg/runtimes/shim/ ./Makefile ./go.mod ./go.sum & submodules)
-              ($ make -j shims)))
-          (subpath ./pkg/runtimes/bin/))))
+    (-> (from (deps+go src)
+          (cd (glob src
+                    ./pkg/runtimes/shim/
+                    ./Makefile
+                    ./go.mod
+                    ./go.sum)
+            ($ make -j shims)))
+        (subpath ./pkg/runtimes/bin/)))
 
   ; creates an archive appropriate for the given platform
   (defn archive [src out os arch]
@@ -177,14 +171,17 @@
 
   ; returns a thunk that will run the tests and return cover.html
   (defn coverage [src testflags]
-    (from (with-bass-and-buildkitd src)
-      (with-go-cache
-        ($ gotestsum --format testname --no-color=false --jsonfile ./tests.log
-           --
-           -cover
-           -coverprofile ./cover.out
-           -covermode count
-           & $testflags))
+    (from
+      (-> ($ gotestsum --format testname --no-color=false --jsonfile ./tests.log
+             --
+             -cover
+             -coverprofile ./cover.out
+             -coverpkg ./...
+             -covermode count
+             & $testflags)
+          with-go-cache
+          (with-env {:SKIP_DAGGER_TESTS "true"})
+          (with-bass-and-buildkitd src))
 
       ; report slow tests
       ($ gotestsum tool slowest --jsonfile ./tests.log --threshold "500ms")
@@ -209,10 +206,10 @@
   ; Returns a memory-backed file, so this can be shimmed in-place.
   (defn undo-wordwrap [src file]
     (mkfile ./wide.txt
-      (-> ($ markdownfmt $file)
-          (with-image (deps+go src))
-          (read :raw)
-          next)))
+            (-> ($ markdownfmt $file)
+                (with-image (deps+go src))
+                (read :raw)
+                next)))
 
   ; returns the path to the release notes for the given version
   (defn release-notes [src version]

bass/test

@@ -6,8 +6,11 @@
 ;
 ; Emits code coverage to *stdout*.
 (defn main testflags
-  (for [{:src src} *stdin*]
-    (let [args (if (empty? testflags) ["./..."] testflags)
-          tests (bass:tests src args)]
-      (log "running tests")
-      (run tests))))
+  (for [{:src src
+         (:out *dir*) out} *stdin*]
+    (log "running tests")
+    (write
+      (subpath
+        (bass:coverage src (if (empty? testflags) ["./..."] testflags))
+        ./cover.html)
+      out/cover.html)))

demos/hello.bass

@@ -1,5 +1,5 @@
 (defn main [lines]
   (-> ($ sh -c "for i in $(seq 1 $0); do echo line $i; sleep 0.5; done" $lines)
-      (with-label :start (now 0))
+      (with-env {:START (now 0)})
       (with-image (linux/alpine))
       run))

demos/multi-fail.bass

@@ -1,7 +1,9 @@
 (defn echo-sleep-exit [msg seconds exit-code]
   (subpath
     (from (linux/alpine)
-      (with-label ($ sleep (str seconds)) :at (now 0))
+      (with-env
+        ($ sleep (str seconds))
+        {:AT (now 0)})
       ($ sh -c (str "echo \"$0\"; exit " exit-code) $msg))
     ./))
 

demos/speedtest.bass

@@ -8,7 +8,7 @@
 (defn speedtest []
   (from (speedtest-cli)
     (-> ($ speedtest --accept-license --format json-pretty)
-        (with-label :at (now 0)))))
+        (with-env {:NOW (now 0)}))))
 
 (defn main []
   (let [results (next (read (speedtest) :json))]

docs/lit/guide.lit

@@ -59,21 +59,13 @@ for common tasks. If you'd like to learn the language, see \reference{bassics}.
       Thunks are cached forever. They can be cleared with \code{bass --prune},
       but this should only be necessary for regaining disk space.
     }{
-      To influence caching, use \b{with-label} to stamp thunks with arbitrary
-      data. Two thunks that differ only in labels will be cached independently.
+      If you want to run a thunk multiple times, just set a different value as
+      an environment variable. Tip: use \b{now} to control cache granularity.
     }{{{
-      (run (with-label
+      (run (with-env
              (from (linux/alpine)
-               ($ echo "Hello, world!"))
-             :foo "bar"))
-    }}}{
-      Tip: to avoid deep nesting like above, consider the alternative \b{->}
-      form.
-    }{{{
-      (-> ($ echo "Hello, world!")
-          (with-image (linux/alpine))
-          (with-label :foo "bar")
-          run)
+               ($ echo "Hi again!"))
+             {:MINUTE (now 60)}))
     }}}
   }
 
@@ -181,8 +173,8 @@ for common tasks. If you'd like to learn the language, see \reference{bassics}.
 
       (defn counter [tag]
         (from (linux/alpine)
-          (-> ($ sh -c "echo x >> /var/cache/file; cat /var/cache/file | wc -l")
-              (with-label :tag tag)
+          (-> ($ sh -c "echo $0 >> /var/cache/file; cat /var/cache/file | wc -l"
+                 $tag)
               (with-mount my-cache /var/cache/))))
 
       (defn count [tag]

docs/lit/index.lit

@@ -81,7 +81,7 @@ release}{https://github.com/vito/bass/releases/latest} and skim the
   (defn ls-remote [repo ref & timestamp]
     (-> ($ git ls-remote $repo $ref)
         (with-image *git-image*)
-        (with-label :at (now 60)) ; rerun every minute
+        (with-env {:MINUTE (now 60)}) ; rerun every minute
         (read :unix-table) ; line and space separated table output
         next    ; first row   : <ref> <sha>
         first)) ; first column: <ref>

pkg/bass/binding_test.go

@@ -167,17 +167,17 @@ func TestBinding(t *testing.T) {
 		},
 		{
 			Name:     "dir match",
-			Bindable: bass.DirPath{"foo"},
-			Value:    bass.DirPath{"foo"},
+			Bindable: bass.NewDirPath("foo"),
+			Value:    bass.NewDirPath("foo"),
 			Bindings: bass.Bindings{},
 		},
 		{
 			Name:     "dir mismatch",
-			Bindable: bass.DirPath{"foo"},
-			Value:    bass.DirPath{"bar"},
+			Bindable: bass.NewDirPath("foo"),
+			Value:    bass.NewDirPath("bar"),
 			Err: bass.BindMismatchError{
-				Need: bass.DirPath{"foo"},
-				Have: bass.DirPath{"bar"},
+				Need: bass.NewDirPath("foo"),
+				Have: bass.NewDirPath("bar"),
 			},
 		},
 		{

pkg/bass/encoding_test.go

@@ -51,7 +51,12 @@ var encodable = []bass.Value{
 			bass.String("hello"),
 		),
 	}.Scope(),
-	bass.DirPath{"directory-path"},
+	bass.NewDirPath("directory-path"),
+	bass.GlobDir(
+		"directory-path",
+		[]string{"*.bash"},
+		[]string{"*.bass"},
+	),
 	bass.FilePath{"file-path"},
 	bass.CommandPath{"command-path"},
 	bass.NewHostPath("./", bass.ParseFileOrDirPath("foo")),
@@ -416,7 +421,7 @@ var validThunkMountSources = []bass.ThunkMountSource{
 		Cache: &bass.CachePath{
 			ID: "some-cache",
 			Path: bass.FileOrDirPath{
-				Dir: &bass.DirPath{"cache/dir"},
+				Dir: &bass.DirPath{Path: "cache/dir"},
 			},
 		},
 	},

pkg/bass/enums_test.go

@@ -30,7 +30,7 @@ func TestEnums(t *testing.T) {
 			Enum: &bass.FileOrDirPath{},
 			Valid: []bass.Value{
 				bass.FilePath{"file"},
-				bass.DirPath{"dir"},
+				bass.NewDirPath("dir"),
 			},
 			Invalid: []bass.Value{
 				bass.CommandPath{"cmd"},
@@ -39,15 +39,15 @@ func TestEnums(t *testing.T) {
 		{
 			Enum: &bass.ThunkDir{},
 			Valid: []bass.Value{
-				bass.DirPath{"dir"},
+				bass.NewDirPath("dir"),
 				bass.ThunkPath{
 					Thunk: bass.Thunk{
 						Args: []bass.Value{
 							bass.CommandPath{"cmd"},
 						},
 					},
 					Path: bass.FileOrDirPath{
-						Dir: &bass.DirPath{"dir"},
+						Dir: &bass.DirPath{Path: "dir"},
 					},
 				},
 			},

pkg/bass/filesystem_path.go

@@ -2,7 +2,6 @@ package bass
 
 import (
 	"fmt"
-	"path"
 	"path/filepath"
 	"strings"
 
@@ -52,23 +51,6 @@ func ParseFileOrDirPath(arg string) FileOrDirPath {
 	return fod
 }
 
-func NewFilePath(p string) FilePath {
-	if p == "" {
-		panic("empty file path")
-	}
-
-	return FilePath{
-		Path: path.Clean(p),
-	}
-}
-
-func NewDirPath(p string) DirPath {
-	return DirPath{
-		// trim suffix left behind from Clean returning "/"
-		Path: strings.TrimSuffix(path.Clean(p), "/"),
-	}
-}
-
 func IsPathLike(arg string) bool {
 	return strings.HasPrefix(arg, "./") ||
 		strings.HasPrefix(arg, "/") ||
@@ -99,6 +81,11 @@ func NewFileOrDirPath(path FilesystemPath) FileOrDirPath {
 	panic(fmt.Sprintf("absurd: non-File or Dir FilesystemPath: %T", path))
 }
 
+// String calls String on whichever value is present.
+func (path FileOrDirPath) String() string {
+	return path.FilesystemPath().String()
+}
+
 // Slash calls Slash on whichever value is present.
 func (path FileOrDirPath) Slash() string {
 	return path.FilesystemPath().Slash()
@@ -193,3 +180,39 @@ func (value *FileOrDirPath) UnmarshalJSON(b []byte) error {
 
 	return value.UnmarshalProto(msg)
 }
+
+var _ Globbable = FileOrDirPath{}
+
+func (value FileOrDirPath) Includes() []string {
+	if value.Dir != nil {
+		return value.Dir.Includes()
+	}
+
+	return nil
+}
+
+func (value FileOrDirPath) Excludes() []string {
+	if value.Dir != nil {
+		return value.Dir.Excludes()
+	}
+
+	return nil
+}
+
+func (value FileOrDirPath) WithInclude(paths ...string) Globbable {
+	if value.Dir != nil {
+		globbed := value.Dir.WithInclude(paths...).(DirPath)
+		value.Dir = &globbed
+	}
+
+	return value
+}
+
+func (value FileOrDirPath) WithExclude(paths ...string) Globbable {
+	if value.Dir != nil {
+		globbed := value.Dir.WithExclude(paths...).(DirPath)
+		value.Dir = &globbed
+	}
+
+	return value
+}