diff --git a/.bazelignore b/.bazelignore index 166c90d9..23f25aa1 100644 --- a/.bazelignore +++ b/.bazelignore @@ -28,3 +28,8 @@ img_tool/bazel-bin img_tool/bazel-img_tool img_tool/bazel-out img_tool/bazel-testlogs +pull_tool/bazel-bin +pull_tool/bazel-out +pull_tool/bazel-pull_tool +pull_tool/bazel-rules_img_pull_tool +pull_tool/bazel-testlogs diff --git a/.bazelrc b/.bazelrc index 5ed8f63d..0aa08fc2 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,9 +1,10 @@ # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=e2e/cc,e2e/cc/base,e2e/cc/custom_standard_library,e2e/cc/patches,e2e/cc/platform,e2e/generic,e2e/generic/build_settings,e2e/generic/extend,e2e/generic/load,e2e/generic/multi_deploy,e2e/generic/platform,e2e/go,e2e/go/customization,e2e/go/image,e2e/go/multiarch,e2e/js,e2e/js/app,e2e/js/platform,e2e/python,e2e/python/platform,e2e/python/requirements,e2e/workspace,img_tool,img_tool/cmd,img_tool/cmd/bes,img_tool/cmd/compress,img_tool/cmd/deploy,img_tool/cmd/dockersave,img_tool/cmd/downloadblob,img_tool/cmd/expandtemplate,img_tool/cmd/img,img_tool/cmd/index,img_tool/cmd/layer,img_tool/cmd/layermeta,img_tool/cmd/manifest,img_tool/cmd/ocilayout,img_tool/cmd/pull,img_tool/cmd/push,img_tool/cmd/registry,img_tool/cmd/validate,img_tool/cmd/validate/layer-presence,img_tool/pkg/api,img_tool/pkg/auth/credential,img_tool/pkg/auth/grpcheaderinterceptor,img_tool/pkg/auth/protohelper,img_tool/pkg/auth/registry,img_tool/pkg/cas,img_tool/pkg/compress,img_tool/pkg/compress/util,img_tool/pkg/containerd,img_tool/pkg/contentmanifest,img_tool/pkg/deployvfs,img_tool/pkg/digestfs,img_tool/pkg/docker,img_tool/pkg/fileopener,img_tool/pkg/load,img_tool/pkg/proto/bazel,img_tool/pkg/proto/bazel/src/main/java/com/google/devtools/build/lib/buildeventstream,img_tool/pkg/proto/bazel/src/main/java/com/google/devtools/build/lib/packages/metrics,img_tool/pkg/proto/bazel/src/main/protobuf,img_tool/pkg/proto/blobcache,img_tool/pkg/proto/build_event_service,img_tool/pkg/proto/remote-apis,img_tool/pkg/proto/remote-apis/build/bazel/remote/execution/v2,img_tool/pkg/proto/remote-apis/build/bazel/semver,img_tool/pkg/push,img_tool/pkg/serve/bes,img_tool/pkg/serve/bes/syncer,img_tool/pkg/serve/blobcache,img_tool/pkg/serve/registry,img_tool/pkg/serve/registry/reapi,img_tool/pkg/serve/registry/s3,img_tool/pkg/serve/registry/upstream,img_tool/pkg/tarcas,img_tool/pkg/tree,img_tool/pkg/tree/merkle,img_tool/pkg/tree/runfiles,img_tool/pkg/tree/treeartifact,img_tool/toolchain,img_tool/tools -query --deleted_packages=e2e/cc,e2e/cc/base,e2e/cc/custom_standard_library,e2e/cc/patches,e2e/cc/platform,e2e/generic,e2e/generic/build_settings,e2e/generic/extend,e2e/generic/load,e2e/generic/multi_deploy,e2e/generic/platform,e2e/go,e2e/go/customization,e2e/go/image,e2e/go/multiarch,e2e/js,e2e/js/app,e2e/js/platform,e2e/python,e2e/python/platform,e2e/python/requirements,e2e/workspace,img_tool,img_tool/cmd,img_tool/cmd/bes,img_tool/cmd/compress,img_tool/cmd/deploy,img_tool/cmd/dockersave,img_tool/cmd/downloadblob,img_tool/cmd/expandtemplate,img_tool/cmd/img,img_tool/cmd/index,img_tool/cmd/layer,img_tool/cmd/layermeta,img_tool/cmd/manifest,img_tool/cmd/ocilayout,img_tool/cmd/pull,img_tool/cmd/push,img_tool/cmd/registry,img_tool/cmd/validate,img_tool/cmd/validate/layer-presence,img_tool/pkg/api,img_tool/pkg/auth/credential,img_tool/pkg/auth/grpcheaderinterceptor,img_tool/pkg/auth/protohelper,img_tool/pkg/auth/registry,img_tool/pkg/cas,img_tool/pkg/compress,img_tool/pkg/compress/util,img_tool/pkg/containerd,img_tool/pkg/contentmanifest,img_tool/pkg/deployvfs,img_tool/pkg/digestfs,img_tool/pkg/docker,img_tool/pkg/fileopener,img_tool/pkg/load,img_tool/pkg/proto/bazel,img_tool/pkg/proto/bazel/src/main/java/com/google/devtools/build/lib/buildeventstream,img_tool/pkg/proto/bazel/src/main/java/com/google/devtools/build/lib/packages/metrics,img_tool/pkg/proto/bazel/src/main/protobuf,img_tool/pkg/proto/blobcache,img_tool/pkg/proto/build_event_service,img_tool/pkg/proto/remote-apis,img_tool/pkg/proto/remote-apis/build/bazel/remote/execution/v2,img_tool/pkg/proto/remote-apis/build/bazel/semver,img_tool/pkg/push,img_tool/pkg/serve/bes,img_tool/pkg/serve/bes/syncer,img_tool/pkg/serve/blobcache,img_tool/pkg/serve/registry,img_tool/pkg/serve/registry/reapi,img_tool/pkg/serve/registry/s3,img_tool/pkg/serve/registry/upstream,img_tool/pkg/tarcas,img_tool/pkg/tree,img_tool/pkg/tree/merkle,img_tool/pkg/tree/runfiles,img_tool/pkg/tree/treeartifact,img_tool/toolchain,img_tool/tools +build --deleted_packages=e2e/cc,e2e/cc/base,e2e/cc/custom_standard_library,e2e/cc/patches,e2e/cc/platform,e2e/generic,e2e/generic/build_settings,e2e/generic/extend,e2e/generic/load,e2e/generic/multi_deploy,e2e/generic/platform,e2e/go,e2e/go/customization,e2e/go/image,e2e/go/multiarch,e2e/js,e2e/js/app,e2e/js/platform,e2e/python,e2e/python/platform,e2e/python/requirements,e2e/workspace,img_tool,img_tool/cmd,img_tool/cmd/bes,img_tool/cmd/compress,img_tool/cmd/deploy,img_tool/cmd/dockersave,img_tool/cmd/downloadblob,img_tool/cmd/expandtemplate,img_tool/cmd/img,img_tool/cmd/index,img_tool/cmd/layer,img_tool/cmd/layermeta,img_tool/cmd/manifest,img_tool/cmd/ocilayout,img_tool/cmd/push,img_tool/cmd/registry,img_tool/cmd/validate,img_tool/cmd/validate/layer-presence,img_tool/pkg/api,img_tool/pkg/auth/credential,img_tool/pkg/auth/grpcheaderinterceptor,img_tool/pkg/auth/protohelper,img_tool/pkg/auth/registry,img_tool/pkg/cas,img_tool/pkg/compress,img_tool/pkg/compress/util,img_tool/pkg/containerd,img_tool/pkg/contentmanifest,img_tool/pkg/deployvfs,img_tool/pkg/digestfs,img_tool/pkg/docker,img_tool/pkg/fileopener,img_tool/pkg/load,img_tool/pkg/proto/bazel,img_tool/pkg/proto/bazel/src/main/java/com/google/devtools/build/lib/buildeventstream,img_tool/pkg/proto/bazel/src/main/java/com/google/devtools/build/lib/packages/metrics,img_tool/pkg/proto/bazel/src/main/protobuf,img_tool/pkg/proto/blobcache,img_tool/pkg/proto/build_event_service,img_tool/pkg/proto/remote-apis,img_tool/pkg/proto/remote-apis/build/bazel/remote/execution/v2,img_tool/pkg/proto/remote-apis/build/bazel/semver,img_tool/pkg/push,img_tool/pkg/serve/bes,img_tool/pkg/serve/bes/syncer,img_tool/pkg/serve/blobcache,img_tool/pkg/serve/registry,img_tool/pkg/serve/registry/reapi,img_tool/pkg/serve/registry/s3,img_tool/pkg/serve/registry/upstream,img_tool/pkg/tarcas,img_tool/pkg/tree,img_tool/pkg/tree/merkle,img_tool/pkg/tree/runfiles,img_tool/pkg/tree/treeartifact,img_tool/toolchain,img_tool/tools,pull_tool,pull_tool/cmd/downloadblob,pull_tool/cmd/internal/pull,pull_tool/cmd/pull_tool,pull_tool/pkg/auth/registry,pull_tool/pull/private +query --deleted_packages=e2e/cc,e2e/cc/base,e2e/cc/custom_standard_library,e2e/cc/patches,e2e/cc/platform,e2e/generic,e2e/generic/build_settings,e2e/generic/extend,e2e/generic/load,e2e/generic/multi_deploy,e2e/generic/platform,e2e/go,e2e/go/customization,e2e/go/image,e2e/go/multiarch,e2e/js,e2e/js/app,e2e/js/platform,e2e/python,e2e/python/platform,e2e/python/requirements,e2e/workspace,img_tool,img_tool/cmd,img_tool/cmd/bes,img_tool/cmd/compress,img_tool/cmd/deploy,img_tool/cmd/dockersave,img_tool/cmd/downloadblob,img_tool/cmd/expandtemplate,img_tool/cmd/img,img_tool/cmd/index,img_tool/cmd/layer,img_tool/cmd/layermeta,img_tool/cmd/manifest,img_tool/cmd/ocilayout,img_tool/cmd/push,img_tool/cmd/registry,img_tool/cmd/validate,img_tool/cmd/validate/layer-presence,img_tool/pkg/api,img_tool/pkg/auth/credential,img_tool/pkg/auth/grpcheaderinterceptor,img_tool/pkg/auth/protohelper,img_tool/pkg/auth/registry,img_tool/pkg/cas,img_tool/pkg/compress,img_tool/pkg/compress/util,img_tool/pkg/containerd,img_tool/pkg/contentmanifest,img_tool/pkg/deployvfs,img_tool/pkg/digestfs,img_tool/pkg/docker,img_tool/pkg/fileopener,img_tool/pkg/load,img_tool/pkg/proto/bazel,img_tool/pkg/proto/bazel/src/main/java/com/google/devtools/build/lib/buildeventstream,img_tool/pkg/proto/bazel/src/main/java/com/google/devtools/build/lib/packages/metrics,img_tool/pkg/proto/bazel/src/main/protobuf,img_tool/pkg/proto/blobcache,img_tool/pkg/proto/build_event_service,img_tool/pkg/proto/remote-apis,img_tool/pkg/proto/remote-apis/build/bazel/remote/execution/v2,img_tool/pkg/proto/remote-apis/build/bazel/semver,img_tool/pkg/push,img_tool/pkg/serve/bes,img_tool/pkg/serve/bes/syncer,img_tool/pkg/serve/blobcache,img_tool/pkg/serve/registry,img_tool/pkg/serve/registry/reapi,img_tool/pkg/serve/registry/s3,img_tool/pkg/serve/registry/upstream,img_tool/pkg/tarcas,img_tool/pkg/tree,img_tool/pkg/tree/merkle,img_tool/pkg/tree/runfiles,img_tool/pkg/tree/treeartifact,img_tool/toolchain,img_tool/tools,pull_tool,pull_tool/cmd/downloadblob,pull_tool/cmd/internal/pull,pull_tool/cmd/pull_tool,pull_tool/pkg/auth/registry,pull_tool/pull/private common --override_module=rules_img_tool=%workspace%/img_tool +common --override_module=rules_img_pull_tool=%workspace%/pull_tool import %workspace%/.bazelrc.common diff --git a/.bcr/pull_tool/metadata.template.json b/.bcr/pull_tool/metadata.template.json new file mode 100644 index 00000000..8bf20ad3 --- /dev/null +++ b/.bcr/pull_tool/metadata.template.json @@ -0,0 +1,15 @@ +{ + "homepage": "https://github.com/bazel-contrib/rules_img", + "maintainers": [ + { + "name": "Malte Poll", + "email": "malte.poll@moduscreate.com", + "github": "malt3" + } + ], + "repository": [ + "github:bazel-contrib/rules_img" + ], + "versions": [], + "yanked_versions": {} +} diff --git a/.bcr/pull_tool/presubmit.yml b/.bcr/pull_tool/presubmit.yml new file mode 100644 index 00000000..09651063 --- /dev/null +++ b/.bcr/pull_tool/presubmit.yml @@ -0,0 +1,14 @@ +bcr_test_module: + module_path: "." + matrix: + platform: ["debian10", "ubuntu2004", "macos", "macos_arm64", "windows", "windows_arm64"] + bazel: [7.x, 8.x] + tasks: + run_tests: + name: "Run test module" + platform: ${{ platform }} + bazel: ${{ bazel }} + build_targets: + - "//..." + test_targets: + - "//..." diff --git a/.bcr/pull_tool/source.template.json b/.bcr/pull_tool/source.template.json new file mode 100644 index 00000000..55251ef3 --- /dev/null +++ b/.bcr/pull_tool/source.template.json @@ -0,0 +1,4 @@ +{ + "integrity": "", + "url": "https://github.com/{OWNER}/{REPO}/releases/download/{TAG}/rules_img_pull_tool-{TAG}.tar.gz" +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb80554a..e99c435d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: repository-cache: true bazelrc: import %workspace%/.github/workflows/ci.bazelrc - name: Execute Tests - run: bazelisk test //... @rules_img_tool//... + run: bazelisk test //... @rules_img_tool//... @rules_img_pull_tool//... integration_test_matrix: strategy: fail-fast: false diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0910cc86..f539c8d8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -28,6 +28,9 @@ jobs: dist/img_linux_* dist/img_darwin_* dist/img_windows_*.exe + dist/pull_tool_linux_* + dist/pull_tool_darwin_* + dist/pull_tool_windows_*.exe prerelease: false tag_name: ${{ inputs.tag_name || github.ref_name }} secrets: diff --git a/BUILD.bazel b/BUILD.bazel index 5e435388..b1f1d55f 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -27,7 +27,24 @@ filegroup( visibility = ["//img/private/release:__subpackages__"], ) +filegroup( + name = "pull_tool_source_files", + srcs = glob( + ["pull_tool/**"], + allow_empty = True, + exclude = [ + "pull_tool/.bazelrc.common", + "pull_tool/.bazelrc", + "pull_tool/.vscode/**", + "pull_tool/tools/**", + ], + ), + visibility = ["//img/private/release:__subpackages__"], +) + exports_files([ ".bcr/metadata.template.json", + ".bcr/pull_tool/metadata.template.json", "prebuilt_lockfile.json", + "pull_tool_lockfile.json", ]) diff --git a/HACKING.md b/HACKING.md index 1812a854..63b38328 100644 --- a/HACKING.md +++ b/HACKING.md @@ -2,6 +2,8 @@ This guide provides instructions for developers working on rules_img. +**Note**: As of v0.2.6, development has been simplified. You no longer need to manually manage lockfiles or build custom binaries. Instead, use Bazel module overrides to work with source-built versions of the Go tools. See [Development with Source-Built Tools](#development-with-source-built-tools) for details. + ## Development Environment ### Prerequisites @@ -126,65 +128,106 @@ cd e2e/go && bazel test //... cd e2e/go && bazel run //:push ``` -### Testing with Prebuilt img Tool +### Development with Source-Built Tools -When developing rules_img, you may need to test with a prebuilt version of the `img` tool. Since the tool is in a separate `rules_img_tool` module, there are a few approaches: +When developing rules_img, you can use source-built versions of the Go tools (`img` and `pull_tool`) instead of prebuilt binaries. This is useful for testing changes to the tool implementations or applying patches. -#### Option 1: Local Development with HTTP Server +#### Setting Up Development Dependencies -Build the tool locally and serve it via HTTP: +Add dependencies on the tool modules and register the source-built toolchains in your `MODULE.bazel`: -```bash -# Build the img tool from the rules_img_tool module -bazel build @rules_img_tool//cmd/img +```starlark +# Add dependencies on the tool modules +bazel_dep(name = "rules_img_tool", version = "", dev_dependency = True) +bazel_dep(name = "rules_img_pull_tool", version = "", dev_dependency = True) -# Copy to a local directory and serve -TMPDIR=$(mktemp -d) -cp bazel-bin/external/rules_img_tool+/cmd/img/img_/img ${TMPDIR}/img -cd ${TMPDIR} && python3 -m http.server 8000 +# Register source-built toolchain +register_toolchains( + "@rules_img_tool//toolchain:all", + dev_dependency = True, +) ``` -Then create a custom lockfile (`prebuilt_lockfile.json`): - -```json -[ - { - "version": "v0.2.5", - "integrity": "sha256-FG5F8mJuRzvL1oiXCRXyOQ94RvJ+43HH+/yLGbWNvP8=", - "os": "linux", - "cpu": "amd64", - "url_templates": [ - "http://localhost:8000/img" - ] - } -] -``` +#### Module Override Options -#### Option 2: Airgapped BCR Module +You can override the tool modules using various Bazel module override mechanisms: -Build a complete local BCR (Bazel Central Registry) module: +##### Local Development Override -```bash -# Build the BCR module and distribution directory -bazel build //img/private/release:bcr -bazel build //img/private/release/distdir +For local development with modifications: -# Set environment variables -export RULES_IMG_BCR=file://$(realpath bazel-bin/img/private/release/bcr.local) -export DISTDIR=$(realpath bazel-bin/img/private/release/distdir/distdir_/distdir) +```starlark +# Override with local directory +local_path_override( + module_name = "rules_img_tool", + path = "../img_tool", # Path to your local checkout +) + +local_path_override( + module_name = "rules_img_pull_tool", + path = "../pull_tool", +) ``` -Then configure your test project's `.bazelrc`: +##### Git Repository Override + +For testing against a specific Git commit or branch: + +```starlark +# Override with Git repository +git_override( + module_name = "rules_img_tool", + remote = "https://github.com/your-fork/rules_img.git", + strip_prefix = "img_tool", + commit = "abc123def456", # Specific commit + # Or use: branch = "feature-branch" +) + +git_override( + module_name = "rules_img_pull_tool", + remote = "https://github.com/your-fork/rules_img.git", + strip_prefix = "pull_tool", + commit = "abc123def456", +) +``` -```bash -# .bazelrc -# Use the local BCR first, then fall back to the official registry -# (you need to replace the placeholder with the values from above). -common --registry=${RULES_IMG_BCR} --registry=https://bcr.bazel.build/ -common --distdir=${DISTDIR} +##### Archive Override + +For testing with a custom archive: + +```starlark +archive_override( + module_name = "rules_img_tool", + urls = ["https://github.com/your-fork/rules_img/archive/abc123def456.tar.gz"], + integrity = "sha256-...", + strip_prefix = "rules_img-abc123def456/img_tool", # Note: includes img_tool subdirectory +) + +archive_override( + module_name = "rules_img_pull_tool", + urls = ["https://github.com/your-fork/rules_img/archive/abc123def456.tar.gz"], + integrity = "sha256-...", + strip_prefix = "rules_img-abc123def456/pull_tool", # Note: includes pull_tool subdirectory +) ``` -This approach provides a complete isolated testing environment with all dependencies. +##### Single Version Override with Patches + +For applying patches to a specific version: + +```starlark +single_version_override( + module_name = "rules_img_tool", + patches = ["//patches:img_tool_performance.patch"], + patch_strip = 2, # For patches created from rules_img root (strips img_tool/ prefix) +) + +single_version_override( + module_name = "rules_img_pull_tool", + patches = ["//patches:pull_tool_auth_fix.patch"], + patch_strip = 2, # For patches created from rules_img root (strips pull_tool/ prefix) +) +``` ## Documentation diff --git a/MODULE.bazel b/MODULE.bazel index 37e24e8e..b6888697 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -25,6 +25,17 @@ register_toolchains( "@img_toolchain//:all", ) +pull_tool = use_extension("@rules_img//img/private/prebuilt:prebuilt.bzl", "pull_tool") +pull_tool.collection(name = "pull_hub_repo") +pull_tool.from_file( + collection = "pull_hub_repo", + file = "@rules_img//:pull_tool_lockfile.json", +) +use_repo( + pull_tool, + "pull_hub_repo", +) + # ✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂ # only dev_dependencies below this line - rules_img is lean # ✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂✂ @@ -38,6 +49,7 @@ bazel_dep(name = "gazelle", version = "0.45.0", dev_dependency = True) bazel_dep(name = "hermetic_cc_toolchain", version = "4.0.1", dev_dependency = True) bazel_dep(name = "rules_bazel_integration_test", version = "0.34.0", dev_dependency = True) bazel_dep(name = "rules_go", version = "0.57.0", dev_dependency = True) +bazel_dep(name = "rules_img_pull_tool", version = "0.2.5", dev_dependency = True) bazel_dep(name = "rules_img_tool", version = "0.2.5", dev_dependency = True) bazel_dep(name = "rules_pkg", version = "1.1.0", dev_dependency = True) bazel_dep(name = "rules_python", version = "1.6.3", dev_dependency = True) diff --git a/e2e/cc/MODULE.bazel b/e2e/cc/MODULE.bazel index cfd49950..ba69f8bb 100644 --- a/e2e/cc/MODULE.bazel +++ b/e2e/cc/MODULE.bazel @@ -1,15 +1,18 @@ module(name = "cc_e2e") # BEGIN BAZEL_DEP -bazel_dep( - name = "rules_img", - version = "0.2.5", -) +bazel_dep(name = "rules_img") local_path_override( module_name = "rules_img", path = "../..", ) +bazel_dep(name = "rules_img_pull_tool") +local_path_override( + module_name = "rules_img_pull_tool", + path = "../../pull_tool", +) + bazel_dep(name = "rules_img_tool") local_path_override( module_name = "rules_img_tool", diff --git a/e2e/generic/MODULE.bazel b/e2e/generic/MODULE.bazel index 00122fb3..6431f1c9 100644 --- a/e2e/generic/MODULE.bazel +++ b/e2e/generic/MODULE.bazel @@ -7,6 +7,12 @@ local_path_override( path = "../..", ) +bazel_dep(name = "rules_img_pull_tool") +local_path_override( + module_name = "rules_img_pull_tool", + path = "../../pull_tool", +) + bazel_dep(name = "rules_img_tool") local_path_override( module_name = "rules_img_tool", diff --git a/e2e/go/MODULE.bazel b/e2e/go/MODULE.bazel index 0c16ea50..62921efc 100644 --- a/e2e/go/MODULE.bazel +++ b/e2e/go/MODULE.bazel @@ -1,15 +1,18 @@ module(name = "go_e2e") # BEGIN BAZEL_DEP -bazel_dep( - name = "rules_img", - version = "0.2.5", -) +bazel_dep(name = "rules_img") local_path_override( module_name = "rules_img", path = "../..", ) +bazel_dep(name = "rules_img_pull_tool") +local_path_override( + module_name = "rules_img_pull_tool", + path = "../../pull_tool", +) + bazel_dep(name = "rules_img_tool") local_path_override( module_name = "rules_img_tool", diff --git a/e2e/js/MODULE.bazel b/e2e/js/MODULE.bazel index c0d688e7..e80ff885 100644 --- a/e2e/js/MODULE.bazel +++ b/e2e/js/MODULE.bazel @@ -1,15 +1,18 @@ module(name = "js_e2e") # BEGIN BAZEL_DEP -bazel_dep( - name = "rules_img", - version = "0.2.5", -) +bazel_dep(name = "rules_img") local_path_override( module_name = "rules_img", path = "../..", ) +bazel_dep(name = "rules_img_pull_tool") +local_path_override( + module_name = "rules_img_pull_tool", + path = "../../pull_tool", +) + bazel_dep(name = "rules_img_tool") local_path_override( module_name = "rules_img_tool", diff --git a/e2e/python/MODULE.bazel b/e2e/python/MODULE.bazel index 87900e0e..b12b5975 100644 --- a/e2e/python/MODULE.bazel +++ b/e2e/python/MODULE.bazel @@ -1,15 +1,18 @@ module(name = "python_e2e") # BEGIN BAZEL_DEP -bazel_dep( - name = "rules_img", - version = "0.2.5", -) +bazel_dep(name = "rules_img") local_path_override( module_name = "rules_img", path = "../..", ) +bazel_dep(name = "rules_img_pull_tool") +local_path_override( + module_name = "rules_img_pull_tool", + path = "../../pull_tool", +) + bazel_dep(name = "rules_img_tool") local_path_override( module_name = "rules_img_tool", diff --git a/e2e/workspace/WORKSPACE.bazel b/e2e/workspace/WORKSPACE.bazel index b8b0e82c..b44cc7e0 100644 --- a/e2e/workspace/WORKSPACE.bazel +++ b/e2e/workspace/WORKSPACE.bazel @@ -10,7 +10,7 @@ local_repository( # Load rules_img dependencies and prebuilt toolchains load("@rules_img//img:dependencies.bzl", "rules_img_dependencies") -load("@rules_img//img:repositories.bzl", "img_register_prebuilt_toolchains") +load("@rules_img//img:repositories.bzl", "img_register_prebuilt_toolchains", "pull_tool_register_prebuilt_repositories") rules_img_dependencies() @@ -19,6 +19,9 @@ img_register_prebuilt_toolchains() register_toolchains("@img_toolchain//:all") +# Register prebuilt pull_tool repositories for WORKSPACE mode +pull_tool_register_prebuilt_repositories() + # Pull Alpine Linux as base image for WORKSPACE mode test load("@rules_img//img:pull.bzl", "pull") diff --git a/img/private/prebuilt/prebuilt.bzl b/img/private/prebuilt/prebuilt.bzl index bf2b564e..c08004a6 100644 --- a/img/private/prebuilt/prebuilt.bzl +++ b/img/private/prebuilt/prebuilt.bzl @@ -5,7 +5,7 @@ different platforms, including downloading, selecting, and registering them as Bazel toolchains. """ -load("//img/private/platforms:platforms.bzl", "platform_for_goos_and_goarch") +load("//img/private/platforms:platforms.bzl", "platform_for_goos_and_goarch", "platform_for_repository_os") def _prebuilt_toolchain_definition_for_platform(platform_name, tool_target): platform = platform_for_goos_and_goarch(platform_name) @@ -70,6 +70,38 @@ prebuilt_collection_hub_repo = repository_rule( }, ) +def _prebuilt_pull_hub_repo_impl(rctx): + rctx.file("BUILD.bazel", """load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +bzl_library( + name = "defs", + srcs = ["defs.bzl"], + visibility = ["//visibility:public"], + deps = ["@rules_img//img/private/platforms:platforms"], +)""") + rctx.file( + "defs.bzl", + """\ +load("@rules_img//img/private/platforms:platforms.bzl", "platform_for_repository_os") + +TOOLS = {} + +def tool_for_repository_os(rctx): + platform = platform_for_repository_os(rctx) + key = platform.name + if key not in TOOLS: + fail("No pull tool for platform " + key) + return Label(TOOLS[key]) +""".format(json.encode_indent(rctx.attr.tools, indent = " ")), + ) + +prebuilt_pull_hub_repo = repository_rule( + implementation = _prebuilt_pull_hub_repo_impl, + attrs = { + "tools": attr.string_dict(), + }, +) + def _prebuilt_img_tool_repo_impl(rctx): extension = "exe" if rctx.attr.os == "windows" else "" dot = "." if len(extension) > 0 else "" @@ -101,14 +133,52 @@ _prebuilt_attrs = { ), } +_prebuilt_pull_tool_attrs = { + "version": attr.string(mandatory = True), + "integrity": attr.string(mandatory = True), + "os": attr.string(values = ["darwin", "linux", "windows"]), + "cpu": attr.string(values = ["amd64", "arm64"]), + "url_templates": attr.string_list( + default = ["https://github.com/bazel-contrib/rules_img/releases/download/{version}/pull_tool_{os}_{cpu}{dot}{extension}"], + ), +} + prebuilt_img_tool_repo = repository_rule( implementation = _prebuilt_img_tool_repo_impl, attrs = _prebuilt_attrs, ) +def _prebuilt_pull_tool_repo_impl(rctx): + extension = "exe" if rctx.attr.os == "windows" else "" + dot = "." if len(extension) > 0 else "" + urls = [template.format( + version = rctx.attr.version, + os = rctx.attr.os, + cpu = rctx.attr.cpu, + dot = dot, + extension = extension, + ) for template in rctx.attr.url_templates] + rctx.download( + urls, + output = "pull_tool.exe", + executable = True, + integrity = rctx.attr.integrity, + ) + rctx.file( + "BUILD.bazel", + content = """exports_files(["pull_tool.exe"])""", + ) + +prebuilt_pull_tool_repo = repository_rule( + implementation = _prebuilt_pull_tool_repo_impl, + attrs = _prebuilt_pull_tool_attrs, +) + _prebuilt_tool_collection = tag_class(attrs = {"name": attr.string(), "override": attr.bool(default = False)}) _prebuilt_tool_from_file = tag_class(attrs = {"collection": attr.string(), "file": attr.label()}) _prebuilt_tool_download = tag_class(attrs = {"collection": attr.string()} | _prebuilt_attrs) +_prebuilt_pull_tool_download = tag_class(attrs = {"collection": attr.string()} | _prebuilt_pull_tool_attrs) +_host_tool = tag_class(attrs = {"binary": attr.label()}) def _lockfile_to_dict(lockfile, basename): requested_tools = {} @@ -132,11 +202,17 @@ def _prebuilt_img_tool_collection_for_module(ctx, mod): name = "%s_%s_%s" % (download.collection, download.os, download.cpu) requested_tools[name] = {member: getattr(download, member) for member in dir(download)} collections[download.collection]["tools"][(download.os, download.cpu)] = "%s_%s_%s" % (download.collection, download.os, download.cpu) - return (requested_tools, collections) + host_tool = None + if len(mod.tags.host_tool) > 1: + fail("module {} requested host_tool tag class more than once".format(mod)) + elif len(mod.tags.host_tool) == 1: + host_tool = mod.tags.host_tool[0].binary + return (requested_tools, collections, host_tool) -def _prebuilt_img_tool(ctx): +def _prebuilt_tool_helper(ctx, tool_path, hub_repo_fn, tool_repo_fn = prebuilt_img_tool_repo): requested_tools = {} collections = {} + host_tool = None root_module = None for mod in ctx.modules: if mod.is_root: @@ -148,6 +224,10 @@ def _prebuilt_img_tool(ctx): fail("Duplicate definitions for prebuilt_img_tool %s. Only root module is allowed to override." % collection_name) requested_tools.update(for_module[0]) collections.update(for_module[1]) + if mod.name == "rules_img_pull_tool": + # only "rules_img_pull_tool" (or the root module) + # are allowed to set this + host_tool = for_module[2] root_module_direct_deps = [] if root_module != None: for_root_module = _prebuilt_img_tool_collection_for_module(ctx, root_module) @@ -157,17 +237,22 @@ def _prebuilt_img_tool(ctx): requested_tools.update(for_root_module[0]) collections.update(for_root_module[1]) root_module_direct_deps = for_root_module[1].keys() + host_tool = for_root_module[2] for item in requested_tools.items(): - prebuilt_img_tool_repo( + tool_repo_fn( name = item[0], **item[1] ) for (collection_name, collection) in collections.items(): tools = {} for ((os, arch), tool_repo_name) in collection["tools"].items(): - tools["%s_%s" % (os, arch)] = "@%s//:img.exe" % tool_repo_name - prebuilt_collection_hub_repo( + tools["%s_%s" % (os, arch)] = "@{}//:{}".format(tool_repo_name, tool_path) + if host_tool != None: + host_platform = platform_for_repository_os(ctx) + tools[host_platform.name] = str(host_tool) + + hub_repo_fn( name = collection_name, tools = tools, ) @@ -178,11 +263,37 @@ def _prebuilt_img_tool(ctx): reproducible = True, ) +def _prebuilt_img_tool_impl(ctx): + return _prebuilt_tool_helper( + ctx, + tool_path = "img.exe", + hub_repo_fn = prebuilt_collection_hub_repo, + ) + prebuilt_img_tool = module_extension( - implementation = _prebuilt_img_tool, + implementation = _prebuilt_img_tool_impl, tag_classes = { "collection": _prebuilt_tool_collection, "from_file": _prebuilt_tool_from_file, "download": _prebuilt_tool_download, + "host_tool": _host_tool, + }, +) + +def _pull_tool_impl(ctx): + return _prebuilt_tool_helper( + ctx, + tool_path = "pull_tool.exe", + hub_repo_fn = prebuilt_pull_hub_repo, + tool_repo_fn = prebuilt_pull_tool_repo, + ) + +pull_tool = module_extension( + implementation = _pull_tool_impl, + tag_classes = { + "collection": _prebuilt_tool_collection, + "from_file": _prebuilt_tool_from_file, + "download": _prebuilt_pull_tool_download, + "host_tool": _host_tool, }, ) diff --git a/img/private/release/BUILD.bazel b/img/private/release/BUILD.bazel index 25310944..2d97f91a 100644 --- a/img/private/release/BUILD.bazel +++ b/img/private/release/BUILD.bazel @@ -18,6 +18,16 @@ pkg_tar( tags = ["manual"], ) +pkg_tar( + name = "pull_tool_src_tar", + srcs = ["//:pull_tool_source_files"], + out = "rules_img_pull_tool.tar.gz", + extension = "tar.gz", + mode = "0755", + strip_prefix = "/pull_tool", + tags = ["manual"], +) + [ platform( name = name(p), @@ -34,18 +44,33 @@ release_platform_flag( release_files( name = "prebuilt_toolchain", executable = "@rules_img_tool//cmd/img", + lockfile_name = "prebuilt_lockfile.json", + tags = ["manual"], +) + +release_files( + name = "pull_tool_toolchain", + basename = "pull_tool", + executable = "@rules_img_pull_tool//cmd/pull_tool", + lockfile_name = "pull_tool_lockfile.json", tags = ["manual"], ) offline_bundle( name = "airgapped", - distdir_contents = ":prebuilt_toolchain", + distdir_contents = [ + ":prebuilt_toolchain", + ":pull_tool_toolchain", + ], ) source_bundle( name = "srcs", srcs = ["//img/private/release/source_files:release_src_files"], - overrides = [":prebuilt_toolchain"], + overrides = [ + ":prebuilt_toolchain", + ":pull_tool_toolchain", + ], tags = ["manual"], ) @@ -75,11 +100,20 @@ versioned_filename_info( tags = ["manual"], ) +versioned_filename_info( + name = "versioned_pull_tool_tar", + src = ":pull_tool_src_tar", + extension = "tar.gz", + module_name = "rules_img_pull_tool", + tags = ["manual"], +) + offline_bcr( name = "bcr", src_tars = [ ":versioned_src_tar", ":versioned_tool_tar", + ":versioned_pull_tool_tar", ], ) @@ -87,6 +121,8 @@ pkg_tar( name = "dist_tar", srcs = [ ":prebuilt_toolchain", + ":pull_tool_toolchain", + ":versioned_pull_tool_tar", ":versioned_src_tar", ":versioned_tool_tar", ], diff --git a/img/private/release/defs.bzl b/img/private/release/defs.bzl index e4e465b3..bf62d8d0 100644 --- a/img/private/release/defs.bzl +++ b/img/private/release/defs.bzl @@ -170,7 +170,7 @@ def _release_files(ctx): attributes[filename] = EXECUTABLE_ATTRIBUTES distdir_contents[filename_basename] = executable output_group_info["%s_files" % platform] = depset([executable]) - lockfile_args.add("--img-tool", "%s=%s" % (platform, executable.path)) + lockfile_args.add("--tool", "%s=%s" % (platform, executable.path)) lockfile = ctx.actions.declare_file("%s_lockfile.json" % ctx.attr.name) lockfile_args.add(lockfile) ctx.actions.run( @@ -186,8 +186,8 @@ def _release_files(ctx): OutputGroupInfo(**output_group_info), PackageFilesInfo(attributes = attributes, dest_src_map = dest_src_map), OverrideSourceFilesInfo( - attributes = {"prebuilt_lockfile.json": DEFAULT_ATTRIBUTES}, - dest_src_map = {"prebuilt_lockfile.json": lockfile}, + attributes = {ctx.attr.lockfile_name: DEFAULT_ATTRIBUTES}, + dest_src_map = {ctx.attr.lockfile_name: lockfile}, ), OfflineBuildDistdirInfo( basename_file_map = distdir_contents, @@ -208,6 +208,9 @@ release_files = rule( default = Label("//img/private/release/lockfile"), cfg = "exec", ), + "lockfile_name": attr.string( + mandatory = True, + ), "version": attr.label( default = "@rules_img_version", providers = [ModuleVersionInfo], @@ -216,13 +219,17 @@ release_files = rule( ) def _offline_bundle_impl(ctx): - mapped_contents = ctx.attr.distdir_contents[OfflineBuildDistdirInfo].basename_file_map - extra_files = ctx.attr.distdir_contents[OfflineBuildDistdirInfo].files contents = {} - for f in extra_files.to_list(): - contents[f.basename] = f - for basename, f in mapped_contents.items(): - contents[basename] = f + + # Handle multiple distdir_contents inputs + for distdir_content in ctx.attr.distdir_contents: + if distdir_content: # Check if not None + mapped_contents = distdir_content[OfflineBuildDistdirInfo].basename_file_map + extra_files = distdir_content[OfflineBuildDistdirInfo].files + for f in extra_files.to_list(): + contents[f.basename] = f + for basename, f in mapped_contents.items(): + contents[basename] = f distdir_args = ctx.actions.args() for basename, f in contents.items(): @@ -242,7 +249,7 @@ def _offline_bundle_impl(ctx): offline_bundle = rule( implementation = _offline_bundle_impl, attrs = { - "distdir_contents": attr.label( + "distdir_contents": attr.label_list( providers = [OfflineBuildDistdirInfo], mandatory = True, ), diff --git a/img/private/release/lockfile/lockfile.go b/img/private/release/lockfile/lockfile.go index cc4499d2..c1a02038 100644 --- a/img/private/release/lockfile/lockfile.go +++ b/img/private/release/lockfile/lockfile.go @@ -56,7 +56,7 @@ var ( func main() { flag.StringVar(&version, "version", "0.0.0", "Version of the project.") - flag.Var(&binaries, "img-tool", "Key-value pairs of platform name to img binary path.") + flag.Var(&binaries, "tool", "Key-value pairs of platform name to tool binary path.") flag.Parse() if flag.NArg() != 1 { fmt.Fprintln(os.Stderr, "expected lockfile output") diff --git a/img/private/release/release_notes/release_notes.go b/img/private/release/release_notes/release_notes.go index c5ff3840..fc459e1b 100644 --- a/img/private/release/release_notes/release_notes.go +++ b/img/private/release/release_notes/release_notes.go @@ -96,10 +96,13 @@ load("@rules_img//img:dependencies.bzl", "rules_img_dependencies") rules_img_dependencies() # Register prebuilt toolchains -load("@rules_img//img:repositories.bzl", "img_register_prebuilt_toolchains") +load("@rules_img//img:repositories.bzl", "img_register_prebuilt_toolchains", "pull_tool_register_prebuilt_repositories") img_register_prebuilt_toolchains() register_toolchains("@img_toolchain//:all") +# Register prebuilt pull_tool repositories (for pull functionality) +pull_tool_register_prebuilt_repositories() + # Example: Pull a base image load("@rules_img//img:pull.bzl", "pull") pull( diff --git a/img/private/repository_rules/BUILD.bazel b/img/private/repository_rules/BUILD.bazel index 4ee6bf1a..60e33a2c 100644 --- a/img/private/repository_rules/BUILD.bazel +++ b/img/private/repository_rules/BUILD.bazel @@ -14,21 +14,21 @@ bzl_library( ":download", "//img/private/platforms", "@bazel_skylib//lib:sets", - "@img_toolchain//:defs", + "@pull_hub_repo//:defs", ], ) bzl_library( - name = "download", - srcs = ["download.bzl"], + name = "pull_blob", + srcs = ["pull_blob.bzl"], visibility = ["//img:__subpackages__"], + deps = ["@pull_hub_repo//:defs"], ) bzl_library( - name = "pull_blob", - srcs = ["pull_blob.bzl"], + name = "download", + srcs = ["download.bzl"], visibility = ["//img:__subpackages__"], - deps = ["@img_toolchain//:defs"], ) bzl_library( diff --git a/img/private/repository_rules/pull.bzl b/img/private/repository_rules/pull.bzl index 8c684260..8c9a0448 100644 --- a/img/private/repository_rules/pull.bzl +++ b/img/private/repository_rules/pull.bzl @@ -1,7 +1,7 @@ """Repository rules for pulling container images.""" load("@bazel_skylib//lib:sets.bzl", "sets") -load("@img_toolchain//:defs.bzl", "tool_for_repository_os") +load("@pull_hub_repo//:defs.bzl", "tool_for_repository_os") load("//img/private/platforms:platforms.bzl", "has_constraint_setting") load( ":download.bzl", diff --git a/img/private/repository_rules/pull_blob.bzl b/img/private/repository_rules/pull_blob.bzl index 02d90664..1476e54a 100644 --- a/img/private/repository_rules/pull_blob.bzl +++ b/img/private/repository_rules/pull_blob.bzl @@ -1,6 +1,6 @@ """Repository rule for pulling individual blobs from a container registry.""" -load("@img_toolchain//:defs.bzl", "tool_for_repository_os") +load("@pull_hub_repo//:defs.bzl", "tool_for_repository_os") def _pull_blob_file_impl(rctx): tool = tool_for_repository_os(rctx) @@ -19,7 +19,9 @@ def _pull_blob_file_impl(rctx): ] if rctx.attr.executable: args.append("--executable") - rctx.execute(args) + result = rctx.execute(args) + if result.return_code != 0: + fail("Failed to download blob: {}".format(result.stderr)) rctx.file( "BUILD.bazel", content = """filegroup( @@ -82,7 +84,9 @@ def _pull_blob_archive_impl(rctx): "--output", output_name, ] - rctx.execute(args) + result = rctx.execute(args) + if result.return_code != 0: + fail("Failed to download blob: {}".format(result.stderr)) rctx.extract( archive = output_name, strip_prefix = rctx.attr.strip_prefix, diff --git a/img/repositories.bzl b/img/repositories.bzl index 7066a966..c3c0db47 100644 --- a/img/repositories.bzl +++ b/img/repositories.bzl @@ -1,6 +1,6 @@ """Repository rules for fetching external tools in WORKSPACE mode""" -load("//img/private/prebuilt:prebuilt.bzl", "prebuilt_collection_hub_repo") +load("//img/private/prebuilt:prebuilt.bzl", "prebuilt_collection_hub_repo", "prebuilt_pull_hub_repo") def _img_prebuilt_tool_from_lockfile_impl(rctx): """Repository rule that reads lockfile and downloads tool for specific platform.""" @@ -111,3 +111,110 @@ def img_register_prebuilt_toolchains( name = name, tools = tools, ) + +def _pull_tool_prebuilt_tool_from_lockfile_impl(rctx): + """Repository rule that reads lockfile and downloads pull_tool for specific platform.""" + lockfile_content = rctx.read(rctx.attr.lockfile) + lockfile_data = json.decode(lockfile_content) + + # Find the tool entry for our target platform + target_tool = None + for tool in lockfile_data: + if tool["os"] == rctx.attr.os and tool["cpu"] == rctx.attr.cpu: + target_tool = tool + break + + if not target_tool: + fail("No pull_tool found in lockfile for platform %s_%s" % (rctx.attr.os, rctx.attr.cpu)) + + # Download the tool using pull_tool URL templates + extension = "exe" if target_tool["os"] == "windows" else "" + dot = "." if len(extension) > 0 else "" + url_templates = target_tool.get("url_templates", ["https://github.com/bazel-contrib/rules_img/releases/download/{version}/pull_tool_{os}_{cpu}{dot}{extension}"]) + + urls = [template.format( + version = target_tool["version"], + os = target_tool["os"], + cpu = target_tool["cpu"], + dot = dot, + extension = extension, + ) for template in url_templates] + + rctx.download( + urls, + output = "pull_tool.exe", + executable = True, + integrity = target_tool["integrity"], + ) + + rctx.file( + "BUILD.bazel", + content = """exports_files(["pull_tool.exe"])""", + ) + +_pull_tool_prebuilt_tool_from_lockfile = repository_rule( + implementation = _pull_tool_prebuilt_tool_from_lockfile_impl, + attrs = { + "lockfile": attr.label(mandatory = True), + "os": attr.string(mandatory = True), + "cpu": attr.string(mandatory = True), + }, +) + +def pull_tool_register_prebuilt_repositories( + name = "pull_hub_repo", + lockfile = Label("@rules_img//:pull_tool_lockfile.json"), + platforms = [ + ("linux", "amd64"), + ("linux", "arm64"), + ("darwin", "amd64"), + ("darwin", "arm64"), + ("windows", "amd64"), + ("windows", "arm64"), + ]): + """Register prebuilt pull_tool repositories for WORKSPACE mode. + + This macro creates repository rules for prebuilt pull_tool binaries from a lockfile + but does NOT register them as toolchains (unlike img_register_prebuilt_toolchains). + This is the WORKSPACE equivalent of the MODULE.bazel pull_tool extension. + + Usage in WORKSPACE: + load("@rules_img//img:repositories.bzl", "pull_tool_register_prebuilt_repositories") + + # Use defaults + pull_tool_register_prebuilt_repositories() + + # Or specify custom platforms/lockfile + pull_tool_register_prebuilt_repositories( + lockfile = "@my_repo//:my_pull_tool_lockfile.json", + platforms = [("linux", "amd64"), ("darwin", "amd64")] + ) + + Args: + name: Name of the pull_tool collection hub repository (default: "pull_hub_repo") + lockfile: Label pointing to the pull_tool_lockfile.json (default: "@rules_img//:pull_tool_lockfile.json") + platforms: List of (os, cpu) tuples for platforms to support + """ + + # Create individual tool repositories for each requested platform + tools = {} + for (os, cpu) in platforms: + repo_name = "%s_%s_%s" % (name, os, cpu) + + # Create repository that reads lockfile and downloads pull_tool for this platform + _pull_tool_prebuilt_tool_from_lockfile( + name = repo_name, + lockfile = lockfile, + os = os, + cpu = cpu, + ) + + # Track this tool for the hub repository + platform_key = "%s_%s" % (os, cpu) + tools[platform_key] = "@%s//:pull_tool.exe" % repo_name + + # Create the hub repository without toolchain definitions (just tool references) + prebuilt_pull_hub_repo( + name = name, + tools = tools, + ) diff --git a/img_tool/cmd/img/BUILD.bazel b/img_tool/cmd/img/BUILD.bazel index ab1c008e..163a7a18 100644 --- a/img_tool/cmd/img/BUILD.bazel +++ b/img_tool/cmd/img/BUILD.bazel @@ -17,7 +17,6 @@ go_library( "//cmd/layermeta", "//cmd/manifest", "//cmd/ocilayout", - "//cmd/pull", "//cmd/push", "//cmd/validate", "@rules_go//go/runfiles", diff --git a/img_tool/cmd/img/img.go b/img_tool/cmd/img/img.go index 0ac8ba3e..fe4208bc 100644 --- a/img_tool/cmd/img/img.go +++ b/img_tool/cmd/img/img.go @@ -17,7 +17,6 @@ import ( "github.com/bazel-contrib/rules_img/img_tool/cmd/layermeta" "github.com/bazel-contrib/rules_img/img_tool/cmd/manifest" "github.com/bazel-contrib/rules_img/img_tool/cmd/ocilayout" - "github.com/bazel-contrib/rules_img/img_tool/cmd/pull" "github.com/bazel-contrib/rules_img/img_tool/cmd/push" "github.com/bazel-contrib/rules_img/img_tool/cmd/validate" ) @@ -34,7 +33,6 @@ Commands: manifest creates an image manifest and config from layers oci-layout assembles an OCI layout directory from manifest and layers validate validates layers and images - pull pulls an image from a registry push pushes an image to a registry deploy-metadata calculates metadata for deploying an image (push/load) deploy-merge merges multiple deploy manifests into a single deployment` @@ -66,8 +64,6 @@ func Run(ctx context.Context, args []string) { index.IndexProcess(ctx, args[2:]) case "validate": validate.ValidationProcess(ctx, args[2:]) - case "pull": - pull.PullProcess(ctx, args[2:]) case "push": push.PushProcess(ctx, args[2:]) case "deploy-metadata": diff --git a/img_tool/cmd/pull/BUILD.bazel b/img_tool/cmd/pull/BUILD.bazel deleted file mode 100644 index 5ccbca72..00000000 --- a/img_tool/cmd/pull/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "pull", - srcs = ["pull.go"], - importpath = "github.com/bazel-contrib/rules_img/img_tool/cmd/pull", - visibility = ["//visibility:public"], - deps = [ - "//pkg/auth/registry", - "@com_github_malt3_go_containerregistry//pkg/name", - "@com_github_malt3_go_containerregistry//pkg/v1:pkg", - "@com_github_malt3_go_containerregistry//pkg/v1/remote", - ], -) diff --git a/img_tool/pkg/auth/registry/registry.go b/img_tool/pkg/auth/registry/registry.go index d7171a18..81041e91 100644 --- a/img_tool/pkg/auth/registry/registry.go +++ b/img_tool/pkg/auth/registry/registry.go @@ -1,11 +1,14 @@ package registry -import( +import ( "github.com/malt3/go-containerregistry/pkg/authn" "github.com/malt3/go-containerregistry/pkg/v1/google" "github.com/malt3/go-containerregistry/pkg/v1/remote" ) +// WithAuthFromMultiKeychain returns a remote.Option that uses a MultiKeychain +// combining the default keychain and the Google keychain for authentication. +// WARNING: keep in sync with the same function in img_tool/pkg/auth/registry/registry.go. func WithAuthFromMultiKeychain() remote.Option { kc := authn.NewMultiKeychain( authn.DefaultKeychain, diff --git a/pull_tool/.bazelrc b/pull_tool/.bazelrc new file mode 100644 index 00000000..db8b8dfe --- /dev/null +++ b/pull_tool/.bazelrc @@ -0,0 +1,5 @@ +common --override_module=rules_img=%workspace%/.. + +import %workspace%/.bazelrc.common + +try-import %workspace%/.bazelrc.user diff --git a/pull_tool/.bazelrc.common b/pull_tool/.bazelrc.common new file mode 120000 index 00000000..95435188 --- /dev/null +++ b/pull_tool/.bazelrc.common @@ -0,0 +1 @@ +../.bazelrc.common \ No newline at end of file diff --git a/pull_tool/BUILD.bazel b/pull_tool/BUILD.bazel new file mode 100644 index 00000000..2c5c336c --- /dev/null +++ b/pull_tool/BUILD.bazel @@ -0,0 +1,19 @@ +# gazelle:resolve_regexp go github.com/bazel-contrib/rules_img/pull_tool/(.*) //$1 + +filegroup( + name = "pull_tool_srcs", + srcs = [ + "go.mod", + "go.sum", + "//cmd/downloadblob:go_srcs", + "//cmd/internal/pull:go_srcs", + "//cmd/pull_tool:go_srcs", + "//pkg/auth/registry:go_srcs", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "source_bootstrapped_pull_tool", + actual = "@pull_bootstrap//:pull_tool.exe", +) diff --git a/pull_tool/MODULE.bazel b/pull_tool/MODULE.bazel new file mode 100644 index 00000000..a90f191e --- /dev/null +++ b/pull_tool/MODULE.bazel @@ -0,0 +1,35 @@ +module( + name = "rules_img_pull_tool", + version = "0.2.5", + compatibility_level = 1, +) + +bazel_dep(name = "rules_img", version = "0.2.5") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "gazelle", version = "0.45.0") +bazel_dep(name = "rules_go", version = "0.57.0") + +go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") + +# Known to exist since it is instantiated by rules_go itself. +use_repo( + go_sdk, + "go_host_compatible_sdk_label", +) + +go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//:go.mod") +use_repo(go_deps, "com_github_google_go_containerregistry") + +pull_bootstrap = use_repo_rule("//pull/private:pull_bootstrap.bzl", "pull_bootstrap") + +pull_bootstrap( + name = "pull_bootstrap", + go_mod = "//:go.mod", + go_sum = "//:go.sum", + pull_tool_srcs = ["//:pull_tool_srcs"], +) + +# register the source bootstrapped pull_tool +pull_tool = use_extension("@rules_img//img/private/prebuilt:prebuilt.bzl", "pull_tool") +pull_tool.host_tool(binary = "@pull_bootstrap//:pull_tool.exe") diff --git a/pull_tool/cmd/downloadblob/BUILD.bazel b/pull_tool/cmd/downloadblob/BUILD.bazel new file mode 100644 index 00000000..e09dcbb0 --- /dev/null +++ b/pull_tool/cmd/downloadblob/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "downloadblob", + srcs = ["downloadblob.go"], + importpath = "github.com/bazel-contrib/rules_img/pull_tool/cmd/downloadblob", + visibility = ["//cmd/pull_tool:__pkg__"], + deps = [ + "//pkg/auth/registry", + "@com_github_google_go_containerregistry//pkg/name", + "@com_github_google_go_containerregistry//pkg/v1/remote", + ], +) + +filegroup( + name = "go_srcs", + srcs = glob(["*.go"]), + visibility = ["//visibility:public"], +) diff --git a/pull_tool/cmd/downloadblob/downloadblob.go b/pull_tool/cmd/downloadblob/downloadblob.go new file mode 100644 index 00000000..95edca1e --- /dev/null +++ b/pull_tool/cmd/downloadblob/downloadblob.go @@ -0,0 +1,145 @@ +package downloadblob + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + + reg "github.com/bazel-contrib/rules_img/pull_tool/pkg/auth/registry" +) + +func DownloadBlobProcess(ctx context.Context, args []string) { + var digest string + var repository string + var outputPath string + var registries stringSliceFlag + var executable bool + + flagSet := flag.NewFlagSet("download-blob", flag.ExitOnError) + flagSet.Usage = func() { + fmt.Fprintf(flagSet.Output(), "Downloads a single blob from a container registry.\n\n") + fmt.Fprintf(flagSet.Output(), "Usage: pull_tool download-blob [OPTIONS]\n") + flagSet.PrintDefaults() + examples := []string{ + "pull_tool download-blob --digest sha256:abc123... --repository myapp --output blob.tar.gz", + "pull_tool download-blob --digest sha256:abc123... --repository myapp --registry docker.io --output blob.tar.gz", + } + fmt.Fprintf(flagSet.Output(), "\nExamples:\n") + for _, example := range examples { + fmt.Fprintf(flagSet.Output(), " $ %s\n", example) + } + } + + flagSet.StringVar(&digest, "digest", "", "The digest of the blob to download (required)") + flagSet.StringVar(&repository, "repository", "", "Repository name of the image (required)") + flagSet.StringVar(&outputPath, "output", "", "Output file path (required)") + flagSet.Var(®istries, "registry", "Registry to use (can be specified multiple times, defaults to docker.io)") + flagSet.BoolVar(&executable, "executable", false, "Mark the output file executable") + + if err := flagSet.Parse(args); err != nil { + flagSet.Usage() + os.Exit(1) + } + + if digest == "" { + fmt.Fprintf(os.Stderr, "Error: --digest is required\n") + flagSet.Usage() + os.Exit(1) + } + if repository == "" { + fmt.Fprintf(os.Stderr, "Error: --repository is required\n") + flagSet.Usage() + os.Exit(1) + } + if outputPath == "" { + fmt.Fprintf(os.Stderr, "Error: --output is required\n") + flagSet.Usage() + os.Exit(1) + } + + // Default to docker.io if no registries specified + if len(registries) == 0 { + registries = []string{"docker.io"} + } + + if !strings.HasPrefix(digest, "sha256:") { + digest = "sha256:" + digest + } + + // Try each registry until success + var lastErr error + for _, registry := range registries { + err := downloadFromRegistry(registry, repository, digest, outputPath) + if err == nil { + break + } + lastErr = err + fmt.Fprintf(os.Stderr, "Failed to download from %s: %v\n", registry, err) + } + + if lastErr != nil { + fmt.Fprintf(os.Stderr, "Error: Failed to download blob from all registries: %v\n", lastErr) + os.Exit(1) + } + + // Set file permissions after successful download + if executable { + if err := os.Chmod(outputPath, 0o755); err != nil { + fmt.Fprintf(os.Stderr, "Error: Failed to set executable permission on output file: %v\n", err) + os.Exit(1) + } + } else { + if err := os.Chmod(outputPath, 0o644); err != nil { + fmt.Fprintf(os.Stderr, "Error: Failed to set permission on output file: %v\n", err) + os.Exit(1) + } + } +} + +func downloadFromRegistry(registry, repository, digest, outputPath string) error { + ref, err := name.NewDigest(fmt.Sprintf("%s/%s@%s", registry, repository, digest)) + if err != nil { + return fmt.Errorf("creating blob reference: %w", err) + } + + layer, err := remote.Layer(ref, reg.WithAuthFromMultiKeychain()) + if err != nil { + return fmt.Errorf("getting layer: %w", err) + } + + outputFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("opening output file: %w", err) + } + defer outputFile.Close() + + rc, err := layer.Compressed() + if err != nil { + return fmt.Errorf("getting compressed layer: %w", err) + } + defer rc.Close() + + _, err = io.Copy(outputFile, rc) + if err != nil { + return fmt.Errorf("writing layer data: %w", err) + } + + return nil +} + +type stringSliceFlag []string + +func (s *stringSliceFlag) String() string { + return strings.Join(*s, ",") +} + +func (s *stringSliceFlag) Set(value string) error { + *s = append(*s, value) + return nil +} diff --git a/pull_tool/cmd/internal/pull/BUILD.bazel b/pull_tool/cmd/internal/pull/BUILD.bazel new file mode 100644 index 00000000..6ca0df39 --- /dev/null +++ b/pull_tool/cmd/internal/pull/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "pull", + srcs = ["pull.go"], + importpath = "github.com/bazel-contrib/rules_img/pull_tool/cmd/internal/pull", + # keep + visibility = ["//cmd:__subpackages__"], + deps = [ + "//pkg/auth/registry", + "@com_github_google_go_containerregistry//pkg/name:go_default_library", + "@com_github_google_go_containerregistry//pkg/v1:go_default_library", + "@com_github_google_go_containerregistry//pkg/v1/remote:go_default_library", + ], +) + +filegroup( + name = "go_srcs", + srcs = glob(["*.go"]), + visibility = ["//visibility:public"], +) diff --git a/img_tool/cmd/pull/pull.go b/pull_tool/cmd/internal/pull/pull.go similarity index 95% rename from img_tool/cmd/pull/pull.go rename to pull_tool/cmd/internal/pull/pull.go index 07d3a28f..f69cf45a 100644 --- a/img_tool/cmd/pull/pull.go +++ b/pull_tool/cmd/internal/pull/pull.go @@ -10,11 +10,10 @@ import ( "strings" "sync" - "github.com/malt3/go-containerregistry/pkg/name" - registryv1 "github.com/malt3/go-containerregistry/pkg/v1" - "github.com/malt3/go-containerregistry/pkg/v1/remote" - - reg "github.com/bazel-contrib/rules_img/img_tool/pkg/auth/registry" + reg "github.com/bazel-contrib/rules_img/pull_tool/pkg/auth/registry" + "github.com/google/go-containerregistry/pkg/name" + registryv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" ) func PullProcess(ctx context.Context, args []string) { @@ -28,11 +27,11 @@ func PullProcess(ctx context.Context, args []string) { flagSet := flag.NewFlagSet("pull", flag.ExitOnError) flagSet.Usage = func() { fmt.Fprintf(flagSet.Output(), "Downloads an image from a container registry.\n\n") - fmt.Fprintf(flagSet.Output(), "Usage: img pull [OPTIONS]\n") + fmt.Fprintf(flagSet.Output(), "Usage: pull_tool pull [OPTIONS]\n") flagSet.PrintDefaults() examples := []string{ - "img pull --reference sha256:abc123... --repository myapp --output ./outdir", - "img pull --reference sha256:abc123... --repository myapp --registry docker.io", + "pull_tool pull --reference sha256:abc123... --repository myapp --output ./outdir", + "pull_tool pull --reference sha256:abc123... --repository myapp --registry docker.io", } fmt.Fprintf(flagSet.Output(), "\nExamples:\n") for _, example := range examples { diff --git a/pull_tool/cmd/pull_tool/BUILD.bazel b/pull_tool/cmd/pull_tool/BUILD.bazel new file mode 100644 index 00000000..e9cf9179 --- /dev/null +++ b/pull_tool/cmd/pull_tool/BUILD.bazel @@ -0,0 +1,30 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "pull_tool_lib", + srcs = ["pull_tool.go"], + importpath = "github.com/bazel-contrib/rules_img/pull_tool/cmd/pull_tool", + visibility = ["//visibility:private"], + deps = [ + "//cmd/downloadblob", + "//cmd/internal/pull", + ], +) + +go_binary( + name = "pull_tool", + embed = [":pull_tool_lib"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "go_srcs", + srcs = glob(["*.go"]), + visibility = ["//visibility:public"], +) + +build_test( + name = "test", + targets = [":pull_tool"], +) diff --git a/pull_tool/cmd/pull_tool/pull_tool.go b/pull_tool/cmd/pull_tool/pull_tool.go new file mode 100644 index 00000000..9f68fc6a --- /dev/null +++ b/pull_tool/cmd/pull_tool/pull_tool.go @@ -0,0 +1,39 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/bazel-contrib/rules_img/pull_tool/cmd/downloadblob" + "github.com/bazel-contrib/rules_img/pull_tool/cmd/internal/pull" +) + +const usage = `Usage: pull_tool [COMMAND] [ARGS...] + +Commands: + pull pulls an image from a registry + download-blob downloads a single blob from a registry` + +func Run(ctx context.Context, args []string) { + if len(args) < 2 { + fmt.Fprintln(os.Stderr, usage) + os.Exit(1) + } + + command := args[1] + switch command { + case "pull": + pull.PullProcess(ctx, args[2:]) + case "download-blob": + downloadblob.DownloadBlobProcess(ctx, args[2:]) + default: + fmt.Fprintln(os.Stderr, usage) + os.Exit(1) + } +} + +func main() { + ctx := context.Background() + Run(ctx, os.Args) +} diff --git a/pull_tool/go.mod b/pull_tool/go.mod new file mode 100644 index 00000000..c9a9f291 --- /dev/null +++ b/pull_tool/go.mod @@ -0,0 +1,23 @@ +module github.com/bazel-contrib/rules_img/pull_tool + +go 1.24.2 + +require github.com/google/go-containerregistry v0.20.6 + +require ( + cloud.google.com/go/compute/metadata v0.7.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect + github.com/docker/cli v28.2.2+incompatible // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/vbatts/tar-split v0.12.1 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.15.0 // indirect + golang.org/x/sys v0.33.0 // indirect +) diff --git a/pull_tool/go.sum b/pull_tool/go.sum new file mode 100644 index 00000000..99046304 --- /dev/null +++ b/pull_tool/go.sum @@ -0,0 +1,50 @@ +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= +github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v28.2.2+incompatible h1:qzx5BNUDFqlvyq4AHzdNB7gSyVTmU4cgsyN9SdInc1A= +github.com/docker/cli v28.2.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.20.6 h1:cvWX87UxxLgaH76b4hIvya6Dzz9qHB31qAwjAohdSTU= +github.com/google/go-containerregistry v0.20.6/go.mod h1:T0x8MuoAoKX/873bkeSfLD2FAkwCDf9/HZgsFJ02E2Y= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= +github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/pull_tool/pkg/auth/registry/BUILD.bazel b/pull_tool/pkg/auth/registry/BUILD.bazel new file mode 100644 index 00000000..d8e4ee44 --- /dev/null +++ b/pull_tool/pkg/auth/registry/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "registry", + srcs = ["registry.go"], + importpath = "github.com/bazel-contrib/rules_img/pull_tool/pkg/auth/registry", + visibility = ["//visibility:public"], + deps = [ + "@com_github_google_go_containerregistry//pkg/authn:go_default_library", + "@com_github_google_go_containerregistry//pkg/v1/google:go_default_library", + "@com_github_google_go_containerregistry//pkg/v1/remote:go_default_library", + ], +) + +filegroup( + name = "go_srcs", + srcs = glob(["*.go"]), + visibility = ["//visibility:public"], +) diff --git a/pull_tool/pkg/auth/registry/registry.go b/pull_tool/pkg/auth/registry/registry.go new file mode 100644 index 00000000..e71d3d46 --- /dev/null +++ b/pull_tool/pkg/auth/registry/registry.go @@ -0,0 +1,19 @@ +package registry + +import ( + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/v1/google" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// WithAuthFromMultiKeychain returns a remote.Option that uses a MultiKeychain +// combining the default keychain and the Google keychain for authentication. +// WARNING: keep in sync with the same function in img_tool/pkg/auth/registry/registry.go. +func WithAuthFromMultiKeychain() remote.Option { + kc := authn.NewMultiKeychain( + authn.DefaultKeychain, + google.Keychain, + ) + + return remote.WithAuthFromKeychain(kc) +} diff --git a/pull_tool/pull/private/BUILD.bazel b/pull_tool/pull/private/BUILD.bazel new file mode 100644 index 00000000..424dc085 --- /dev/null +++ b/pull_tool/pull/private/BUILD.bazel @@ -0,0 +1,7 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +bzl_library( + name = "pull_bootstrap", + srcs = ["pull_bootstrap.bzl"], + visibility = ["//pull_tool/pull:__subpackages__"], +) diff --git a/pull_tool/pull/private/pull_bootstrap.bzl b/pull_tool/pull/private/pull_bootstrap.bzl new file mode 100644 index 00000000..05e105f0 --- /dev/null +++ b/pull_tool/pull/private/pull_bootstrap.bzl @@ -0,0 +1,62 @@ +"""Repository rule that builds pull_tool using Go binary""" + +load("@go_host_compatible_sdk_label//:defs.bzl", "HOST_COMPATIBLE_SDK") + +def _pull_bootstrap_impl(rctx): + go_sdk_label = Label("@" + rctx.attr._go_sdk_name + "//:ROOT") + go_root = str(rctx.path(go_sdk_label).dirname) + extension = _executable_extension(rctx) + go_tool = go_root + "/bin/go" + extension + go_mod = rctx.path(rctx.attr.go_mod) + go_sum = rctx.path(rctx.attr.go_sum) + src_root = rctx.path(go_mod).dirname + cmd_dir = src_root.get_child("cmd") + pkg_dir = src_root.get_child("pkg") + rctx.watch(go_tool) + rctx.watch(go_mod) + rctx.watch(go_sum) + rctx.watch_tree(cmd_dir) + rctx.watch_tree(pkg_dir) + rctx.symlink(go_mod, "go.mod") + rctx.symlink(go_sum, "go.sum") + rctx.symlink(cmd_dir, "cmd") + rctx.symlink(pkg_dir, "pkg") + for src in rctx.attr.pull_tool_srcs: + rctx.watch(src) + args = [ + go_tool, + "build", + "-o", + rctx.path("./pull_tool.exe"), + "-ldflags=-s -w", + "-trimpath", + "./cmd/pull_tool", + ] + exec_result = rctx.execute( + args, + environment = { + "CGO_ENABLED": "0", + }, + ) + if exec_result.return_code != 0: + fail("go build failed {}: {}{}".format(args, exec_result.stderr, exec_result.stdout)) + rctx.file( + "BUILD.bazel", + """exports_files(["pull_tool.exe"])""", + ) + +pull_bootstrap = repository_rule( + implementation = _pull_bootstrap_impl, + attrs = { + "pull_tool_srcs": attr.label_list(mandatory = True), + "go_mod": attr.label(mandatory = True), + "go_sum": attr.label(mandatory = True), + "_go_sdk_name": attr.string(default = "@" + HOST_COMPATIBLE_SDK.repo_name), + }, +) + +def _executable_extension(rctx): + extension = "" + if rctx.os.name.startswith("windows"): + extension = ".exe" + return extension diff --git a/pull_tool_lockfile.json b/pull_tool_lockfile.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/pull_tool_lockfile.json @@ -0,0 +1 @@ +[]