From 9628735232c1ef7e9d8e44b2abd19bd0f8f2ecf1 Mon Sep 17 00:00:00 2001 From: Max Howell Date: Thu, 30 Nov 2023 12:05:50 -0500 Subject: [PATCH 1/2] brewkit v1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * rewrite in mostly TypeScript * `pkg` -> `bk` to avoid confusion with `pkgx` * proper actions * pkgx pantry specific infra moved to pantry * irrelevant commands removed to be put into `mash` * changes are now fully tested in CI without needing pkgx pantry packages * Fixes #173 * builds occur to separate prefix to aid relocatability testing (Fixes #166) * Refs #160 * Fixes #158 * Fixes #157 * Fixes #129 * Fixes #117 (though I didn’t explicitly fix this we have a fixture showing this working now) * Feat: initializes a git repo in the sources so you can easily get a diff for your build script --- .github/workflows/ci.actions.yml | 138 +++++++++++ .github/workflows/ci.cli.yml | 84 +++++++ .github/workflows/ci.yml | 71 +----- .github/workflows/shellcheck.yml | 13 - .gitignore | 1 + README.md | 159 ++++-------- actions/bottle/action.yml | 37 --- actions/bottle/bottle.ts | 78 ------ actions/cache/action.yml | 32 --- actions/fetch-pr-artifacts/action.yml | 35 --- .../fetch-pr-artifacts/fetch-pr-artifacts.ts | 128 ---------- actions/get-platform/action.yml | 53 ---- actions/get-platform/get-platform.ts | 127 ---------- actions/has-artifacts/action.yml | 48 ---- actions/has-artifacts/has-artifacts.ts | 40 --- actions/setup-brewkit/action.yml | 59 ----- actions/setup-codesign/action.yml | 46 ---- actions/stage-build-artifacts/action.yml | 40 --- .../stage-build-artifacts/cache-artifacts.ts | 34 --- actions/upload/action.yml | 63 ----- actions/upload/upload.ts | 174 ------------- actions/utils/args.ts | 58 ----- actions/utils/gha.ts | 22 -- audit/action.yml | 36 +++ bin/bk | 50 ++++ bin/cache | 85 ------- libexec/audit.ts => bin/cmd/audit | 38 ++- bin/cmd/build | 181 ++++++++++++++ bin/cmd/docker | 33 +++ bin/{pkg-edit => cmd/edit} | 34 ++- bin/{pkg-init => cmd/init} | 16 +- bin/cmd/status | 41 +++ bin/cmd/test | 99 ++++++++ bin/pkg | 47 ---- bin/pkg-build | 143 ----------- bin/pkg-clean | 35 --- bin/pkg-docker | 72 ------ bin/pkg-fetch | 61 ----- bin/pkg-fixup | 32 --- bin/pkg-inventory | 5 - bin/pkg-query | 5 - bin/pkg-search | 5 - bin/pkg-status | 1 - bin/pkg-test | 64 ----- bin/run | 40 --- bottle/action.yml | 31 +++ build/action.yml | 71 ++++++ deno.jsonc | 5 +- download-build-artifact/action.yml | 77 ++++++ lib/actions/platform-key.ts | 5 + lib/actions/stage.ts | 12 + {share/brewkit => lib/bin}/fix-elf.ts | 23 +- {share/brewkit => lib/bin}/fix-machos.rb | 2 +- lib/config.ts | 133 ++++++++++ lib/{ => hooks}/useCache.test.ts | 0 lib/{ => hooks}/useCache.ts | 0 lib/{ => hooks}/useGitHubAPI.ts | 19 +- lib/{ => hooks}/useGitLabAPI.ts | 0 lib/hooks/usePantry.getScript.ts | 233 ++++++++++++++++++ lib/{ => hooks}/usePantry.getVersions.test.ts | 0 lib/{ => hooks}/usePantry.getVersions.ts | 0 lib/{ => hooks}/usePantry.ts | 0 lib/{ => hooks}/useSourceUnarchiver.ts | 2 +- lib/pkgx.ts | 6 +- lib/porcelain/build-script.ts | 83 +++++++ lib/porcelain/fetch.ts | 113 +++++++++ lib/porcelain/fix-up.ts | 94 +++++++ lib/repair.ts | 36 --- lib/run/parse-pkg-str.ts | 21 -- lib/usePantry.getScript.ts | 131 ---------- lib/utils.ts | 26 ++ lib/{ => utils}/Unarchiver.ts | 0 libexec/available.ts | 14 -- libexec/deps.ts | 37 --- libexec/extract.ts | 33 --- libexec/fetch.ts | 152 ------------ libexec/find.ts | 27 -- libexec/fixup.ts | 97 -------- libexec/init.ts | 20 -- libexec/install.ts | 22 -- libexec/inventory.ts | 18 -- libexec/link.ts | 11 - libexec/peek.sh | 40 --- libexec/query.ts | 106 -------- libexec/resolve.ts | 23 -- libexec/sample-blend.rb | 51 ---- libexec/search.ts | 40 --- libexec/sort.ts | 28 --- libexec/stage-test.ts | 100 -------- libexec/stage.ts | 168 ------------- projects/pyapp.com/1/myapp/__init__.py | 0 projects/pyapp.com/1/myapp/main.py | 9 + projects/pyapp.com/1/package.yml | 18 ++ projects/pyapp.com/1/requirements.txt | 1 + projects/pyapp.com/1/setup.py | 15 ++ projects/pyapp.com/2/myapp/__init__.py | 0 projects/pyapp.com/2/myapp/main.py | 9 + projects/pyapp.com/2/package.yml | 18 ++ projects/pyapp.com/2/requirements.txt | 1 + projects/pyapp.com/2/setup.py | 15 ++ projects/stark.com/foo/package-mod.yml | 22 ++ projects/stark.com/foo/package.yml | 42 ++++ projects/toolchain.com/foo.c | 9 + projects/toolchain.com/package.yml | 17 ++ projects/unavailable.com/package.yml | 5 + share/.DS_Store | Bin 6148 -> 0 bytes share/brewkit/install | 11 + share/brewkit/make | 2 +- share/brewkit/patch | 2 +- share/brewkit/pkgx | 11 + share/brewkit/sed | 2 +- share/toolchain/.DS_Store | Bin 6148 -> 0 bytes share/toolchain/shim | 9 +- test/action.yml | 35 +++ upload-build-artifact/action.yml | 45 ++++ 115 files changed, 1971 insertions(+), 3100 deletions(-) create mode 100644 .github/workflows/ci.actions.yml create mode 100644 .github/workflows/ci.cli.yml delete mode 100644 .github/workflows/shellcheck.yml delete mode 100644 actions/bottle/action.yml delete mode 100755 actions/bottle/bottle.ts delete mode 100644 actions/cache/action.yml delete mode 100644 actions/fetch-pr-artifacts/action.yml delete mode 100755 actions/fetch-pr-artifacts/fetch-pr-artifacts.ts delete mode 100644 actions/get-platform/action.yml delete mode 100755 actions/get-platform/get-platform.ts delete mode 100644 actions/has-artifacts/action.yml delete mode 100755 actions/has-artifacts/has-artifacts.ts delete mode 100644 actions/setup-brewkit/action.yml delete mode 100644 actions/setup-codesign/action.yml delete mode 100644 actions/stage-build-artifacts/action.yml delete mode 100755 actions/stage-build-artifacts/cache-artifacts.ts delete mode 100644 actions/upload/action.yml delete mode 100755 actions/upload/upload.ts delete mode 100644 actions/utils/args.ts delete mode 100644 actions/utils/gha.ts create mode 100644 audit/action.yml create mode 100755 bin/bk delete mode 100755 bin/cache rename libexec/audit.ts => bin/cmd/audit (70%) create mode 100755 bin/cmd/build create mode 100755 bin/cmd/docker rename bin/{pkg-edit => cmd/edit} (55%) rename bin/{pkg-init => cmd/init} (76%) create mode 100755 bin/cmd/status create mode 100755 bin/cmd/test delete mode 100755 bin/pkg delete mode 100755 bin/pkg-build delete mode 100755 bin/pkg-clean delete mode 100755 bin/pkg-docker delete mode 100755 bin/pkg-fetch delete mode 100755 bin/pkg-fixup delete mode 100755 bin/pkg-inventory delete mode 100755 bin/pkg-query delete mode 100755 bin/pkg-search delete mode 120000 bin/pkg-status delete mode 100755 bin/pkg-test delete mode 100755 bin/run create mode 100644 bottle/action.yml create mode 100644 build/action.yml create mode 100644 download-build-artifact/action.yml create mode 100755 lib/actions/platform-key.ts create mode 100755 lib/actions/stage.ts rename {share/brewkit => lib/bin}/fix-elf.ts (87%) rename {share/brewkit => lib/bin}/fix-machos.rb (99%) create mode 100644 lib/config.ts rename lib/{ => hooks}/useCache.test.ts (100%) rename lib/{ => hooks}/useCache.ts (100%) rename lib/{ => hooks}/useGitHubAPI.ts (88%) rename lib/{ => hooks}/useGitLabAPI.ts (100%) create mode 100644 lib/hooks/usePantry.getScript.ts rename lib/{ => hooks}/usePantry.getVersions.test.ts (100%) rename lib/{ => hooks}/usePantry.getVersions.ts (100%) rename lib/{ => hooks}/usePantry.ts (100%) rename lib/{ => hooks}/useSourceUnarchiver.ts (93%) create mode 100644 lib/porcelain/build-script.ts create mode 100644 lib/porcelain/fetch.ts create mode 100644 lib/porcelain/fix-up.ts delete mode 100644 lib/repair.ts delete mode 100644 lib/run/parse-pkg-str.ts delete mode 100644 lib/usePantry.getScript.ts rename lib/{ => utils}/Unarchiver.ts (100%) delete mode 100755 libexec/available.ts delete mode 100755 libexec/deps.ts delete mode 100755 libexec/extract.ts delete mode 100755 libexec/fetch.ts delete mode 100755 libexec/find.ts delete mode 100755 libexec/fixup.ts delete mode 100755 libexec/init.ts delete mode 100755 libexec/install.ts delete mode 100755 libexec/inventory.ts delete mode 100755 libexec/link.ts delete mode 100755 libexec/peek.sh delete mode 100755 libexec/query.ts delete mode 100755 libexec/resolve.ts delete mode 100755 libexec/sample-blend.rb delete mode 100755 libexec/search.ts delete mode 100755 libexec/sort.ts delete mode 100755 libexec/stage-test.ts delete mode 100755 libexec/stage.ts create mode 100644 projects/pyapp.com/1/myapp/__init__.py create mode 100644 projects/pyapp.com/1/myapp/main.py create mode 100644 projects/pyapp.com/1/package.yml create mode 100644 projects/pyapp.com/1/requirements.txt create mode 100644 projects/pyapp.com/1/setup.py create mode 100644 projects/pyapp.com/2/myapp/__init__.py create mode 100644 projects/pyapp.com/2/myapp/main.py create mode 100644 projects/pyapp.com/2/package.yml create mode 100644 projects/pyapp.com/2/requirements.txt create mode 100644 projects/pyapp.com/2/setup.py create mode 100644 projects/stark.com/foo/package-mod.yml create mode 100644 projects/stark.com/foo/package.yml create mode 100644 projects/toolchain.com/foo.c create mode 100644 projects/toolchain.com/package.yml create mode 100644 projects/unavailable.com/package.yml delete mode 100644 share/.DS_Store create mode 100755 share/brewkit/install create mode 100755 share/brewkit/pkgx delete mode 100644 share/toolchain/.DS_Store create mode 100644 test/action.yml create mode 100644 upload-build-artifact/action.yml diff --git a/.github/workflows/ci.actions.yml b/.github/workflows/ci.actions.yml new file mode 100644 index 00000000..24a947fb --- /dev/null +++ b/.github/workflows/ci.actions.yml @@ -0,0 +1,138 @@ +on: + pull_request: + workflow_call: + +env: + # FIXME + # we must set this or pkgx won’t find the main pantry after $HOME is changed + # because the default is based on $HOME and we don’t sync if $PKGX_PANTRY_PATH is set + # FIXME add PKGX_HOME to libpkgx + XDG_DATA_HOME: ${{github.workspace}}/.data + +jobs: + build: + runs-on: ${{ matrix.platform.os }} + outputs: + pkgspec: ${{ steps.build.outputs.pkgspec }} + env: + PKGX_PANTRY_PATH: ${{github.workspace}}/co + strategy: + matrix: + platform: + - os: ubuntu-latest + name: linux + - os: macos-latest + name: darwin + steps: + - uses: actions/checkout@v4 + with: + path: co # test that things working isn’t just a coincident of PWD == BREKWIT_DIR + + - uses: pkgxdev/setup@v2 + + - uses: ./co/build + id: build + with: + pkg: stark.com/foo + + - run: test ${{ steps.build.outputs.pkgspec }} = stark.com/foo=2.3.4 + - run: test ${{ steps.build.outputs.project }} = stark.com/foo + - run: test ${{ steps.build.outputs.version }} = 2.3.4 + - run: test ${{ steps.build.outputs.platform }} = ${{ matrix.platform.name }} + - run: test ${{ steps.build.outputs.arch }} = x86-64 + - run: test -d ${{ steps.build.outputs.prefix }} + - run: test $BREWKIT_PKGSPEC = ${{ steps.build.outputs.pkgspec }} + - run: test $BREWKIT_PREFIX = ${{ steps.build.outputs.prefix }} + + - uses: ./co/upload-build-artifact + + test: + needs: build + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + runs-on: ${{ matrix.os }} + env: + PKGX_PANTRY_PATH: ${{github.workspace}}/co + steps: + - uses: actions/checkout@v4 + with: + path: co # test that things working isn’t just a coincident of PWD == BREKWIT_DIR + - uses: pkgxdev/setup@v2 + - uses: ./co/download-build-artifact + - uses: ./co/test + with: + pkg: ${{ needs.build.outputs.pkgspec }} + + audit: + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + env: + PKGX_PANTRY_PATH: ${{github.workspace}}/co + steps: + - uses: actions/checkout@v4 + with: + path: co # test that things working isn’t just a coincident of PWD == BREKWIT_DIR + - uses: pkgxdev/setup@v2 + - uses: ./co/download-build-artifact + - uses: ./co/audit + with: + pkg: ${{ needs.build.outputs.pkgspec }} + + bottle: + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + compression: [xz, gz] + platform: + - darwin+x86-64 + - linux+x86-64 + steps: + - uses: actions/checkout@v4 + with: + path: co # test that things working isn’t just a coincident of PWD == BREKWIT_DIR + + - uses: pkgxdev/setup@v2 + + - uses: ./co/download-build-artifact + id: dl + with: + extract: false + platform: ${{ matrix.platform }} + + - uses: ./co/bottle + id: bottle + with: + file: ${{ steps.dl.outputs.filename }} + compression: ${{matrix.compression}} + + - uses: actions/upload-artifact@v3 + with: + path: ${{ steps.bottle.outputs.filename }} + name: ${{ steps.bottle.outputs.name }} + + unavailable: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pkgxdev/setup@v2 + + - uses: ./build + id: build + with: + pkg: unavailable.com + env: + PKGX_PANTRY_PATH: ${{github.workspace}} + + - run: echo ${{steps.build.outputs.noop}} + + - run: exit 1 + if: ${{steps.build.outputs.noop != 'true'}} diff --git a/.github/workflows/ci.cli.yml b/.github/workflows/ci.cli.yml new file mode 100644 index 00000000..8278f165 --- /dev/null +++ b/.github/workflows/ci.cli.yml @@ -0,0 +1,84 @@ +on: + pull_request: + workflow_call: + +env: + GITHUB_TOKEN: ${{github.token}} + PKGX_PANTRY_PATH: ${{github.workspace}} + # FIXME + # we must set this or pkgx won’t find the main pantry after $HOME is changed + # because the default is based on $HOME and we don’t sync if $PKGX_PANTRY_PATH is set + # FIXME add PKGX_HOME to libpkgx + XDG_DATA_HOME: ${{github.workspace}}/.data + +jobs: + integration-tests: + name: ${{matrix.pkg}} (${{matrix.platform.tag}}) + strategy: + matrix: + platform: + - os: ubuntu-latest + img: debian:buster-slim + tag: linux + - os: macos-latest + tag: mac + pkg: + - stark.com/foo + - toolchain.com + - pyapp.com/1 + - pyapp.com/2 + - stark.com/foo@1.2.3 + runs-on: ${{ matrix.platform.os }} + container: ${{ matrix.platform.img }} + steps: + - uses: actions/checkout@v4 + - uses: pkgxdev/setup@v2 + + - name: precache # makes it easier to read the real output + run: | # we have to make symlinks because deno cache doesn’t take a --ext=ts flag + ln -s bin/cmd/build build.ts + ln -s bin/cmd/test test.ts + pkgx deno cache *.ts + rm *.ts + + - run: bin/bk build ${{matrix.pkg}} + - run: bin/bk test ${{matrix.pkg}} + - run: bin/bk audit ${{matrix.pkg}} + + local-edit-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pkgxdev/setup@v2 + - run: pkgx --sync #FIXME bug where shebangs don’t cause an auto-sync + - uses: fregante/setup-git-user@v2 + - run: git update-ref refs/remotes/origin/main HEAD + - run: '! bin/bk status' + - run: cp package-mod.yml package.yml + working-directory: projects/stark.com/foo + - run: test $(bin/bk status) = stark.com/foo + - run: bin/bk build + - run: bin/bk test + - run: test $(pkgx +stark.com/foo -- stark) = not_much_u + - run: bin/bk audit + + unit-tests: + runs-on: ubuntu-latest + env: + PKGX_PANTRY_PATH: null + steps: + - uses: actions/checkout@v4 + - uses: pkgxdev/dev@main + - run: deno test --allow-env --allow-net --ignore=.data + + unavailable: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pkgxdev/setup@v2 + - run: pkgx --sync # FIXME PKGX_PANTRY_PATH causes auto sync to fail + - name: build + run: | + set +e + bin/bk build unavailable.com + test $? -eq 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6a6cba4..3d10e272 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,71 +1,8 @@ on: - pull_request: workflow_dispatch: jobs: - integration-tests: - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ github.token }} - strategy: - fail-fast: false - matrix: - pkgs: - - zlib.net - - openssl.org^1.1 curl.se/ca-certs - - pipenv.pypa.io - - poppler.freedesktop.org/poppler-data - - catb.org/wumpus - - c-ares.org - - vim.org - - curl.se - # FIXME? requires a darwin platform to run, and needs - # macOS 12 to test. That'll require a more complex matrix - # using get-platform. - # - github.com/realm/SwiftLint - container: - image: debian:buster-slim - steps: - - uses: actions/checkout@v4 - - uses: ./actions/setup-brewkit - - # prefetch deno deps to make the output from the build step more legible - - run: pkgx deno cache **/*.ts - - - run: pkg build ${{matrix.pkgs}} - id: build - - - run: test -n '${{ steps.build.outputs.pkgs }}' - - run: test -n '${{ steps.build.outputs.relative-paths }}' - - - run: | - if pkg query ${{ steps.build.outputs.pkgs }} --src 2>&1 | grep -E '^warn: pkg has no srcs:'; then - echo "srcs=false" >> $GITHUB_OUTPUT - else - echo "srcs=true" >> $GITHUB_OUTPUT - fi - id: srcs - - - run: test -n '${{ steps.build.outputs.srcs }}' - if: steps.srcs.outputs.srcs == 'true' - - - run: test -n '${{ steps.build.outputs.srcs-relative-paths }}' - if: steps.srcs.outputs.srcs == 'true' - - - run: pkg test ${{matrix.pkgs}} - - typecheck: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: pkgxdev/dev@main - - run: deno task typecheck - - unit-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: pkgxdev/dev@main - - run: deno test --allow-env --allow-net - env: - GITHUB_TOKEN: ${{ github.token }} \ No newline at end of file + cli: + uses: ./.github/workflows/ci.cli.yml + actions: + uses: ./.github/workflows/ci.actions.yml diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml deleted file mode 100644 index ba8fe273..00000000 --- a/.github/workflows/shellcheck.yml +++ /dev/null @@ -1,13 +0,0 @@ -on: - pull_request: - paths: - - '**/*.sh' - - 'bin/*' - -jobs: - shellcheck: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: pkgxdev/setup@v1 - - run: pkgx xc shellcheck diff --git a/.gitignore b/.gitignore index be5ebb28..717b5667 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /.DS_Store /deno.lock +/.data diff --git a/README.md b/README.md index ac8c5785..55454474 100644 --- a/README.md +++ b/README.md @@ -2,118 +2,55 @@ # BrewKit -BrewKit is build infrastructure for `pkgx`. +## Usage ```sh -$ env +brewkit -$ pkg build node +$ bk build zlib.net +$ bk test zlib.net ``` -If you are inside a pantry then BrewKit will figure out what packages you are -editing and build them. - -```sh -$ cd pantry -$ dev -$ pkg edit openssl -# make some edits… -$ pkg build -brewkit: building openssl.org -``` - -You can build for Linux (via Docker) using `-L`, e.g.: - -```sh -pkg -L build -``` - -To run `pkg` without shellcode you can do eg. `pkgx +brewkit -- pkg build`, -this is necessary because `pkg` is not listed as a provided program by brewkit -since many other projects have decided to provide `pkg` and we didn’t want -to trump them. - -## Some Details - -* The `pkg build` environment ensures you have a c/c++ compiler and `make` - * We do this to ensure builds on Mac and Linux have a consistent base layer - of tooling - * We also provide shims so use of tools like `pkg-config` just work - * We shim `sed` since Mac comes with BSD sed and Linux typically comes with - GNU sed and they are subtly different -* The `pkg test` environment automatically adds pkgs when called - * We emit a warning for this since sometimes these deps should be explicit - -## Outside a Pantry Checkout - -Outside a pantry checkout we operate against your `pkgx` installation -(which defaults to `~/.pkgx`). Builds occur in a temporary directory rather -than local to your pantry checkout. - -```sh -env "$(pkgx +brewkit)" pkg build zlib.net -``` - - -## Additions - -This repo is for tooling built on top of the `pkgx` primitives with the purpose -of generalized building and testing of open source packages. - -If you have an idea for an addition open a [discussion]! - - -### Stuff That Needs to be Added - -Getting the `rpath` out of a macOS binary: - -```sh -lsrpath() { - otool -l "$@" | - awk ' - /^[^ ]/ {f = 0} - $2 == "LC_RPATH" && $1 == "cmd" {f = 1} - f && gsub(/^ *path | \(offset [0-9]+\)$/, "") == 2 - ' -} -``` - -This should be added to a `pkg doctor` type thing I reckon. E.g. -`pkg doctor zlib.net -Q:rpath`. - - -### Hacking on brewkit - -In a pantry clone, if you do `../brewkit/bin/pkg build` for example your local -brewkit will be used rather than that which is installed. - -  - - - -# Tasks - -## Bump - -Priority is one of `major|minor|patch|prerelease|VERSION` - -Inputs: PRIORITY - -```sh -./scripts/publish-release.sh $PRIORITY -``` - - -## Shellcheck - -```sh -for x in bin/*; do - if file $x | grep 'shell script'; then - pkgx shellcheck --shell=dash --severity=warning $x - fi -done - -pkgx shellcheck --shell=dash --severity=warning **/*.sh -``` - - -[discussion]: https://github.com/orgs/pkgxdev/discussions +> ![TIP] +> If you’re inside a pantry clone then the + +## Build Process Details + +> ![NOTE] +> `$BREWROOT` is either your pantry clone or +> `${XDG_DATA_HOME:-$HOME/.local/share}/brewkit`. + +> ![NOTE] +> `$PKGSLUG` is the pkg project name with slashes replaced with unicode +> slashes and the version appended in `-1.2.3` form. + +1. srcs are placed at `$BREWROOT/srcs/$PKGSLUG` + * if the source type is a tarball, it is stored as `$BREWROOT/srcs/$PKGSLUG.ext` +2. if the src type is not source control a git repository is initialized for + the sources and all files are added to the git stage. + * this is so you can make modifications and get a diff with `git diff` that + you can then save for use in your build script +3. the sources are cloned to `$BREWROOT/builds/$PKGSLUG` via rsync +4. the build script is generated and placed at `$BREWROOT/builds/$PKGSLUG.sh` + * the build script uses pkgx machinery to work, check it out +5. a `pkgx.yaml` is generated to `$BREWROOT/builds/$PKGSLUG` + * this way if you step into that directory and `dev` you get the full build + environment for your pkg +6. the build script is run + * the prefix your build script is fed is `$BREWROOT/installs/$PKGSLUG` +7. `$BREWROOT/builds/$PKGSLUG` is moved to `${PKGX_DIR:-$HOME/.pkgx}` + * without this step you would not be able to use the package from within + your pantry clone +8. Some key tools are always provided by brewkit (via shims that install on + demand). These are: + * `cc`, and the associated build toolchain (specifically latest LLVM) + * if you want a specific LLVM or GCC then specify the dep and that will + be used instead + * `make` + * `patch` + * GNU `install` + * `pkg-config` + * GNU `sed` + +## Apologies + +This repo is optimized for the GitHub Actions calling site and not for +readability. I hate this but it’s the right choice for our users. diff --git a/actions/bottle/action.yml b/actions/bottle/action.yml deleted file mode 100644 index 0ec1c5b1..00000000 --- a/actions/bottle/action.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: pkgx/brewkit/bottle -description: internal pkgx.dev specific at this time - -inputs: - gpg-key-id: - description: gpg key id - required: true - built: - description: packages to bottle - required: true - compression: - description: compression to use (gz or xz) - required: true - platform: - description: eg. darwin+aarch64 - required: false -outputs: - bottles: - description: bottle files - value: ${{ steps.bottle.outputs.bottles }} - checksums: - description: checksum files - value: ${{ steps.bottle.outputs.checksums }} - signatures: - description: signature files - value: ${{ steps.bottle.outputs.signatures }} - -runs: - using: composite - steps: - - run: ${{ github.action_path }}/bottle.ts ${{ inputs.built }} - id: bottle - shell: sh - env: - COMPRESSION: ${{ inputs.compression }} - GPG_KEY_ID: ${{ inputs.gpg-key-id }} - PLATFORM: ${{ inputs.platform }} diff --git a/actions/bottle/bottle.ts b/actions/bottle/bottle.ts deleted file mode 100755 index 4a54f0a2..00000000 --- a/actions/bottle/bottle.ts +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env -S pkgx +gnu.org/tar^1.34 +tukaani.org/xz^5 +zlib.net^1 +gnupg.org^2 +deno.land>=1.32<1.36.1 deno run -A - -import { encode as base64Encode } from "deno/encoding/base64.ts" -import { Installation, Path, hooks, utils } from "pkgx" -import { backticks } from "../../lib/utils.ts" -import { encode } from "deno/encoding/hex.ts" -import { set_output } from "../utils/gha.ts" -import { crypto } from "deno/crypto/mod.ts" -import * as ARGV from "../utils/args.ts" - -const { useCellar, usePrefix, useCache } = hooks -const cellar = useCellar() -const { panic } = utils - -//-------------------------------------------------------------------------- main - -if (import.meta.main) { - const compression = Deno.env.get("COMPRESSION") == 'xz' ? 'xz' : 'gz' - const gpgKey = Deno.env.get("GPG_KEY_ID") ?? panic("missing GPG_KEY_ID") - const checksums: string[] = [] - const signatures: string[] = [] - const bottles: Path[] = [] - - for await (const pkg of ARGV.pkgs()) { - console.info({ bottling: pkg }) - - const installation = await cellar.resolve(pkg) - const path = await bottle(installation, compression) - const checksum = await sha256(path) - const signature = await gpg(path, gpgKey) - - console.info({ bottled: path }) - - bottles.push(path) - checksums.push(checksum) - signatures.push(signature) - } - - await set_output("bottles", bottles.map(b => b.relative({ to: usePrefix() }))) - await set_output("checksums", checksums) - await set_output("signatures", signatures) -} - - -//------------------------------------------------------------------------- funcs -export async function bottle({ path: kegdir, pkg }: Installation, compression: 'gz' | 'xz'): Promise { - const [platform, arch] = Deno.env.get("PLATFORM")?.split('+') ?? [] - const host = platform && arch ? {platform, arch} as any : undefined - const tarball = useCache().path({ pkg, type: 'bottle', compression, host }) - const z = compression == 'gz' ? 'z' : 'J' - const cwd = usePrefix() - const cmd = ["tar", `c${z}f`, tarball, kegdir.relative({ to: cwd })].map(x => x.toString()) - const { success } = await Deno.run({ cmd, cwd: cwd.string }).status() - if (!success) throw new Error("failed to bottle via tar") - return tarball -} - -export async function sha256(file: Path): Promise { - return await Deno.open(file.string, { read: true }) - .then(file => crypto.subtle.digest("SHA-256", file.readable)) - .then(buf => new TextDecoder().decode(encode(new Uint8Array(buf)))) -} - -async function gpg(file: Path, gpgKey: string): Promise { - const rv = await backticks({ - cmd: [ - "gpg", - "--detach-sign", - "--armor", - "--output", - "-", - "--local-user", - gpgKey, - file.string - ] - }) - return base64Encode(rv) -} \ No newline at end of file diff --git a/actions/cache/action.yml b/actions/cache/action.yml deleted file mode 100644 index 8e745545..00000000 --- a/actions/cache/action.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: pkgx/brewkit/actions/cache -description: cache deno deps - -inputs: - cache-name: - description: name of the job to use on the cache key - required: true - -runs: - using: composite - steps: - - run: | - if test "$RUNNER_OS" = "macOS"; then - echo "cache=~/Library/Caches/deno" >> $GITHUB_OUTPUT - else - echo "cache=~/.cache/deno" >> $GITHUB_OUTPUT - fi - echo "GITHUB_WORKSPACE=$GITHUB_WORKSPACE" >> $GITHUB_OUTPUT - id: os-cache - shell: sh - - - uses: actions/cache@v3 - # in containers this has issues because github.workspace != $GITHUB_WORKSPACE - if: github.workspace == steps.os-cache.outputs.GITHUB_WORKSPACE - with: - path: | - ~/.deno - ${{ steps.os-cache.outputs.cache }} - # This isn't perfect (can't hash stuff outside github.workspace, and if the there scripts change, the hash won't) - # additionally, in containerized runs, github.workspace != $GITHUB_WORKSPACE, so we can't use this at all, hence the if above. - # but it's good enough for now. It's slightly conservative, since it monitors all .ts files, but that's fine. - key: ${{ runner.os }}-deno-${{ inputs.cache-name }}-${{ hashFiles('**/deno.jsonc', '**/*.ts') }} diff --git a/actions/fetch-pr-artifacts/action.yml b/actions/fetch-pr-artifacts/action.yml deleted file mode 100644 index 2ecabba8..00000000 --- a/actions/fetch-pr-artifacts/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: pkgx/pantry/fetch-pr-artifacts -description: internal pkgx.dev specific at this time - -inputs: - platform: - description: platform+arch to fetch - required: true - token: - description: github token - default: ${{ github.token }} - required: true - AWS_S3_BUCKET: - description: AWS S3 bucket to use for cache - required: true - AWS_ACCESS_KEY_ID: - description: AWS access key id - required: true - AWS_SECRET_ACCESS_KEY: - description: AWS secret access key - required: true - -runs: - using: composite - steps: - - run: - ${{ github.action_path }}/fetch-pr-artifacts.ts - ${{ github.repository }} - ${{ github.sha }} - ${{ inputs.platform }} >>$GITHUB_ENV - shell: sh - env: - GITHUB_TOKEN: ${{ inputs.token }} - AWS_S3_BUCKET: ${{ inputs.AWS_S3_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }} diff --git a/actions/fetch-pr-artifacts/fetch-pr-artifacts.ts b/actions/fetch-pr-artifacts/fetch-pr-artifacts.ts deleted file mode 100755 index 0f0c9810..00000000 --- a/actions/fetch-pr-artifacts/fetch-pr-artifacts.ts +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env -S pkgx deno run --allow-net --allow-env --allow-write=./artifacts.tgz - -/// Test -/// ./scripts/fetch-pr-artifacts.ts e582b03fe6efedde80f9569403555f4513dbec91 - -import undent from "outdent" -import { utils } from "pkgx" -const { panic } = utils -import { S3 } from "s3" - -/// Main -/// ------------------------------------------------------------------------------- - -if (import.meta.main) { - const usage = "usage: fetch-pr-artifacts.ts {REPO} {SHA} {platform+arch}" - const repo = Deno.args[0] ?? panic(usage) - const ref = Deno.args[1] ?? panic(usage) - const flavor = Deno.args[2] ?? panic(usage) - - const pr = await find_pr(repo, ref) - - if (!pr) throw new Error(`No PR found for commit ${ref} in ${repo}`) - - const s3 = new S3({ - accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!, - secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, - region: "us-east-1", - }) - const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!) - - const key = `pull-request/${repo.split("/")[1]}/${pr}/${flavor}` - const artifacts = (await bucket.getObject(key)) ?? panic("No artifacts found") - - const file = await Deno.open("artifacts.tgz", { create: true, write: true }) - await artifacts.body.pipeTo(file.writable) - - Deno.stdout.write(new TextEncoder().encode(`PR=${pr}`)) -} - -/// Functions -/// ------------------------------------------------------------------------------- - -export async function find_pr(repo: string, ref: string): Promise { - const res = await queryGraphQL(prQuery(repo)) - - const node = res.repository?.ref?.target?.history?.edges.find(n => n.node.oid === ref) - const nodes = node?.node.associatedPullRequests.nodes - if (!nodes || nodes.length === 0) return - return nodes[0].number -} - -async function queryGraphQL(query: string): Promise { - const headers: HeadersInit = {} - const token = Deno.env.get("GITHUB_TOKEN") ?? panic("GitHub GraphQL requires you set $GITHUB_TOKEN") - if (token) headers['Authorization'] = `bearer ${token}` - - const rsp = await fetch('https://api.github.com/graphql', { - method: 'POST', - body: JSON.stringify({ query }), - headers - }) - const json = await rsp.json() - - if (!rsp.ok) { - console.error({ rsp, json }) - throw new Error() - } - - return json.data as T ?? panic("No `data` returns from GraphQL endpoint") -} - -/// Types -/// ------------------------------------------------------------------------------- - -type CommitQuery = { - repository: { - ref: { - target: { - history: { - edges: Node[] - } - } - } - } -} - -type Node = { - node: { - url: URL - oid: string - associatedPullRequests: { nodes: PullRequest[] } - } -} - -type PullRequest = { - number: number -} - -/// Queries -/// ------------------------------------------------------------------------------- - -function prQuery(repo: string): string { - const [owner, name] = repo.split("/") - return undent` - query { - repository(name: "${name}", owner: "${owner}") { - ref(qualifiedName: "main") { - target { - ... on Commit { - history(first: 100) { - edges { - node { - url - oid - associatedPullRequests(first: 1) { - nodes { - number - } - } - } - } - } - } - } - } - } - }` -} \ No newline at end of file diff --git a/actions/get-platform/action.yml b/actions/get-platform/action.yml deleted file mode 100644 index a4f1c94b..00000000 --- a/actions/get-platform/action.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: pkgx/brewkit/get-platform -description: Outputs the platform spec we need for builds - -inputs: - platform: - description: > - The platform+arch to get specs for - required: true - projects: - description: > - The projects to check for availability - required: false - default: '' - -outputs: - os: - description: the OS for general tasks - value: ${{ steps.platform.outputs.os }} - build-os: - description: the OS for build tasks - value: ${{ steps.platform.outputs.build-os }} - container: - description: the container for build tasks - value: ${{ steps.platform.outputs.container }} - test-matrix: - description: the matrix of os/containers for test tasks - value: ${{ steps.platform.outputs.test-matrix }} - available: - description: which packages are available for the requested platform - value: ${{ steps.platform.outputs.available }} - -runs: - using: composite - steps: - - uses: pkgxdev/setup@v1 - - - uses: pkgxdev/brewkit/actions/cache@v0 - with: - cache-name: get-platform - - - uses: actions/checkout@v4 - - - run: echo "GITHUB_WORKSPACE=$GITHUB_WORKSPACE" >> $GITHUB_OUTPUT - id: workspace - shell: sh - - - run: ${{github.action_path}}/get-platform.ts - shell: sh - id: platform - env: - PLATFORM: ${{ inputs.platform }} - PROJECTS: ${{ inputs.projects }} - PKGX_PANTRY_PATH: ${{ steps.workspace.outputs.GITHUB_WORKSPACE }} diff --git a/actions/get-platform/get-platform.ts b/actions/get-platform/get-platform.ts deleted file mode 100755 index 43ff4350..00000000 --- a/actions/get-platform/get-platform.ts +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env -S pkgx deno run -A - -import { utils, hooks } from "pkgx" -const { parse, str } = utils.pkg -const { usePantry } = hooks -const { panic } = utils - -// These are only needed if we switch back to GHA runners - -// const exceptions: { [project: string]: number } = { -// "deno.land": 4, -// "ziglang.org": 8, -// } - -const requiresMacOS12 = ["github.com/realm/SwiftLint"] - -const packages = Deno.env.get("PROJECTS")?.trim().split(" ").filter(x => x).map(parse) - -type Output = { - os: OS, - buildOs: OS, - container?: string, - testMatrix: { os: OS, container?: string }[] - available: string -} - -type OS = string | string[] | { group: string } - -const platform = Deno.env.get("PLATFORM") ?? panic("$PLATFORM not set") - -const available = await (async () => { - const pantry = usePantry() - const platform_ = platform.replace(/\+/, "/") - - if (!packages) return "" - - const rv = [] - - for (const pkg of packages) { - const a = await pantry.getPlatforms(pkg) - if (a.includes(platform_)) rv.push(pkg) - } - - return rv.map(str).join(" ") -})() - -const output: Output = (() => { - switch(platform) { - case "darwin+x86-64": { - // Some packages need macOS 12 - const os = (() => { - if (packages?.some(pkg => requiresMacOS12.includes(pkg.project))) - return "macos-12" - return "macos-11" - })() - return { - os, - buildOs: ["self-hosted", "macOS", "X64"], - testMatrix: [{ os }], - available, - } - } - case "darwin+aarch64": { - const os = ["self-hosted", "macOS", "ARM64"] - return { - os, - buildOs: os, - testMatrix: [{ os }], - available, - } - } - case "linux+aarch64": { - const os = ["self-hosted", "linux", "ARM64"] - return { - os, - buildOs: os, - testMatrix: [{ os }], - available, - } - } - case "linux+x86-64": { - // buildOs: sizedUbuntu(packages), - // const os = "ubuntu-latest" - const os = { group: "linux-x86-64" } // runner group - return { - os, - // buildOs: ["self-hosted", "linux", "X64"], - buildOs: os, - container: "pkgxdev/pkgx:latest", - testMatrix: [ - { os, container: "ubuntu:latest", 'name-extra': "(ubuntu latest)" }, - { os, container: "ubuntu:focal", 'name-extra': "(ubuntu focal)" }, - { os, container: "debian:buster-slim", 'name-extra': "(debian buster)" }, - ], - available, - } - } - default: - throw new Error(`Invalid platform description: ${platform}`) -}})() - -const rv = `os=${JSON.stringify(output.os)}\n` + - `build-os=${JSON.stringify(output.buildOs)}\n` + - `container=${JSON.stringify(output.container)}\n` + - `test-matrix=${JSON.stringify(output.testMatrix)}\n` + - `available=${output.available}\n` - -Deno.stdout.write(new TextEncoder().encode(rv)) - -if (Deno.env.get("GITHUB_OUTPUT")) { - const envFile = Deno.env.get("GITHUB_OUTPUT")! - await Deno.writeTextFile(envFile, rv, { append: true}) -} - -// Leaving this in case we need to switch back to GHA runners - -// function sizedUbuntu(packages: (Package | PackageRequirement)[]): string { -// const size = Math.max(2, ...packages.map(p => exceptions[p.project] ?? 2)) - -// if (size == 2) { -// return "ubuntu-latest" -// } else if ([4, 8, 16].includes(size)) { -// return `ubuntu-latest-${size}-cores` -// } else { -// panic(`Invalid size: ${size}`) -// } -// } \ No newline at end of file diff --git a/actions/has-artifacts/action.yml b/actions/has-artifacts/action.yml deleted file mode 100644 index 1ba1b282..00000000 --- a/actions/has-artifacts/action.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: pkgx/brewkit/has-artifacts -description: Determines if PR artifacts exist for a given SHA - -inputs: - repo: - description: The repo to check for artifacts - required: true - sha: - description: The SHA to check for artifacts - required: true - token: - description: The GitHub token to use - required: true - s3-bucket: - description: The S3 bucket to use - required: true - aws-access-key-id: - description: The AWS access key ID to use - required: true - aws-secret-access-key: - description: The AWS secret access key to use - required: true - -outputs: - has-artifacts: - description: whether there is a PR associated with that SHA with artifacts in staging - value: ${{ steps.has-artifacts.outputs.has-artifacts }} - platforms: - description: the platforms that have artifacts - value: ${{ steps.has-artifacts.outputs.platforms }} - -runs: - using: composite - steps: - - uses: pkgxdev/setup@v1 - - - uses: pkgxdev/brewkit/actions/cache@v0 - with: - cache-name: has-artifacts - - - run: ${{github.action_path}}/has-artifacts.ts ${{ inputs.REPO }} ${{ inputs.SHA }} >>$GITHUB_OUTPUT - shell: sh - id: has-artifacts - env: - GITHUB_TOKEN: ${{ inputs.token }} - AWS_S3_CACHE: ${{ inputs.s3-bucket }} - AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} diff --git a/actions/has-artifacts/has-artifacts.ts b/actions/has-artifacts/has-artifacts.ts deleted file mode 100755 index 9a724671..00000000 --- a/actions/has-artifacts/has-artifacts.ts +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env -S pkgx deno run -A - -/// Test -/// ./scripts/has-artifacts.ts e582b03fe6efedde80f9569403555f4513dbec91 - -import { find_pr } from "../fetch-pr-artifacts/fetch-pr-artifacts.ts" -import { utils } from "pkgx" -import { S3 } from "s3" -const { panic } = utils - -/// Main -/// ------------------------------------------------------------------------------- - -if (import.meta.main) { - const usage = "usage: has-artifacts.ts {REPO} {SHA}" - const repo = Deno.args[0] ?? panic(usage) - const ref = Deno.args[1] ?? panic(usage) - - const pr = await find_pr(repo, ref) - - if (!pr) { - Deno.stdout.write(new TextEncoder().encode("has-artifacts=false")) - Deno.exit(0) - } - - const s3 = new S3({ - accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!, - secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, - region: "us-east-1", - }) - const bucket = s3.getBucket(Deno.env.get("AWS_S3_CACHE")!) - - const objects = await bucket.listObjects({ prefix: `pull-request/${repo.split("/")[1]}/${pr}/` }) - - const hasArtifacts = (objects?.contents?.length || 0) > 0 - const platforms = objects?.contents?.map(o => o.key?.split("/").pop()) ?? [] - - Deno.stdout.write(new TextEncoder().encode(`has-artifacts=${hasArtifacts ? "true" : "false"}\n`)) - Deno.stdout.write(new TextEncoder().encode(`platforms=${JSON.stringify(platforms)}\n`)) -} diff --git a/actions/setup-brewkit/action.yml b/actions/setup-brewkit/action.yml deleted file mode 100644 index 7a76ae8e..00000000 --- a/actions/setup-brewkit/action.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: pkgx/brewkit/setup -description: sets up pkgx, brewkit & caching - -inputs: - prefix: - description: > - Where pkgx stows its packages. - Defaults to `$HOME/.pkgx`. - required: false - version: - description: > - The version of pkgx to install. - Defaults to `latest`. - required: false - pkgs: - description: additional pkgs - required: false -outputs: - PKGX_DIR: - description: See PKGX_DIR in our docs. - value: ${{ steps.foo.outputs.prefix }} - -runs: - using: composite - steps: - - uses: pkgxdev/setup@v1 - id: pkgx - with: - PKGX_DIR: ${{ inputs.prefix }} - version: ${{ inputs.version }} - +: ${{inputs.pkgs }} - - - name: install brewkit - # If v0 is not a symlink, then brewkit cannot - # build itself. This is a workaround for - # https://github.com/pkgxdev/libpkgx/pull/46 - run: | - d="${PKGX_DIR:-$HOME/.pkgx}/pkgx.sh/brewkit" - mkdir -p "$d" - cp -R $GITHUB_ACTION_PATH/../.. "$d/gha-temp" - ln -s gha-temp "$d/v0" - echo "$d/v0/bin" >> $GITHUB_PATH - shell: bash - - - uses: pkgxdev/brewkit/actions/cache@v0 - with: - cache-name: setup - - # $GITHUB_WORKSPACE is correct when using a container - # ${{ github.workspace }} is the runner - - run: | - if test -d "$GITHUB_WORKSPACE"/projects; then - echo "PKGX_PANTRY_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - fi - shell: bash - - - run: echo "PKGX_DIR=${PKGX_DIR:-$HOME/.pkgx}" >> $GITHUB_OUTPUT - id: foo - shell: bash diff --git a/actions/setup-codesign/action.yml b/actions/setup-codesign/action.yml deleted file mode 100644 index 36ebd17c..00000000 --- a/actions/setup-codesign/action.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: pkgx/brewkit/setup-codesign -description: Codesigns macOS binaries using Apple tools - -inputs: - p12-file-base64: - description: Base64 encoded p12 file - required: true - p12-password: - description: Password for p12 file - required: true - -runs: - using: composite - steps: - # Only runs on macOS - - name: Check platform - shell: sh - run: | - if [[ "$RUNNER_OS" != "macOS" ]]; then - echo "This action only runs on macOS" - exit 1 - fi - - # the next three steps bless our code for Apple. It might be the case they should be - # encapulated separately. - # FIXME: using an explicit commit in a PR isn't great, but the last release was almost 3 years - # ago, and we need bugfixes. - # FIXME: replace this with a pkgx script based on https://localazy.com/blog/how-to-automatically-sign-macos-apps-using-github-actions - # github has a doc with similar content, but it's not returning to me atm. - - # apple-actions/import-codesign-certs will fail if the keychain already exists, so we prophylactically - # delete it if it does. - - name: Delete keychain - shell: sh - run: security delete-keychain signing_temp.keychain || true - - - uses: apple-actions/import-codesign-certs@v2 - with: - p12-file-base64: ${{ inputs.p12-file-base64 }} - p12-password: ${{ inputs.p12-password }} - - # Needed for self-hosted runner, since it doesn't destroy itself automatically. - - name: Delete keychain - uses: webiny/action-post-run@3.0.0 - with: - run: security delete-keychain signing_temp.keychain diff --git a/actions/stage-build-artifacts/action.yml b/actions/stage-build-artifacts/action.yml deleted file mode 100644 index abf987ac..00000000 --- a/actions/stage-build-artifacts/action.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: pkgx/brewkit/stage-build-artifacts -description: internal pkgx.dev specific at this time - -inputs: - platform: - description: '' - required: true - AWS_S3_BUCKET: - description: '' - required: true - AWS_ACCESS_KEY_ID: - description: '' - required: true - AWS_SECRET_ACCESS_KEY: - description: '' - required: true - -runs: - using: composite - steps: - - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.platform }} - - - uses: pkgxdev/setup@v1 - - - uses: pkgxdev/brewkit/actions/cache@v0 - with: - cache-name: stage - - - run: ${{ github.action_path }}/cache-artifacts.ts - ${{github.repository}} - ${{github.ref}} - ${{inputs.platform}} - artifacts.tgz - shell: sh - env: - AWS_S3_BUCKET: ${{ inputs.AWS_S3_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }} diff --git a/actions/stage-build-artifacts/cache-artifacts.ts b/actions/stage-build-artifacts/cache-artifacts.ts deleted file mode 100755 index 3a9a53d1..00000000 --- a/actions/stage-build-artifacts/cache-artifacts.ts +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env -S pkgx deno run -A - -import { utils, Path } from "pkgx" -import { S3 } from "s3" -const { panic } = utils - -const usage = "usage: cache-artifacts.ts {REPO} {REF} {destname} {file}" -const repo = Deno.args[0] ?? panic(usage); -const ref = Deno.args[1] ?? panic(usage); -const dest = Deno.args[2] ?? panic(usage); -const artifacts = Deno.args[3] ?? panic(usage); - -if (!repo.startsWith("pkgxdev/")) throw new Error(`offical pkgxdev repos only: ${repo}`) -const pr = parseInt(ref.replace(/refs\/pull\/(\d+)\/merge/, "$1")) -if (isNaN(pr)) throw new Error(`invalid ref: ${ref}`) - -console.info({artifacts}) -console.info({file: Path.cwd().join(artifacts)}) -console.info({exists: Path.cwd().join(artifacts).isFile()}) -console.info({cwd: Path.cwd()}) -const file = Path.cwd().join(artifacts).isFile() ?? panic(`invalid archive: ${Path.cwd().join(artifacts)}`) - -const s3 = new S3({ - accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!, - secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, - region: "us-east-1", -}) -const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!) - -const key = `pull-request/${repo.split("/")[1]}/${pr}/${dest}` -const body = await Deno.readFile(file.string) - -console.info({ uploadingTo: key }) -await bucket.putObject(key, body) diff --git a/actions/upload/action.yml b/actions/upload/action.yml deleted file mode 100644 index 433342e1..00000000 --- a/actions/upload/action.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: pkgx/brewkit/upload -description: internal pkgxdev specific at this time - -inputs: - pkgs: - description: packages to upload - required: true - qa: - description: whether to QA packages - required: true - srcs: - description: source tarballs - required: true - bottles: - description: bottles - required: true - checksums: - description: checksums - required: true - signatures: - description: signature files - required: true - AWS_S3_BUCKET: - description: AWS S3 bucket - required: true - AWS_S3_STAGING_BUCKET: - description: AWS S3 staging bucket for QA packages - required: true - AWS_ACCESS_KEY_ID: - description: AWS access key ID - required: true - AWS_SECRET_ACCESS_KEY: - description: AWS secret access key - required: true -outputs: - cf-invalidation-paths: - description: CloudFront invalidation paths - value: ${{ steps.upload.outputs.cf-invalidation-paths }} - qa-required: - description: JSON array of QA required packages - value: ${{ steps.upload.outputs.qa-required }} - -runs: - using: composite - steps: - - uses: pkgxdev/brewkit/actions/cache@v0 - with: - cache-name: upload - - - run: ${{ github.action_path }}/upload.ts - --pkgs ${{ inputs.pkgs }} - --srcs ${{ inputs.srcs }} - --bottles ${{ inputs.bottles }} - --checksums ${{ inputs.checksums }} - --signatures ${{ inputs.signatures }} - --qa ${{ inputs.qa }} - shell: sh - id: upload - env: - AWS_S3_BUCKET: ${{ inputs.AWS_S3_BUCKET }} - AWS_S3_STAGING_BUCKET: ${{ inputs.AWS_S3_STAGING_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }} diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts deleted file mode 100755 index 3a8aeed0..00000000 --- a/actions/upload/upload.ts +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env -S pkgx +aws.amazon.com/cli +deno.land>=1.32<1.36.1 deno run -A - -import { Package, PackageRequirement, SemVer, Path, semver, hooks, utils } from "pkgx" -import { decode as base64Decode } from "deno/encoding/base64.ts" -const { useOffLicense, useCache } = hooks -import { basename, dirname } from "deno/path/mod.ts" -import { set_output } from "../utils/gha.ts" -import { sha256 } from "../bottle/bottle.ts" -import { retry } from "deno/async/retry.ts" -import { S3, S3Bucket } from "s3" -import usePantry from "../../lib/usePantry.ts" - -//------------------------------------------------------------------------- funcs -function args_get(key: string): string[] { - const it = Deno.args[Symbol.iterator]() - while (true) { - const { value, done } = it.next() - if (done) throw new Error() - if (value === `--${key}`) break - } - const rv: string[] = [] - while (true) { - const { value, done } = it.next() - if (done) return rv - if (value.startsWith("--")) return rv - rv.push(value) - } -} - -function assert_pkg(pkg: Package | PackageRequirement) { - if ("version" in pkg) { - return pkg - } else { - return { - project: pkg.project, - version: new SemVer(pkg.constraint), - } - } -} - -async function get_versions(key: string, pkg: Package, bucket: S3Bucket): Promise { - const prefix = dirname(key) - const got = new Set([pkg.version.toString()]) - - for await (const obj of await bucket.listAllObjects({ batchSize: 200, prefix })) { - if (!obj.key) continue - const base = basename(obj.key) - if (!base.match(/v.*\.tar\.gz$/)) continue - const version = base.replace(/v(.*)\.tar\.gz/, "$1") - got.add(version) - } - - return [...got] - .compact(semver.parse) - .sort(semver.compare) -} - -class ExtBucket { - name: string - bucket: S3Bucket - - constructor(name: string, s3: S3) { - this.bucket = s3.getBucket(name) - this.name = name - } -} - -function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBucket, qaRequired: boolean) { - const key = qaRequired ? `qa/${key_}` : key_ - console.info({ uploading: body, to: key }) - if (!qaRequired) rv.push(`/${key}`) - if (body instanceof Path) { - const args = [ - "s3", - "cp", - body.string, - `s3://${bucket.name}/${key}`, - ] - const env = { - AWS_ACCESS_KEY_ID: Deno.env.get("AWS_ACCESS_KEY_ID")!, - AWS_SECRET_ACCESS_KEY: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, - AWS_DEFAULT_REGION: "us-east-1", - } - return retry(async() => { - const cmd = new Deno.Command("aws", { args, env }).spawn() - const status = await cmd.status - if (!status.success) { - throw new Error(`aws failed with status ${status.code}`) - } - return - }) - } else if (typeof body === "string") { - body = encode(body) - } - // @ts-ignore typescript doesn't narrow the types properly here - return retry(()=>bucket.bucket.putObject(key, body)) -} - -//------------------------------------------------------------------------- main - -if (Deno.args.length === 0) throw new Error("no args supplied") - -const s3 = new S3({ - accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!, - secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, - region: "us-east-1", -}) - -const bucket = new ExtBucket(Deno.env.get("AWS_S3_BUCKET")!, s3) -const stagingBucket = new ExtBucket(Deno.env.get("AWS_S3_STAGING_BUCKET")!, s3) - -const encode = (() => { - const e = new TextEncoder() - return e.encode.bind(e) -})() -const cache = useCache() - -const pkgs = args_get("pkgs").map(utils.pkg.parse).map(assert_pkg) -const srcs = args_get("srcs") -const bottles = args_get("bottles") -const checksums = args_get("checksums") -const signatures = args_get("signatures") -const qaPackages = args_get("qa")[0] === "true" - -const rv: string[] = [] -const qa = new Set() - -for (const [index, pkg] of pkgs.entries()) { - const yml = await usePantry().project(pkg.project).yaml() - const qaRequired = qaPackages && yml?.["test"]?.["qa-required"] === true - const dst = qaRequired ? stagingBucket : bucket - - const bottle = Path.cwd().join(bottles[index]) - const checksum = checksums[index] - const signature = base64Decode(signatures[index]) - const stowed = cache.decode(bottle)! - const key = useOffLicense("s3").key(stowed) - const versions = await get_versions(key, pkg, dst.bucket) - - //FIXME stream the bottle (at least) to S3 - await put(key, bottle, dst, qaRequired) - await put(`${key}.sha256sum`, `${checksum} ${basename(key)}`, dst, qaRequired) - await put(`${key}.asc`, signature, dst, qaRequired) - await put(`${dirname(key)}/versions.txt`, versions.join("\n"), dst, qaRequired) - - // mirror the sources - if (srcs[index] != "~") { - const src = Path.cwd().join(srcs[index]) - if (src.isDirectory()) { - // we almost certainly expanded `~` to the user’s home directory - continue - } - const srcKey = useOffLicense("s3").key({ - pkg: stowed.pkg, - type: "src", - extname: src.extname(), - }) - const srcChecksum = await sha256(src) - const srcVersions = await get_versions(srcKey, pkg, dst.bucket) - await put(srcKey, src, dst, qaRequired) - await put(`${srcKey}.sha256sum`, `${srcChecksum} ${basename(srcKey)}`, dst, qaRequired) - await put(`${dirname(srcKey)}/versions.txt`, srcVersions.join("\n"), dst, qaRequired) - } - - // write .pkgroot for better crawling - await put(`${pkg.project}/.pkgroot`, pkg.project, dst, qaRequired) - - if (qaRequired) { - qa.add(`${pkg.project}@${pkg.version}`) - } -} - -await set_output("cf-invalidation-paths", rv) -await set_output("qa-required", [JSON.stringify([...qa])]) diff --git a/actions/utils/args.ts b/actions/utils/args.ts deleted file mode 100644 index 54af25a2..00000000 --- a/actions/utils/args.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { hooks, utils, Installation, Package, PackageRequirement } from "pkgx" -const { useCellar } = hooks - -/// processes Deno.args unless STDIN is not a TTY and has input -export async function *args(): AsyncGenerator { - if (Deno.isatty(Deno.stdin.rid)) { - for (const arg of Deno.args) { - if (arg[0] != '-') yield arg - } - } else { - let yielded_something = false - const buf = new Uint8Array(10) - const decode = (() => { const d = new TextDecoder(); return d.decode.bind(d) })() - let n: number | null - let txt = '' - const rx = /\s*(.*?)\s+/ - while ((n = await Deno.stdin.read(buf)) !== null) { - txt += decode(buf.subarray(0, n)) - while (true) { - const match = txt.match(rx) - if (!match) break - yield match[1] - txt = txt.slice(match[0].length) - yielded_something = true - } - } - if (txt) { - yield txt - } else if (!yielded_something) { - for (const arg of Deno.args) { - yield arg - } - } - } -} - -export async function *pkgs(): AsyncGenerator { - for await (const arg of args()) { - const match = arg.match(/projects\/(.*)\/package.yml/) - const project = match ? match[1] : arg - yield utils.pkg.parse(project) - } -} - -export async function *installs(): AsyncGenerator { - const cellar = useCellar() - for await (const pkg of pkgs()) { - yield await cellar.resolve(pkg) - } -} - -export async function toArray(input: AsyncGenerator) { - const rv: T[] = [] - for await (const i of input) { - rv.push(i) - } - return rv -} diff --git a/actions/utils/gha.ts b/actions/utils/gha.ts deleted file mode 100644 index 591e386b..00000000 --- a/actions/utils/gha.ts +++ /dev/null @@ -1,22 +0,0 @@ -const e = new TextEncoder() -const encode = e.encode.bind(e) - -export async function set_output(name: string, arr: T[], separator = " ") { - const value = arr.map(escape).join(separator) - const txt = `${name}=${value}` - const outfile = Deno.env.get("GITHUB_OUTPUT") - if (outfile) { - await Deno.writeTextFile(outfile, `${name}=${value}\n`, { append: true}) - } - return await Deno.stdout.write(encode(`${txt}\n`)) -} - -//TODO HTML escapes probs -function escape(input: T): string { - const out = `${input}` - if (/[<>~]/.test(out)) { - return `"${out}"` - } else { - return out - } -} diff --git a/audit/action.yml b/audit/action.yml new file mode 100644 index 00000000..43c9c44d --- /dev/null +++ b/audit/action.yml @@ -0,0 +1,36 @@ +name: pkgx/brewkit/audit + +inputs: + pkg: + description: | + * eg. pkgx.sh@1.1 + * if unset uses `$BREWKIT_PKGSPEC` + * if unset checks for modifications in `$PKGX_PANTRY_PATH` + required: false + token: + default: ${{ github.token }} + required: true + +runs: + using: composite + steps: + - run: | + if ! pkgx --sync; then + echo "::error::you must use: pkgxdev/setup before using this action" + exit 1 + fi + shell: bash + + - name: fetch deno deps + shell: bash + run: | + echo "::group::fetch deno deps" + tmpdir=$(mktemp -d) + ln -s ${GITHUB_ACTION_PATH}/../bin/cmd/audit $tmpdir/audit.ts + pkgx deno cache $tmpdir/audit.ts --config=${GITHUB_ACTION_PATH}/../deno.jsonc + echo "::endgroup::" + + - run: ${GITHUB_ACTION_PATH}/../bin/cmd/audit ${{ inputs.pkg }} + shell: bash + env: + GITHUB_TOKEN: ${{inputs.token}} diff --git a/bin/bk b/bin/bk new file mode 100755 index 00000000..0e5fbb91 --- /dev/null +++ b/bin/bk @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +set -e + +d="$(cd "$(dirname "$0")" && pwd)" + +if [ -z "$1" ] || [ "$1" = '--help' ] || [ "$1" = '-h' ]; then + echo usage: >&2 + echo " $0 [args]" >&2 + echo >&2 + echo available commands: >&2 + for x in $(ls "$d"/cmd); do + echo " $x" >&2 + done + echo >&2 + echo • some commands provide \`--help\` >&2 + echo • "you can (unambiguously) shorten commands, eg. \`bk b\`" + echo • https://github.com/pkgxdev/brewkit + if [ -z "$1" ]; then + exit 64 + else + exit 0 + fi +fi + +arg="$1" +shift + +if test "$arg" = "-v" || test "$arg" = "--version"; then + d="$(cd "$(dirname "$0")"/.. && pwd)" + if echo "$d" | grep -q "${PKGX_DIR:-$HOME/.pkgx}/pkgx.sh/brewkit"; then + V="${d#"${PKGX_DIR:-$HOME/.pkgx}"/pkgx.sh/brewkit/}" + else + V="$(git describe --tags --abbrev=0 --match "v[0-9]*.[0-9]*.[0-9]*")-dev" + fi + echo "brewkit $V" + exit 0 +fi + +if test -x "$d/cmd/$arg"; then + exec "$d/cmd/$arg" "$@" +elif test "$(find "$d/cmd" -name "$arg*" | wc -l)" -eq 1; then + exec "$d"/cmd/"$arg"* "$@" +else + echo "error: unknown or ambiguous command \`$arg\`, available commands:" >&2 + echo >&2 + echo " "$(ls "$d"/cmd) >&2 + exit 64 +fi diff --git a/bin/cache b/bin/cache deleted file mode 100755 index 34129c4b..00000000 --- a/bin/cache +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env -S pkgx deno run -A --unstable --ext=ts - -import { hooks, semver, SemVer, utils } from "pkgx" -import { Command } from "cliffy/command/mod.ts" -const { useConfig, useCellar } = hooks -import repair from "../lib/repair.ts" - -const { args: pkgnames, options: flags, cmd } = await new Command() - .name("pkgx cache") - .description("pkgx/brewkit cellar management") - .arguments("[pkgspecs...]") - .option("-R, --repair", "fixes the vx, v.x.y and v* symlinks (if necessary)") - .option("-p, --prune", "prune old versions") - .option("-d, --dryrun", "don’t do stuff, just print") - .example("pkgx pkg cellar -Rp", "prunes and repairs all installed pkgs") - .parse(Deno.args) - -if (!flags.prune && !flags.repair) { - cmd.showHelp() - Deno.exit(1) -} - -if (flags.prune) { - const versions: Record = {} - for await (const {pkg: {project, version}} of ls()) { - versions[project] ??= [] - versions[project].push(version) - } - - for (const project in versions) { - switch (project) { - case 'python.org': - case 'node.org': - case 'haskell.org': - console.warn("not pruning", project, "due to laziness. PR welcome!") - break - default: - for (const version of versions[project].sort(semver.compare).slice(0, -1)) { - console.debug({ version, project }) - const install = await useCellar().resolve({ project, version }) - //TODO only print if needs pruning - console.error("pruning:", install.path) - if (!flags.dryrun) install.path.rm({ recursive: true }) - } - } - } - - if (flags.repair == undefined) { - flags.repair = true - } -} - -if (flags.repair) { - for await (const {pkg: {project}} of ls()) { - //TODO only print (and obv. run repair) if needs to be repaired - console.error("repairing:", project) - if (!flags.dryrun) await repair(project) - } -} - -async function* ls() { - if (pkgnames.length == 0) { - const stack = [useConfig().prefix] - while (stack.length > 0) { - const dir = stack.pop()! - for await (const {isDirectory, isSymlink, name} of Deno.readDir(dir.string)) { - const path = dir.join(name) - - if (isDirectory && !isSymlink) { - const version = semver.parse(name) - if (version) { - const project = path.parent().relative({ to: useConfig().prefix }) - yield { pkg: { project, version }, path } - } else { - // only descend if not a vx.y.z dir - stack.push(path) - } - } - } - } - } else for (const pkgname of pkgnames) { - const rq = utils.pkg.parse(pkgname) - yield useCellar().resolve(rq) - } -} diff --git a/libexec/audit.ts b/bin/cmd/audit similarity index 70% rename from libexec/audit.ts rename to bin/cmd/audit index bd92f9b3..d808ea02 100755 --- a/libexec/audit.ts +++ b/bin/cmd/audit @@ -1,20 +1,38 @@ -#!//usr/bin/env -S pkgx deno run --allow-env --allow-read +#!//usr/bin/env -S pkgx +git +gh deno run --allow-env --allow-read --allow-run --allow-net --ext=ts -import { parseFlags } from "cliffy/flags/mod.ts" -import { hooks, utils } from "pkgx" -import { swallow } from "../lib/utils.ts" +import { Command } from "cliffy/command/mod.ts" +import { swallow } from "brewkit/utils.ts" +import get_config from "brewkit/config.ts" +import { Package } from "pkgx" +import { hooks } from "pkgx" const { useCellar, usePantry, useMoustaches } = hooks -const { parse } = utils.pkg -const { unknown: pkgnames } = parseFlags(Deno.args) + +const { args } = await new Command() + .name("bk audit") + .description("Audit pkgx pantry pkgs for common issues") + .arguments("[pkgspec]") + .parse(); + +if (args.length > 1) { + throw new Error("too many arguments, we can only handle one pkgspec at a time currently") +} + +const config = await get_config(args[0]) const pantry = usePantry() const cellar = useCellar() const moustaches = useMoustaches() -const missing = [] +const missing: [string, string][] = [] + +if (!config.pkg) { + fail('error: no packages specified') +} + +await audit(config.pkg) -for(const pkg of pkgnames.map(parse)) { +async function audit(pkg: Package) { const { path, pkg: { version } } = await cellar.resolve(pkg) const versionMap = moustaches.tokenize.version(version) @@ -24,7 +42,9 @@ for(const pkg of pkgnames.map(parse)) { const name = moustaches.apply(provide, versionMap) const bin = path.join('bin', name) const sbin = path.join('sbin', name) - if (!bin.isExecutableFile() && !sbin.isExecutableFile()) missing.push([pkg.project, name]) + if (!bin.isExecutableFile() && !sbin.isExecutableFile()) { + missing.push([pkg.project, name]) + } } if (missing.length) { diff --git a/bin/cmd/build b/bin/cmd/build new file mode 100755 index 00000000..1a8ac89f --- /dev/null +++ b/bin/cmd/build @@ -0,0 +1,181 @@ +#!/usr/bin/env -S pkgx +rsync +git +bash +gum +gh +curl +bzip2 +xz deno run --ext=ts --allow-env --allow-read --allow-write --allow-run --allow-net + +import make_build_script from "brewkit/porcelain/build-script.ts" +import { gum, rsync } from "brewkit/utils.ts" +import fix_up from "brewkit/porcelain/fix-up.ts" +import { Command } from "cliffy/command/mod.ts" +import fetch from "brewkit/porcelain/fetch.ts" +import get_config from "brewkit/config.ts" +import { Path, hooks, utils } from "pkgx" +import * as YAML from "deno/yaml/mod.ts" +const { useConfig } = hooks +const { host } = utils + +const { options, args } = await new Command() + .name("build") + .description("Build pkgx pantry pkgs with brewkit") + .option("-C, --clean=", "Clean everything first") + .arguments("[pkgspec]") + .parse(); + +if (args.length > 1) { + throw new Error("too many arguments, we can only handle one pkgspec at a time currently") +} + +await gum('computing configuration') +const config = await get_config(args[0]) +console.log("pkg:", utils.pkg.str(config.pkg)) +console.log("src:", config.path.src) +console.log("build:", config.path.build) +console.log("home:", config.path.home) +console.log("install:", config.path.install) +console.log("build-install:", config.path.build_install) + +if (options.clean) { + gum("# cleaning") + config.path.home.rm({recursive: true}) + config.path.install.rm({recursive: true}) + config.path.build_install.rm({recursive: true}) + config.path.build.rm({recursive: true}) + config.path.src.rm({recursive: true}) + config.path.test.rm({recursive: true}) +} + +/// warnings +if (config.path.build.string.includes(" ")) { + console.warn("warn: build directory contains spaces, many many build tools choke, so we’re aborting") + console.warn(" ", config.path.build.string) + console.warn("note: we intend to fix this by building to /tmp. open a ticket and we’ll do it") + Deno.exit(1) +} + +if (!await hooks.usePantry().project(config.pkg.project).available()) { + console.warn("warn: project not available on this platform") + Deno.exit(2) +} + +const yml = YAML.parse(await config.path.yaml.read()) as any + +/// fetch +await gum('fetch & extract') +if (yml.distributable) { + await fetch(config) +} else { + console.log("no srcs, skipping fetch") +} + +/// rsync sources & props to build stage +await gum('stage') +if (yml.distributable) { + await rsync(config.path.src, config.path.build, ['--exclude=.git']) +} +await rsync(config.path.yaml.parent(), config.path.build.join("props")) + +/// write out pkgx.yaml +// config.path.build.join("pkgx.yaml").write({ text: YAML.stringify({ +// env: sh.flatten(env), +// dependencies: deps.reduce((acc, {pkg}) => { +// acc[pkg.project] = `=${pkg.version}` +// return acc +// }, {} as Record) +// }), force: true }) + +/// create toolchain if necessary +const toolchain_PATH = make_toolchain() + +/// write script +const script_content = await make_build_script(config, toolchain_PATH) +const script = new Path(`${config.path.build}.sh`) +console.log('writing', script) +script.write({text: script_content, force: true}).chmod(0o755) + +/// run script +await gum('build') + +const env: Record = { + PATH: '/usr/bin:/bin:/usr/sbin:/sbin', + PKGX_DIR: useConfig().prefix.string +} +for (const key of ['HOME', 'PKGX_PANTRY_PATH', 'XDG_DATA_HOME', 'GITHUB_TOKEN']) { + const value = Deno.env.get(key) + if (value) env[key] = value +} +if (env['GITHUB_TOKEN']) { + // for `gh` + env['GH_TOKEN'] = env['GITHUB_TOKEN'] +} + +const proc = new Deno.Command(script.string, {clearEnv: true, env}).spawn() +const rv = await proc.status +if (!rv.success) throw new Error(`${rv.code} ${rv.signal}`) + +/// move installation products to destination +await gum(`rsync install to final path`) +await rsync(config.path.build_install, config.path.install) +config.path.build_install.rm({recursive: true}) + +/// fix rpaths and other relocatability issues +await gum('fix-ups') +await fix_up(config) + +const ghout = Deno.env.get("GITHUB_OUTPUT") +if (ghout) { + const { platform, arch } = host() + const pkgspec = `${utils.pkg.str(config.pkg)}` + Deno.writeTextFileSync(ghout, `pkgspec=${pkgspec}\n`, { append: true}) + Deno.writeTextFileSync(ghout, `project=${config.pkg.project}\n`, { append: true}) + Deno.writeTextFileSync(ghout, `version=${config.pkg.version}\n`, { append: true}) + Deno.writeTextFileSync(ghout, `prefix=${config.path.install.string}\n`, { append: true}) + Deno.writeTextFileSync(ghout, `platform=${platform}\n`, { append: true}) + Deno.writeTextFileSync(ghout, `arch=${arch}\n`, { append: true}) + const ghenv = Deno.env.get("GITHUB_ENV") + if (ghenv) { + Deno.writeTextFileSync(ghenv, `BREWKIT_PKGSPEC=${pkgspec}\n`, { append: true }) + Deno.writeTextFileSync(ghenv, `BREWKIT_PREFIX=${config.path.install.string}\n`, { append: true }) + } +} + +/////////////////////////////////////////////////////////////////// +function make_toolchain() { + const deps = new Set(config.deps.dry.build.concat(config.deps.dry.runtime).map(x => x.project)) + if (deps.has('llvm.org') || deps.has('gnu.org/gcc')) { + return + } + + if (host().platform != "darwin") { + const d = config.path.home.join('toolchain').mkdir('p') + + const symlink = (names: string[], {to}: {to: string}) => { + for (const name of names) { + const path = d.join(name) + if (path.exists()) continue + const target = useConfig().prefix.join('llvm.org/v*/bin', to) + path.ln('s', { target }) + } + } + + symlink(["cc", "gcc", "clang"], {to: "clang"}) + symlink(["c++", "g++", "clang++"], {to: "clang++"}) + symlink(["cpp"], {to: "clang-cpp"}) + + if (host().platform == "linux") { + symlink(["ld"], {to: "ld.lld"}) + } else if (host().platform == "windows") { + symlink(["ld"], {to: "lld-link"}) + } + + symlink(["ld.lld"], {to: "ld.lld"}) + symlink(["lld-link"], {to: "lld-link"}) + + symlink(["ar"], {to: "llvm-ar"}) + symlink(["as"], {to: "llvm-as"}) + symlink(["nm"], {to: "llvm-nm"}) + symlink(["objcopy"], {to: "llvm-objcopy"}) + symlink(["ranlib"], {to: "llvm-ranlib"}) + symlink(["readelf"], {to: "llvm-readelf"}) + symlink(["strings"], {to: "llvm-strings"}) + symlink(["strip"], {to: "llvm-strip"}) + } + + return new Path(new URL(import.meta.url).pathname).join("../../../share/toolchain/bin") +} diff --git a/bin/cmd/docker b/bin/cmd/docker new file mode 100755 index 00000000..fc4a028d --- /dev/null +++ b/bin/cmd/docker @@ -0,0 +1,33 @@ +#!/usr/bin/env -S pkgx +docker bash + +set -eo pipefail + +if [ ! -d "$PKGX_PANTRY_PATH" ]; then + echo "error: \`brewkit docker\` can only be run from a pantry checkout" >&2 + exit 64 +fi + +d="$(cd "$(dirname "$0")/../.." && pwd)" + +if [ -z "$GITHUB_TOKEN" ]; then + GITHUB_TOKEN=$(pkgx gh auth token) +fi + +CMD=$1 +shift + +mkdir -p "$PKGX_PANTRY_PATH/prefixes/linux" + +exec docker run \ + --name brewkit.pkgx.dev \ + --rm \ + --tty \ + --volume "$d:/brewkit" \ + --volume "$PKGX_PANTRY_PATH:/work" \ + --volume "${XDG_CACHE_HOME:-$HOME/Library/Caches/pkgx}:/root/.cache/pkgx" \ + --env PKGX_PANTRY_PATH=/work \ + --env GITHUB_TOKEN=$GITHUB_TOKEN \ + --env CLICOLOR_FORCE=1 \ + --workdir /work \ + pkgxdev/pkgx \ + /brewkit/bin/cmd/$CMD "$@" diff --git a/bin/pkg-edit b/bin/cmd/edit similarity index 55% rename from bin/pkg-edit rename to bin/cmd/edit index 7e836e90..a96b17c1 100755 --- a/bin/pkg-edit +++ b/bin/cmd/edit @@ -1,24 +1,30 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash +#!/usr/bin/env -S pkgx +deno bash set -eo pipefail -d="$(cd "$(dirname "$0")"/.. && pwd)" -PATH="$d/libexec:$PATH" - if test -z "$1"; then if test -z "$PKGX_PANTRY_PATH"; then echo "error: PKGX_PANTRY_PATH is not set" >&2 exit 1 fi + d="$(cd "$(dirname "$0")" && pwd)" for x in $(echo "$PKGX_PANTRY_PATH" | tr ':' '\n'); do if test -z "$PKGS" -a -d "$x"/.git; then #^^^^^^^^^^^^^^^ stop at first match - PKGS="$(GIT_WORK_TREE="$x" peek.sh --print-paths)" + PKGS=($(GIT_WORK_TREE="$x" "$d/status" --print-paths)) fi done else - PKGS="$(_PATHS=1 find.ts $@)" + PKGS=($(deno run --allow-read --allow-env - < hooks.usePantry().find(arg).then(x => x.map(y => y.path.string))); + const paths = await Promise.all(pp); + console.log(paths.flatMap(x => x).join("\n")); + Deno.exit(0); +EoTS + )) + + echo "${PKGS[@]}" fi if test -z "$EDITOR"; then @@ -37,17 +43,19 @@ if test -z "$EDITOR"; then elif test "$EDITOR" = code_wait; then # this is for mxcl who generally prefers his editor to wait # but not in this case. #perks-of-making-the-thing - EDITOR="code" + EDITOR=code fi if test -z "$PKGS"; then - echo "error: no new packages in \`\$PKGX_PANTRY_PATH\`" >&2 + if [ -n "$PKGX_PANTRY_PATH" ]; then + echo "error: no new packages in \`\$PKGX_PANTRY_PATH\`" >&2 + else + echo "usage: bk edit " >&2 + fi exit 1 elif test "$EDITOR" = code; then - # shellcheck disable=2086 - exec $EDITOR "$PKGX_PANTRY_PATH" $PKGS + exec $EDITOR "$PKGX_PANTRY_PATH" "${PKGS[@]}" # ^^ be more useful else - # shellcheck disable=2086 - exec $EDITOR $PKGS + exec $EDITOR "${PKGS[@]}" fi diff --git a/bin/pkg-init b/bin/cmd/init similarity index 76% rename from bin/pkg-init rename to bin/cmd/init index 90ae2023..fc30dd84 100755 --- a/bin/pkg-init +++ b/bin/cmd/init @@ -1,4 +1,4 @@ -#!/usr/bin/env -S pkgx bash>=4 +#!/usr/bin/env -S pkgx +shuf bash>=4 # shellcheck shell=bash set -eo pipefail @@ -22,14 +22,13 @@ if ! git status --porcelain; then fi d="$(cd "$(dirname "$0")"/.. && pwd)" -PATH="$d/libexec:$PATH" -if ! command -v ruby >/dev/null; then - RUBY="pkgx +ruby" +if [ -f /usr/share/dict/words ]; then + BLEND=$(paste <(shuf -n 1 /usr/share/dict/words) <(shuf -n 1 /usr/share/dict/words) -d '-') +else + BLEND=example.com fi -BLEND="$(init.ts "$1")" - mkdir -p "$WHERE/projects/$BLEND" cp -i "$d/share/TEMPLATE.pkg.yml" "$WHERE/projects/$BLEND/package.yml" @@ -41,12 +40,13 @@ fi if test "$(git rev-parse --abbrev-ref HEAD)" != "new/$BLEND"; then git checkout -b "new/$BLEND" origin/main + git branch --unset-upstream # ask jacob why fi pkgx gum format <&2 + exit 1 +fi + +if ! test -d "$d"/projects; then + echo "brewkit: error: \`$PWD\` is not a pantry" >&2 + exit 2 +fi + +# sadly we seemingly need to reference origin/main +DIVERGENCE_SHA="$(git merge-base HEAD origin/main)" +CHANGED_FILES="$(git diff --name-only "$DIVERGENCE_SHA") $(git status --untracked-files --porcelain)" + +OUTPUT="" + +for CHANGED_FILE in $CHANGED_FILES; do + PROJECT=$(echo "$CHANGED_FILE" | sed -n 's#projects/\(.*\)/package\.yml$#\1#p') + if test -z "$PROJECT" + then + true # noop + elif test "$1" = "--print-paths"; then + OUTPUT="$OUTPUT $CHANGED_FILE" + else + OUTPUT="$OUTPUT $PROJECT" + fi +done + +if [ -z "$OUTPUT" ]; then + echo "nothing modified in pantries" >&2 + exit 1 +else + echo $OUTPUT | tr ' ' '\n' | sort | uniq | column +fi diff --git a/bin/cmd/test b/bin/cmd/test new file mode 100755 index 00000000..611d273e --- /dev/null +++ b/bin/cmd/test @@ -0,0 +1,99 @@ +#!/usr/bin/env -S pkgx +bash +gum +gh +rsync deno run --ext=ts --allow-env --allow-read --allow-write --allow-net --allow-run + +//TODO net required because we go to github for version info, but really we should require +// a built product that is then recorded for us to use + +import { Package, PackageRequirement, Path, hooks, utils } from "pkgx" +import { gum, find_pkgx, rsync, find_in_PATH } from "brewkit/utils.ts" +import get_config from "brewkit/config.ts" +import * as YAML from "deno/yaml/mod.ts" +import undent from "outdent" +const { useConfig, usePantry } = hooks + +await gum('computing configuration') +const config = await get_config(Deno.args[0]) +console.log("pkg:", utils.pkg.str(config.pkg)) +console.log("testbed:", config.path.test) +console.log("home:", config.path.home) +console.log("install:", config.path.install) + +const yml = YAML.parse(await config.path.yaml.read()) as any +if (!yml.test) throw "no `test` node in package.yml" + +await gum("stage") +await rsync(config.path.yaml.parent(), config.path.test, ['--exclude=package.yml']) + + +await gum('writing test script') + +const pkgx = find_pkgx() +const bash = find_in_PATH('bash') +const gum_ = find_in_PATH('gum') + +const depstr = (deps: (PackageRequirement | Package)[]) => deps.map(x => `+${utils.pkg.str(x)}`).join(' ') +const env_plus = `${depstr([config.pkg])} ${depstr(config.deps.dry.runtime)} ${depstr(config.deps.dry.test)}`.trim() + +const user_script = await usePantry().getScript(config.pkg, 'test', config.deps.gas, config.path.install) + +const script_text = undent` + #!${bash} + + set -eo pipefail + + export PKGX_DIR="${useConfig().prefix}" + + ${gum_} format "## env" + set -a + eval "$(CLICOLOR_FORCE=1 ${pkgx} ${env_plus})" || exit $? + set +a + + command_not_found_handle() { + echo "::warning::\\\`$1\\\` is not an explicit dependency!" + case $1 in + cc|c++|ld) + ${pkgx} +llvm.org -- "$@";; + *) + ${pkgx} "$@";; + esac + } + + export HOME="${config.path.home}" + ${fixture()} + + env -u GH_TOKEN -u GITHUB_TOKEN + + ${gum_} format "## pantry script start" + set -x + cd "${config.path.test}" + + ${user_script} + ` + +function fixture() { + if (yml.test.fixture) { + const fixture = config.path.test.join("dev.pkgx.fixture").write({ text: yml.test.fixture.toString() }) + return `export FIXTURE="${fixture}"` + } else { + return '' + } +} + +const script = new Path(`${config.path.test}.sh`) +console.log("writing", script) +script.write({ force: true, text: script_text }).chmod(0o755) + +await gum("test") + +const env: Record = {PATH: '/usr/bin:/bin:/usr/sbin:/sbin'} +for (const key of ['HOME', 'PKGX_PANTRY_PATH', 'XDG_DATA_HOME', 'GITHUB_TOKEN']) { + const value = Deno.env.get(key) + if (value) env[key] = value +} +if (env['GITHUB_TOKEN']) { + // for `gh` + env['GH_TOKEN'] = env['GITHUB_TOKEN'] +} + +const proc = new Deno.Command(script.string, {clearEnv: true, env}).spawn() +const rv = await proc.status +if (!rv.success) throw new Error(`${rv.code} ${rv.signal}`) diff --git a/bin/pkg b/bin/pkg deleted file mode 100755 index 1ace5cb5..00000000 --- a/bin/pkg +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -set -e - -if test -z "$1"; then - echo available commands: - echo - d="$(cd "$(dirname "$0")" && pwd)" - find "$d" -type f | sed -ne 's/^.*\/pkg-/pkg /p' - exit 1 -fi - -arg="$1" -shift - -if test "$arg" = "-v" || test "$arg" = "--version"; then - d="$(cd "$(dirname "$0")"/.. && pwd)" - if echo "$d" | grep -q "${PKGX_DIR:-$HOME/.pkgx}/pkgx.sh/brewkit"; then - V="${d#"${PKGX_DIR:-$HOME/.pkgx}"/pkgx.sh/brewkit/}" - else - V="$(git describe --tags --abbrev=0 --match "v[0-9]*.[0-9]*.[0-9]*")-dev" - fi - echo "brewkit: $V" - exit 0 -fi - -if test "$arg" = "-L"; then - export PKGX_DOCKER=1 - - if test -n "$1"; then - arg="$1" - shift - else - arg="docker" - fi -fi - -d="$(cd "$(dirname "$0")" && pwd)" -if test -x "$d"/pkg-"$arg"; then - exec "$d"/pkg-"$arg" "$@" -elif test "$(find "$d" -name "pkg-$arg*" | wc -l)" -eq 1; then - exec "$d"/pkg-"$arg"* "$@" -else - echo "brewkit: error: unknown or ambiguous command '$arg'" >&2 - exit 1 -fi diff --git a/bin/pkg-build b/bin/pkg-build deleted file mode 100755 index f5c883f9..00000000 --- a/bin/pkg-build +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -set -eo pipefail - -PKGX_DIR=${PKGX_DIR:-"$HOME/.pkgx"} - -if pwd | grep -q ' '; then - echo "brewkit: warning: current directory contains spaces. builds *will* break!" >&2 -fi - -if test -n "$RUNNER_DEBUG" -a -n "$GITHUB_ACTIONS" -o -n "$VERBOSE"; then - set -x -fi - -d="$(cd "$(dirname "$0")"/.. && pwd)" -PATH="$d/libexec:$PATH" - -if test -z "$1"; then - if test -z "$PKGX_PANTRY_PATH"; then - echo "error: PKGX_PANTRY_PATH is not set" >&2 - exit 1 - fi - for x in $(echo "$PKGX_PANTRY_PATH" | tr ':' '\n'); do - if test -d "$x"/.git; then - PKGS="$(GIT_WORK_TREE="$x" peek.sh) $PKGS" - fi - done -else - PKGS="$(find.ts $@)" -fi - -if test -n "$PKGX_DOCKER"; then - exec "$d"/bin/pkg-docker build "$PKGS" -fi - -if ! available.ts $PKGS; then - echo "warning: not supported on current platform" >&2 - exit 0 -fi - -if test -z "$GITHUB_TOKEN"; then - GITHUB_TOKEN=$(pkgx gh auth token) - if test -z "$GITHUB_TOKEN"; then - echo "error: builds need a GITHUB_TOKEN. Either export that or run \`gh auth login\`" >&2 - exit 1 - fi - export GITHUB_TOKEN -fi - -# shellcheck disable=SC2086 -PKGS="$(sort.ts $PKGS --delimiter ' ')" -# ^^ ensures that pkgs build topologically - -gum() { - pkgx gum format -- "$PKGS" -} - -for PKG in $PKGS; do - gum "# building $PKG" - - PKG="$(resolve.ts "$PKG")" - ZIP="$(query.ts "$PKG" --src)" - PREFIX="$(query.ts "$PKG" --prefix)" - SRCDIR="$(query.ts "$PKG" --srcdir)" - BLDDIR="$(query.ts "$PKG" --build-dir)" - - if test -n "$ZIP"; then - gum "# fetching $ZIP" - fetch.ts "$PKG" -o "$ZIP" - fi - - if test -f "$ZIP"; then - gum "# extracting $SRCDIR" - extract.ts "$ZIP" --pkg "$PKG" --output-dir "$SRCDIR" - else - mkdir -p "$SRCDIR" - fi - - gum "# staging for build" - - DEPS=$(deps.ts "$PKG" --build) # eg. nodejs.org^4 - - # shellcheck disable=SC2086 - DEPS=$(install.ts $DEPS) # eg. ~/.pkgx/nodejs.org/v4.3.2 - - mkdir -p "$SRCDIR" - BUILD_SCRIPT="$(stage.ts "$PKG" --srcdir "$SRCDIR" --prefix "$PREFIX" --deps "$DEPS" --build-dir "$BLDDIR")" - - if command -v bash >/dev/null; then - BASH=bash - else - BASH="pkgx bash" - fi - - if test -n "$GITHUB_ACTIONS"; then - echo "build-dir=$BLDDIR" >> $GITHUB_OUTPUT - fi - - gum "# running build" - - env -i GITHUB_TOKEN="$GITHUB_TOKEN" "$BASH" --noprofile --norc -e "$BUILD_SCRIPT" - - if ! test -d "$PREFIX"; then - echo "error: nothing installed to \`$PREFIX\`" >&2 - echo "maybe: did you forget to \`make install\` or equivalent?" >&2 - exit 1 - fi - - gum "# fixing up built products" - - DEPS="$(deps.ts "$PKG")" # needs to be runtime deps AND include self - # shellcheck disable=SC2086 - DEPS=$(install.ts $DEPS) - PATH="$d/share/brewkit:$PATH" fixup.ts "$PKG" "$PREFIX" --deps "$DEPS" - - find "$PREFIX" -type f -name \*.la -delete - - gum "# symlinking installations" - - link.ts "$PREFIX" "$PKG" - - if test -n "$GITHUB_ACTIONS"; then - GH_PKGS="$GH_PKGS $PKG" - GH_RELATIVE_PATHS="$GH_RELATIVE_PATHS ${PREFIX#"$PKGX_DIR"/}" - - if test -f "$ZIP"; then - GH_SRCS="$GH_SRCS ${ZIP#"$PKGX_DIR"/}" - GH_SRCS_RELATIVE_PATHS="$GH_SRCS_RELATIVE_PATHS ${ZIP#"$PKGX_DIR"/}" - else - GH_SRCS="$GH_SRCS ~" - fi - fi -done - -if test -n "$GITHUB_ACTIONS"; then - { - echo "pkgs=$GH_PKGS" - echo "relative-paths=$GH_RELATIVE_PATHS" - echo "srcs=$GH_SRCS" - echo "srcs-relative-paths=$GH_SRCS_RELATIVE_PATHS" - } >> "$GITHUB_OUTPUT" -fi diff --git a/bin/pkg-clean b/bin/pkg-clean deleted file mode 100755 index 4931132c..00000000 --- a/bin/pkg-clean +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -set -eo pipefail - -d="$(cd "$(dirname "$0")"/.. && pwd)" -PATH="$d/libexec:$PATH" - -if test -z "$1"; then - if test -z "$PKGX_PANTRY_PATH"; then - echo "error: PKGX_PANTRY_PATH is not set" >&2 - exit 1 - fi - for x in $(echo "$PKGX_PANTRY_PATH" | tr ':' '\n'); do - if test -d "$x"/.git; then - PKGS="$(GIT_WORK_TREE="$x" peek.sh) $PKGS" - fi - done -else - PKGS="$(find.ts $@)" -fi - -if test -z "$GITHUB_TOKEN"; then - GITHUB_TOKEN=$(pkgx gh auth token) - export GITHUB_TOKEN -fi - -for PKG in $PKGS; do - PKG="$(resolve.ts "$PKG")" - BLDDIR="$(query.ts "$PKG" --build-dir)" - DSTDIR="$(query.ts "$PKG" --prefix)" - TSTDIR="$(query.ts "$PKG" --testdir)" - - rm -rf $BLDDIR $DSTDIR $TSTDIR -done diff --git a/bin/pkg-docker b/bin/pkg-docker deleted file mode 100755 index 4f5dae1f..00000000 --- a/bin/pkg-docker +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -set -e - -d="$(cd "$(dirname "$0")/.." && pwd)" - -arg0="$1" - -PKGX_DIR="${PKGX_DIR:-$HOME/.pkgx}" - -if test -z "$GITHUB_TOKEN"; then - GITHUB_TOKEN=$(pkgx gh auth token) -fi - -if test "$arg0" = docker; then - echo "brewkit: error: invalid usage" >&2 - exit 1 -elif shift; then - # ensure this brewkit is used - CMD="$d/bin/pkg-$arg0" - ADDARGS="--volume ""$d"":""$d" -else - # we were invoked as `pkg -L` by itself - ADDARGS="--interactive --workdir /root" -fi - -STDARGS=" - --name brewkit.pkgx.dev - --rm - --tty - --env GITHUB_TOKEN=$GITHUB_TOKEN - --env CLICOLOR_FORCE=1 - " - -if test "$PKGX_PANTRY_PATH" = "$SRCROOT"; then - # we’re running in a pantry checkout - - mkdir -p "$PKGX_PANTRY_PATH/prefixes/linux" - - pkgx gum format "# docker pull" - - docker image pull pkgxdev/pkgx - - pkgx gum format "# docker run" - - docker run \ - $STDARGS \ - $ADDARGS \ - --volume "$PKGX_PANTRY_PATH":/work \ - --env SRCROOT=/work \ - --volume /root/pantry/prefixes \ - --volume "$PKGX_PANTRY_PATH"/prefixes/linux:/root/.pkgx \ - --volume "$PKGX_PANTRY_PATH"/projects:/root/.local/share/pkgx/pantry/projects \ - --env DENO_DIR=/root/.pkgx/.cache/deno \ - pkgxdev/pkgx \ - $CMD "$@" -else - if test -n "$PKGX_PANTRY_PATH"; then - ADDARGS="--volume ""$PKGX_PANTRY_PATH"":/root/.local/share/pkgx/pantry ""$ADDARGS" - fi - - docker image pull pkgxdev/pkgx - - docker run \ - $STDARGS \ - $ADDARGS \ - pkgxdev/pkgx \ - $CMD "$@" -fi - -# ^^ `--volume $d:$d` for debugging: so errors in brewkit are the same as paths on the host diff --git a/bin/pkg-fetch b/bin/pkg-fetch deleted file mode 100755 index 4b7d7047..00000000 --- a/bin/pkg-fetch +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -set -eo pipefail - -d="$(cd "$(dirname "$0")"/.. && pwd)" -PATH="$d/libexec:$PATH" - -while getopts 'I' OPTION; do - case "$OPTION" in - I) - OPT_HEADERS=1;; - ?) - echo "usage: $(basename $0) [-I] " - exit 1 - ;; - esac -done - -shift "$(($OPTIND -1))" - -if test -z "$1"; then - if test -z "$PKGX_PANTRY_PATH"; then - echo "error: PKGX_PANTRY_PATH is not set" >&2 - exit 1 - fi - for x in $(echo "$PKGX_PANTRY_PATH" | tr ':' '\n'); do - if test -d "$x"/.git; then - PKGS="$(GIT_WORK_TREE="$x" peek.sh) $PKGS" - fi - done -else - PKGS="$(find.ts $1)" -fi - -if test -z "$GITHUB_TOKEN"; then - GITHUB_TOKEN=$(pkgx gh auth token) - export GITHUB_TOKEN -fi - -for PKG in $PKGS; do - PKG="$(resolve.ts $PKG)" - - if test "$OPT_HEADERS"; then - URL="$(query.ts $PKG --url)" - curl -I "$URL" - else - ZIP="$(query.ts $PKG --src)" - SRCDIR="$(query.ts $PKG --srcdir)" - - if test -n "$ZIP"; then - fetch.ts $PKG -o "$ZIP" - fi - - if test -f "$ZIP"; then - extract.ts "$ZIP" --pkg $PKG --output-dir "$SRCDIR" - fi - fi - - echo "$SRCDIR" -done diff --git a/bin/pkg-fixup b/bin/pkg-fixup deleted file mode 100755 index 8126eacc..00000000 --- a/bin/pkg-fixup +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -set -eo pipefail - -d="$(cd "$(dirname "$0")"/.. && pwd)" -PATH="$d/libexec:$PATH" - -if test -z "$1"; then - if test -z "$PKGX_PANTRY_PATH"; then - echo "error: PKGX_PANTRY_PATH is not set" >&2 - exit 1 - fi - for x in $(echo "$PKGX_PANTRY_PATH" | tr ':' '\n'); do - if test -d "$x"/.git; then - PKGS="$(GIT_WORK_TREE="$x" peek.sh) $PKGS" - fi - done -else - PKGS="$(find.ts $1)" -fi - -for PKG in $PKGS; do - PKG="$(resolve.ts "$PKG" --cellar)" - PREFIX="$(query.ts "$PKG" --prefix)" - - DEPS=$(deps.ts "$PKG") - # shellcheck disable=SC2086 - DEPS=$(install.ts $DEPS) - - PATH="$d/share/brewkit:$PATH" fixup.ts "$PKG" "$PREFIX" --deps "$DEPS" -done diff --git a/bin/pkg-inventory b/bin/pkg-inventory deleted file mode 100755 index e6e4caa3..00000000 --- a/bin/pkg-inventory +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -d="$(cd "$(dirname "$0")/.." && pwd)" -exec "$d"/libexec/inventory.ts "$@" diff --git a/bin/pkg-query b/bin/pkg-query deleted file mode 100755 index 84a14032..00000000 --- a/bin/pkg-query +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -d="$(cd "$(dirname "$0")/.." && pwd)" -exec "$d"/libexec/query.ts "$@" diff --git a/bin/pkg-search b/bin/pkg-search deleted file mode 100755 index 8ac5f55c..00000000 --- a/bin/pkg-search +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -d="$(cd "$(dirname "$0")/.." && pwd)" -exec "$d"/libexec/search.ts "$@" diff --git a/bin/pkg-status b/bin/pkg-status deleted file mode 120000 index 6b85d3a0..00000000 --- a/bin/pkg-status +++ /dev/null @@ -1 +0,0 @@ -../libexec/peek.sh \ No newline at end of file diff --git a/bin/pkg-test b/bin/pkg-test deleted file mode 100755 index 35c311b2..00000000 --- a/bin/pkg-test +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -set -eo pipefail - -d="$(cd "$(dirname "$0")"/.. && pwd)" -PATH="$d/libexec:$PATH" - -if test -z "$1"; then - if test -z "$PKGX_PANTRY_PATH"; then - echo "error: PKGX_PANTRY_PATH is not set" >&2 - exit 1 - fi - for x in $(echo "$PKGX_PANTRY_PATH" | tr ':' '\n'); do - if test -d "$x"/.git; then - PKGS="$(GIT_WORK_TREE="$x" peek.sh) $PKGS" - fi - done -else - PKGS="$(find.ts "$@")" -fi - -if test -n "$PKGX_DOCKER"; then - exec "$d"/bin/pkg-docker test "$PKGS" -fi - -if ! available.ts "$@"; then - echo "warning: not supported on current platform" >&2 - exit 0 -fi - -gum() { - pkgx gum format -- "$@" -} - -for PKG in $PKGS; do - gum "# testing $PKG" - gum "## preparing testbed" - - PKG="$(resolve.ts "$PKG" --cellar)" - DEPS="$(deps.ts "$PKG" --test)" - # shellcheck disable=2086 - DEPS="$(install.ts $DEPS)" - DSTDIR="$(query.ts "$PKG" --testdir)" - - if test -d "$DSTDIR"; then - for x in "$DSTDIR"/*; do - if test $x != "$DSTDIR"/.cargo; then - rm -rf "$x" - fi - done - fi - mkdir -p "$DSTDIR" - - stage-test.ts "$PKG" --deps "$DEPS" --dstdir "$DSTDIR" - - gum "## running test" - - "$DSTDIR"/dev.pkgx.test.sh - - gum "## running audit" - - audit.ts "$PKG" -done diff --git a/bin/run b/bin/run deleted file mode 100755 index d1bb2312..00000000 --- a/bin/run +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env -S pkgx +deno bash +exo pipefail - -eval "$(pkgx --shellcode)" - -d="$(cd "$(dirname "$0")"/.. && pwd)" - -get_parameters() { - deno run --config="$d/deno.jsonc" -A - < x?.['entrypoint']) - if (!entrypoint) Deno.exit(1) - console.log(pkg.project, entrypoint) -EoTS -} - -get_keg() { - deno run --config="$d/deno.jsonc" -A - <> $GITHUB_OUTPUT + id: bottle + shell: bash diff --git a/build/action.yml b/build/action.yml new file mode 100644 index 00000000..e582e61e --- /dev/null +++ b/build/action.yml @@ -0,0 +1,71 @@ +name: pkgx/brewkit/build + +inputs: + pkg: + description: > + eg. pkgx.sh@1.1 + required: true + token: + default: ${{github.token}} + required: true + +outputs: + pkgspec: + description: > + the precise pkg and version we built, eg. pkgx.sh@1.1.1 + value: ${{ steps.build.outputs.pkgspec }} + prefix: + description: > + the path to the built and installed pkg + value: ${{ steps.build.outputs.prefix }} + noop: + description: > + the build is a noop if it cannot be built and run on this platform + value: ${{ steps.build.outputs.noop }} + project: + value: ${{ steps.build.outputs.project }} + version: + value: ${{ steps.build.outputs.version }} + platform: + value: ${{ steps.build.outputs.platform }} + arch: + value: ${{ steps.build.outputs.arch }} + +runs: + using: composite + steps: + - name: prep + run: | + if ! pkgx --sync; then + echo "::error::you must use: pkgxdev/setup before using this action" + exit 1 + fi + shell: bash + + - name: fetch deno deps + shell: bash + run: | + echo "::group::fetch deno deps" + tmpdir=$(mktemp -d) + ln -s ${GITHUB_ACTION_PATH}/../bin/cmd/build $tmpdir/build.ts + pkgx deno cache $tmpdir/build.ts --config=${GITHUB_ACTION_PATH}/../deno.jsonc + echo "::endgroup::" + + - name: build + run: | + if [ -d projects -a -z "$PKGX_PANTRY_PATH" ]; then + export PKGX_PANTRY_PATH="$PWD/projects" + fi + + set +e + ${GITHUB_ACTION_PATH}/../bin/cmd/build ${{ inputs.pkg }} + status=$? + if [ $status -eq 2 ]; then + echo noop=true >> $GITHUB_OUTPUT + else + exit $status + fi + id: build + env: + GITHUB_TOKEN: ${{inputs.token}} + shell: bash diff --git a/deno.jsonc b/deno.jsonc index 10e24e22..49d29f2d 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -9,11 +9,12 @@ }, "pkgx": "deno^1.36.1", "imports": { + "brewkit/": "./lib/", "is-what": "https://deno.land/x/is_what@v4.1.15/src/index.ts", "outdent": "https://deno.land/x/outdent@v0.8.0/mod.ts", - "cliffy/": "https://deno.land/x/cliffy@v0.25.7/", + "cliffy/": "https://deno.land/x/cliffy@v1.0.0-rc.3/", "s3": "https://deno.land/x/s3@0.5.0/mod.ts", - "deno/": "https://deno.land/std@0.196.0/", + "deno/": "https://deno.land/std@0.208.0/", "libpkgx": "https://deno.land/x/libpkgx@v0.14.1/mod.ts", "libpkgx/": "https://deno.land/x/libpkgx@v0.14.1/src/", "pkgx": "./lib/pkgx.ts" diff --git a/download-build-artifact/action.yml b/download-build-artifact/action.yml new file mode 100644 index 00000000..8f8c0a7c --- /dev/null +++ b/download-build-artifact/action.yml @@ -0,0 +1,77 @@ +name: pkgx/brewkit/download-build-artifacts +description: | + Downloads a single previously uploaded build artifact to a + temporary location and sets `outputs.filename`. + + Artifacts are *uncompressed* tarballs with format: + + platform+arch.tar # eg. darwin+aarch64.tar + + By default will download the tarball for the active platform. This can + be overridden with `inputs.platform`. + + By default extracts the artifact to `${PKGX_DIR:-$HOME/.pkgx}`. + This can be disabled. + + > ![NOTE] + > This means you can only build one package per workflow. This is a + > limitation designed to simplify our implementation and your usage. We are + > open to discussing removing this limitation. + +inputs: + platform: + required: false + description: eg. `darwin+aarch64` or `linux+x86-64` + extract: + required: true + default: true + +outputs: + filename: + description: > + if you specified a platform this will be set to the downloaded artifact. + if you didn’t specify a platform it will be unset as there will likely + be multiple artifacts for each platform you built. + value: ${{steps.download.outputs.download-path}}/${{steps.github-suck.outputs.filename}} + +runs: + using: composite + steps: + - run: | + if ! pkgx --sync; then + echo "::error::you must use: pkgxdev/setup before using this action" + exit 1 + fi + shell: bash + + - name: fetch deno deps + shell: bash + run: | + echo "::group::fetch deno deps" + pkgx deno cache ${GITHUB_ACTION_PATH}/../lib/actions/*.ts --config=${GITHUB_ACTION_PATH}/../deno.jsonc + echo "::endgroup::" + + - name: prep + run: | + echo "tmpdir=$(mktemp -d)" >> $GITHUB_OUTPUT + key=$(${GITHUB_ACTION_PATH}/../lib/actions/platform-key.ts) + echo "platform=$key" >> $GITHUB_OUTPUT + id: prep + shell: bash + + - uses: actions/download-artifact@v3 + id: download + with: + name: ${{ inputs.platform || steps.prep.outputs.platform }} + path: ${{steps.prep.outputs.tmpdir}} + + - name: github-make-shit-things + id: github-suck + shell: bash + run: | + echo filename=$(ls ${{steps.download.outputs.download-path}}) >> $GITHUB_OUTPUT + + - name: untar + if: ${{ inputs.extract == 'true' }} + run: tar xf ${{steps.download.outputs.download-path}}/${{steps.github-suck.outputs.filename}} -C "${PKGX_DIR:-$HOME/.pkgx}" + shell: bash diff --git a/lib/actions/platform-key.ts b/lib/actions/platform-key.ts new file mode 100755 index 00000000..ef04380d --- /dev/null +++ b/lib/actions/platform-key.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env -S pkgx deno run --allow-env + +import { utils } from 'pkgx' +const { arch, platform } = utils.host() +console.log(`${platform}+${arch}`) diff --git a/lib/actions/stage.ts b/lib/actions/stage.ts new file mode 100755 index 00000000..a0082925 --- /dev/null +++ b/lib/actions/stage.ts @@ -0,0 +1,12 @@ +#!/usr/bin/env -S pkgx deno run --allow-read --allow-env --allow-write + +import { hooks, utils, Path } from "pkgx" + +const pkg = utils.pkg.parse(Deno.args[1]?.trim() || Deno.env.get('BREWKIT_PKGSPEC')!) +const { path } = await hooks.useCellar().resolve(pkg) + +const stage = new Path(Deno.args[0]) + +path.mv({ into: stage.join(pkg.project).mkdir('p') }) + +console.log(stage) diff --git a/share/brewkit/fix-elf.ts b/lib/bin/fix-elf.ts similarity index 87% rename from share/brewkit/fix-elf.ts rename to lib/bin/fix-elf.ts index e7376868..3e47c078 100755 --- a/share/brewkit/fix-elf.ts +++ b/lib/bin/fix-elf.ts @@ -3,18 +3,16 @@ // FIXME ^^ patchelf 0.18.0 has a regression that breaks libraries // https://github.com/NixOS/patchelf/issues/492#issuecomment-1561912775 -import { utils, PackageRequirement, Installation, Package, hooks, Path } from "pkgx" +import { utils, Installation, hooks, Path } from "pkgx" import { backticks } from "../../lib/utils.ts" -const { host, pkg: pkgutils } = utils const { useCellar } = hooks +const { host } = utils if (import.meta.main) { const cellar = useCellar() - const [installation, ...pkgs] = Deno.args - await fix_rpaths( - await cellar.resolve(new Path(installation)), - pkgs.map(pkgutils.parse) - ) + const [installation_path, ...pkgs] = Deno.args + const installed = await cellar.resolve(new Path(installation_path)) + await fix_rpaths(installed, pkgs) } @@ -22,7 +20,7 @@ if (import.meta.main) { //NOTE solution is to have the rpath reference major version (or more specific if poss) /// fix rpaths or install names for executables and dynamic libraries -export default async function fix_rpaths(installation: Installation, pkgs: (Package | PackageRequirement)[]) { +export default async function fix_rpaths(installation: Installation, pkgs: string[]) { console.info("doing SLOW rpath fixes…") for await (const [exename] of exefiles(installation.path)) { await set_rpaths(exename, pkgs, installation) @@ -34,12 +32,9 @@ export default async function fix_rpaths(installation: Installation, pkgs: (Pack //NOTE we should have a `safety-inspector` step before bottling to check for this sort of thing // and then have virtual env manager be more specific via (DY)?LD_LIBRARY_PATH //FIXME somewhat inefficient for eg. git since git is mostly hardlinks to the same file -async function set_rpaths(exename: Path, pkgs: (Package | PackageRequirement)[], installation: Installation) { +async function set_rpaths(exename: Path, our_rpaths: string[], installation: Installation) { if (host().platform != 'linux') throw new Error() - const cellar = useCellar() - const our_rpaths = await Promise.all(pkgs.map(pkg => prefix(pkg))) - const cmd = await (async () => { //FIXME we need this for perl // however really we should just have an escape hatch *just* for stuff that sets its own rpaths @@ -79,10 +74,6 @@ async function set_rpaths(exename: Path, pkgs: (Package | PackageRequirement)[], // and we don't yet have a good way to detect and skip such files } } - - async function prefix(pkg: Package | PackageRequirement) { - return (await cellar.resolve(pkg)).path.join("lib").string - } } //FIXME pretty slow since we execute `file` for every file diff --git a/share/brewkit/fix-machos.rb b/lib/bin/fix-machos.rb similarity index 99% rename from share/brewkit/fix-machos.rb rename to lib/bin/fix-machos.rb index 423bb108..aa9ea65b 100755 --- a/share/brewkit/fix-machos.rb +++ b/lib/bin/fix-machos.rb @@ -16,7 +16,7 @@ require 'macho' require 'find' -$PKGX_DIR = ENV['PKGX_DIR'] || ENV["HOME"] + "/.pkgx" +$PKGX_DIR = ENV['PKGX_DIR'] || File.join(ENV["HOME"], ".pkgx") $pkg_prefix = ARGV.shift abort "arg1 should be pkg-prefix" if $pkg_prefix.empty? diff --git a/lib/config.ts b/lib/config.ts new file mode 100644 index 00000000..e28d601e --- /dev/null +++ b/lib/config.ts @@ -0,0 +1,133 @@ +import useConfig from "libpkgx/hooks/useConfig.ts" +import { Path, Package, PackageRequirement, utils, hooks, plumbing, Installation } from "pkgx" +const { flatmap, host } = utils +const { usePantry } = hooks +const { hydrate } = plumbing + +export interface Config { + pkg: Package + pkgspec: PackageRequirement + path: ConfigPath + deps: { + dry: { + runtime: PackageRequirement[] + build: PackageRequirement[] + test: PackageRequirement[] + }, + wet: PackageRequirement[], + gas: Installation[] + } +} + +export interface ConfigPath { + yaml: Path /// the package.yml + pantry: Path /// the pantry we found this yaml in + src: Path /// full path to extracted sources + tarball_dir: Path /// full path to the directory to store tarballs + build: Path /// {{brewroot}}/builds/foo-1.2.3 + install: Path /// ${PKGX_DIR:-$HOME/.pkgx}}/foo-1.2.3 + build_install: Path /// ${PKGX_DIR:-$HOME/.pkgx}}/foo-1.2.3+brewing + home: Path /// we make a new HOME for robust builds and your own system’s sanity + test: Path /// where we stage and run tests + cache: Path // persistent cache between builds, use for big things like AI models +} + +export default async function config(arg?: string): Promise { + if (!arg) { + arg ??= Deno.env.get("BREWKIT_PKGSPEC") ?? (await get_pantry_status())?.[0] + if (!arg) throw new Error(`usage: ${Deno.execPath()} `) + } + + const { constraint, project } = utils.pkg.parse(arg) + const [found, ...rest] = await usePantry().find(project) + if (rest.length) throw new Error("ambiguous pkg spec") + const pkg = await usePantry().resolve({project: found.project, constraint}) + + let pantry = found.path + for (let x = 0, N = found.project.split('/').length + 1; x < N; x++) { + pantry = pantry.parent() + } + + if (Deno.env.get("PKGX_PANTRY_PATH")) { + const checkout = new Path(Deno.env.get("PKGX_PANTRY_PATH")!) + const slug = pkg.project.replace(/\//g, "∕") // this is a unicode separator + const pkgspec = `${slug}-${pkg.version}` + return await construct_config({ + home: checkout.join('homes', pkgspec), + src: checkout.join('srcs', pkgspec), + build: checkout.join('builds', pkgspec), + test: checkout.join('testbeds', pkgspec), + tarball_dir: checkout.join('srcs'), + }) + } else { + const {platform, arch} = host() + const datahome = platform_datahome().join("brewkit") + const bkroot = datahome.join(`${platform}+${arch}`, pkg.project, `v${pkg.version}`) + return await construct_config({ + home: bkroot, + src: bkroot.join('src'), + build: bkroot.join('build'), + test: bkroot.join('testbed'), + tarball_dir: datahome + }) + } + + async function construct_config({home, src, build, test, tarball_dir}: {home: Path, src: Path, build: Path, test: Path, tarball_dir: Path, suffix?: string}): Promise { + const dry = await usePantry().getDeps(pkg) + + const { pkgs: wet } = await hydrate(dry.runtime.concat(dry.build)) + const gas = await plumbing.resolve(wet) + // predetermining these just for the build script generation step + const installs = gas.pkgs.map(pkg => ({ + pkg, + path: useConfig().prefix.join(pkg.project, `v${pkg.version}`) + })) + + const cache = platform_cache() + const install = useConfig().prefix.join(pkg.project, `v${pkg.version}`) + const build_install = new Path(`${install.string}+brewing`) + + return { + pkg, + pkgspec: {project: found.project, constraint}, + deps: { + dry, + wet, + gas: installs + }, + path: { + cache, + pantry, + yaml: found.path, + src, + build, + install, + home, + test, + build_install, + tarball_dir + } + } + } +} + +async function get_pantry_status() { + const bkroot = new Path(new URL(import.meta.url).pathname).parent().parent() + const proc = new Deno.Command("bash", {args: [bkroot.join('bin/cmd/status').string], stdout: 'piped'}).spawn() + const [out, { success }] = await Promise.all([proc.output(), proc.status]) + if (success) { + return new TextDecoder().decode(out.stdout).split(/\s+/).filter(x => x) + } +} + +function platform_cache() { + return flatmap(Deno.env.get('XDG_CACHE_HOME'), Path.abs) ?? (platform => + platform == 'darwin' ? Path.home().join('Library/Caches') : Path.home().join(".cache") + )(host().platform) +} + +function platform_datahome() { + return flatmap(Deno.env.get('XDG_DATA_HOME'), Path.abs) ?? (platform => + platform == 'darwin' ? Path.home().join('Library/Application Support') : Path.home().join(".local/share") + )(host().platform) +} \ No newline at end of file diff --git a/lib/useCache.test.ts b/lib/hooks/useCache.test.ts similarity index 100% rename from lib/useCache.test.ts rename to lib/hooks/useCache.test.ts diff --git a/lib/useCache.ts b/lib/hooks/useCache.ts similarity index 100% rename from lib/useCache.ts rename to lib/hooks/useCache.ts diff --git a/lib/useGitHubAPI.ts b/lib/hooks/useGitHubAPI.ts similarity index 88% rename from lib/useGitHubAPI.ts rename to lib/hooks/useGitHubAPI.ts index 919cf1f4..83ea1ea8 100644 --- a/lib/useGitHubAPI.ts +++ b/lib/hooks/useGitHubAPI.ts @@ -21,15 +21,22 @@ export default function useGitHubAPI() { return { getVersions } } +async function gh() { + const proc = new Deno.Command("gh", { + args: ["auth", "token"], + stdout: "piped", + }) + const { success } = await proc.spawn().status + if (!success) throw new Error("Either set GITHUB_TOKEN or run `gh auth login`") + return new TextDecoder().decode((await proc.output()).stdout).trim() +} async function GET2(url: URL | string, headers?: Headers): Promise<[T, Response]> { if (isString(url)) url = new URL(url) if (url.host == "api.github.com") { - const token = Deno.env.get("GITHUB_TOKEN") - if (token) { - headers ??= new Headers() - headers.append("Authorization", `bearer ${token}`) - } + const token = Deno.env.get("GITHUB_TOKEN") ?? await gh() + headers ??= new Headers() + headers.append("Authorization", `bearer ${token}`) } const rsp = await fetch(url, { headers }) if (!rsp.ok) throw new Error(`http: ${url}`) @@ -79,7 +86,7 @@ async function *getVersionsLong({ user, repo, type }: GetVersionsOptions): Async do { const headers: HeadersInit = {} - const token = Deno.env.get("GITHUB_TOKEN") + const token = Deno.env.get("GITHUB_TOKEN") ?? await gh() if (token) headers['Authorization'] = `bearer ${token}` const query = undent` diff --git a/lib/useGitLabAPI.ts b/lib/hooks/useGitLabAPI.ts similarity index 100% rename from lib/useGitLabAPI.ts rename to lib/hooks/useGitLabAPI.ts diff --git a/lib/hooks/usePantry.getScript.ts b/lib/hooks/usePantry.getScript.ts new file mode 100644 index 00000000..85f5e25f --- /dev/null +++ b/lib/hooks/usePantry.getScript.ts @@ -0,0 +1,233 @@ +import useConfig from "libpkgx/hooks/useConfig.ts" +import { SupportedPlatforms, SupportedArchitectures } from "libpkgx/utils/host.ts" +import { isArray, isString, isPlainObject, PlainObject, isPrimitive, isBoolean, isNumber } from "is-what" +import { Package, Installation, hooks, utils, semver, Path, PantryParseError } from "libpkgx" +import undent from "outdent" + +const { validate, host } = utils +const { useMoustaches } = hooks + +export const getScript = async (pkg: Package, key: 'build' | 'test', deps: Installation[], install_prefix: Path) => { + const yml = await hooks.usePantry().project(pkg).yaml() + const node = yml[key] + + const mm = useMoustaches() + const tokens = mm.tokenize.all(pkg, deps) + tokens.push({ + from: "pkgx.dir", to: useConfig().prefix.string + }) + + for (const [index, token] of tokens.entries()) { + if (token.from == "prefix") { + tokens[index] = { from: "prefix", to: install_prefix.string } + } + } + + const script = (input: unknown) => { + if (isArray(input)) input = input.map(obj => { + if (isPlainObject(obj)) { + + const condition = obj["if"] + if (condition) { + if (SupportedPlatforms.includes(condition) && host().platform != condition) return '' + if (SupportedArchitectures.includes(condition) && host().arch != condition) return '' + if (condition.includes("/")) { + const [platform, arch] = condition.split("/") + if (SupportedPlatforms.includes(platform) && + SupportedArchitectures.includes(arch) && + (host().platform != platform || + host().arch != arch)) return '' + } + + const range = semver.Range.parse(condition) + if (range && !range.satisfies(pkg.version)) return '' + } + + let run = obj['run'] + if (isArray(run)) { + run = run.join("\n") + } else if (!isString(run)) { + throw new Error('every node in a script YAML array must contain a `run` key') + } + + let cd = obj['working-directory'] + if (cd) { + cd = mm.apply(validate.str(cd), tokens) + run = undent` + OLDWD="$PWD" + mkdir -p "${cd}" + cd "${cd}" + ${run.trim()} + cd "$OLDWD" + unset OLDWD + ` + } + + let fixture_key = key == 'build' ? 'prop' : 'fixture' + let fixture = obj[fixture_key] + if (fixture) { + fixture_key = fixture_key.toUpperCase() + fixture = mm.apply(validate.str(fixture), tokens) + run = undent` + OLD_${fixture_key}=$${fixture_key} + ${fixture_key}=$(mktemp) + + cat < $${fixture_key} + ${fixture} + DEV_PKGX_EOF + + ${run} + + rm -f $${fixture_key} + + if test -n "$${fixture_key}"; then + ${fixture_key}=$OLD_${fixture_key} + else + unset ${fixture_key} + fi + ` + } + + return run.trim() + } else { + return `${obj}`.trim() + } + }).join("\n\n") + return mm.apply(validate.str(input), tokens) + } + + if (isPlainObject(node)) { + let raw = script(node.script) + + const pkg_tokens = (tokens => { + const rv = [] + for (const token of tokens) { + if (token.from == "prefix") { + rv.push({from: "prefix", to: install_prefix.string}) + } else { + rv.push(token) + } + } + return rv + })(mm.tokenize.pkg(pkg)) + + let wd = node["working-directory"] + if (wd) { + wd = mm.apply(wd, [ + ...mm.tokenize.version(pkg.version), + ...mm.tokenize.host(), + ...pkg_tokens + ]) + raw = undent` + mkdir -p ${wd} + cd ${wd} + + ${raw} + ` + } + + const env = node.env + if (isPlainObject(env)) { + raw = `${expand_env(env, pkg, tokens)}\n\n${raw}` + } + return raw + } else { + return script(node) + } +} + +function expand_env(env: PlainObject, pkg: Package, tokens: { from: string, to: string }[]): string { + return Object.entries(expand_env_obj(env, pkg, tokens)).map(([key,value]) => { + // weird POSIX string escaping/concat stuff + // eg. export FOO="bar ""$baz"" bun" + value = `"${value.trim().replace(/"/g, '""')}"` + while (value.startsWith('""')) value = value.slice(1) //FIXME lol better pls + while (value.endsWith('""')) value = value.slice(0,-1) //FIXME lol better pls + + return `export ${key}=${value}` + }).join("\n") +} + +//FIXME these are copy pasta from usePantry because we build to a different prefix so need control over the moustaches +export function expand_env_obj(env_: PlainObject, pkg: Package, tokens: { from: string, to: string }[]): Record { + const env = {...env_} + + platform_reduce(env) + + const rv: Record = {} + + for (let [key, value] of Object.entries(env)) { + if (isArray(value)) { + value = value.map(x => transform(x)).join(" ") + } else { + value = transform(value) + } + + if (Deno.build.os == 'windows') { + // we standardize on UNIX directory separators + // NOTE hopefully this won’t break anything :/ + value = value.replaceAll('/', '\\') + } + + rv[key] = value + } + + return rv + + // deno-lint-ignore no-explicit-any + function transform(value: any): string { + if (!isPrimitive(value)) throw new PantryParseError(pkg.project, undefined, JSON.stringify(value)) + + if (isBoolean(value)) { + return value ? "1" : "0" + } else if (value === undefined || value === null) { + return "0" + } else if (isString(value)) { + const mm = useMoustaches() + const home = Path.home().string + const obj = [ + { from: 'home', to: home }, // remove, stick with just ~ + ...tokens + ] + return mm.apply(value, obj) + } else if (isNumber(value)) { + return value.toString() + } + + const e = new Error("unexpected error") + e.cause = value + throw e + } +} + +function platform_reduce(env: PlainObject) { + const sys = host() + for (const [key, value] of Object.entries(env)) { + const [os, arch] = (() => { + let match = key.match(/^(darwin|linux)\/(aarch64|x86-64)$/) + if (match) return [match[1], match[2]] + if ((match = key.match(/^(darwin|linux)$/))) return [match[1]] + if ((match = key.match(/^(aarch64|x86-64)$/))) return [,match[1]] + return [] + })() + + if (!os && !arch) continue + delete env[key] + if (os && os != sys.platform) continue + if (arch && arch != sys.arch) continue + + const dict = validate.obj(value) + for (const [key, value] of Object.entries(dict)) { + // if user specifies an array then we assume we are supplementing + // otherwise we are replacing. If this is too magical let us know + if (isArray(value)) { + if (!env[key]) env[key] = [] + else if (!isArray(env[key])) env[key] = [env[key]] + //TODO if all-platforms version comes after the specific then order accordingly + env[key].push(...value) + } else { + env[key] = value + } + } + } +} diff --git a/lib/usePantry.getVersions.test.ts b/lib/hooks/usePantry.getVersions.test.ts similarity index 100% rename from lib/usePantry.getVersions.test.ts rename to lib/hooks/usePantry.getVersions.test.ts diff --git a/lib/usePantry.getVersions.ts b/lib/hooks/usePantry.getVersions.ts similarity index 100% rename from lib/usePantry.getVersions.ts rename to lib/hooks/usePantry.getVersions.ts diff --git a/lib/usePantry.ts b/lib/hooks/usePantry.ts similarity index 100% rename from lib/usePantry.ts rename to lib/hooks/usePantry.ts diff --git a/lib/useSourceUnarchiver.ts b/lib/hooks/useSourceUnarchiver.ts similarity index 93% rename from lib/useSourceUnarchiver.ts rename to lib/hooks/useSourceUnarchiver.ts index 90dc2fb2..cea99d9b 100644 --- a/lib/useSourceUnarchiver.ts +++ b/lib/hooks/useSourceUnarchiver.ts @@ -1,4 +1,4 @@ -import { Unarchiver, TarballUnarchiver, ZipUnarchiver } from "./Unarchiver.ts" +import { Unarchiver, TarballUnarchiver, ZipUnarchiver } from "../utils/Unarchiver.ts" import { Path } from "pkgx" //FIXME assuming strip 1 on components is going to trip people up diff --git a/lib/pkgx.ts b/lib/pkgx.ts index 9646f4f9..6757bf72 100644 --- a/lib/pkgx.ts +++ b/lib/pkgx.ts @@ -1,8 +1,8 @@ export * from "libpkgx" -import usePantry from "./usePantry.ts" -import useCache from "./useCache.ts" -import useSourceUnarchiver from "./useSourceUnarchiver.ts" +import usePantry from "./hooks/usePantry.ts" +import useCache from "./hooks/useCache.ts" +import useSourceUnarchiver from "./hooks/useSourceUnarchiver.ts" import { hooks as vanilla_hooks } from "libpkgx" function usePrefix() { diff --git a/lib/porcelain/build-script.ts b/lib/porcelain/build-script.ts new file mode 100644 index 00000000..9645a338 --- /dev/null +++ b/lib/porcelain/build-script.ts @@ -0,0 +1,83 @@ +import { hooks, PackageRequirement, utils, Path } from "pkgx" +import { find_in_PATH } from "brewkit/utils.ts" +import { Config } from "brewkit/config.ts" +import undent from "outdent" + +const { usePantry, useConfig } = hooks +const { host } = utils + +export default async function(config: Config, PATH?: Path): Promise { + const depstr = (deps: PackageRequirement[]) => deps.map(x => `+${utils.pkg.str(x)}`).join(' ') + const env_plus = `${depstr(config.deps.dry.runtime)} ${depstr(config.deps.dry.build)}`.trim() + const user_script = await usePantry().getScript(config.pkg, 'build', config.deps.gas, config.path.build_install) + + const pkgx = find_in_PATH('pkgx') + const bash = find_in_PATH('bash') + const gum = find_in_PATH('gum') + + const brewkitd = new Path(new URL(import.meta.url).pathname).parent().parent().parent() + const brewkit_PATHs = [ + brewkitd.join("share/brewkit"), + PATH + ].compact(x => x?.string).join(':') + + const FLAGS = flags() + if (host().platform == 'darwin') { + FLAGS.push("export MACOSX_DEPLOYMENT_TARGET=11.0") + } + + return undent` + #!/${bash} + + set -eo pipefail + + ${gum} format "## env" + export PATH="${brewkit_PATHs}:$PATH" + set -a + ${env_plus ? `eval "$(CLICOLOR_FORCE=1 ${pkgx} ${env_plus})"` : ''} + set +a + + export PKGX="${pkgx}" + export SRCROOT=${config.path.build.string} + #TODO export XDG_CACHE_HOME="${config.path.cache.string}" + export HOME=${config.path.home.string} + if [ -n "$CI" ]; then + export FORCE_UNSAFE_CONFIGURE=1 + fi + mkdir -p $HOME + ${flags().join('\n')} + + env -u GH_TOKEN -u GITHUB_TOKEN + + ${gum} format "## pantry script start" + set -x + cd ${config.path.build} + + ${user_script} + ` +} + +function flags(): string[] { + const {platform, arch} = host() + const is_linux_x86_64 = platform == 'linux' && arch == 'x86-64' + + const LDFLAGS = (() => { + switch (platform) { + case 'darwin': + return `-Wl,-rpath,${useConfig().prefix.string}` + case 'linux': + if (arch != 'x86-64') return + return '-pie' + } + })() + + const rv: [string, string][] = [] + if (LDFLAGS) { + rv.push(['LDFLAGS', LDFLAGS]) + } + if (is_linux_x86_64) { + rv.push(['CFLAGS', '-fPIC'], ['CXXFLAGS', '-fPIC']) + } + + return rv.map(([key, value]) => `export ${key}="${value} $${key}"`) +} \ No newline at end of file diff --git a/lib/porcelain/fetch.ts b/lib/porcelain/fetch.ts new file mode 100644 index 00000000..14c1ec86 --- /dev/null +++ b/lib/porcelain/fetch.ts @@ -0,0 +1,113 @@ +import { hooks, utils, Stowage, Path, Package } from "pkgx" +import usePantry from "brewkit/hooks/usePantry.ts" +import { Config, ConfigPath } from "brewkit/config.ts"; + +const { useOffLicense } = hooks +const pantry = usePantry() + +export default async function fetch({pkg, path: {tarball_dir, src: dstdir}}: Pick & {path: Pick}) { + const { url, ref, type, stripComponents } = await pantry.getDistributable(pkg) ?? {} + + if (!url) { + throw new Error(`warn: pkg has no srcs: ${utils.pkg.str(pkg)}`) + } + + if (dstdir.isDirectory() && !dstdir.isEmpty()) { + return dstdir + } + + if (type === "git") { + return clone({ dst: dstdir, src: url, ref }) + } else { + const zipfile = await download(url, tarball_dir, pkg) + await hooks.useSourceUnarchiver().unarchive({ dstdir, zipfile, stripComponents }) + if (!Deno.env.get("CI")) { + const cwd = dstdir.string + await new Deno.Command("git", {args: ["init"], cwd}).spawn().status + await new Deno.Command("git", {args: ["add", "."], cwd}).spawn().status + } + return dstdir + } +} + +// Clones a git repo, then builds a src tarball from it +// This allows our system to treat git repos as if they were +// tarballs, improving internal consistency +async function clone({ dst, src, ref }: { dst: Path, src: URL, ref?: string }) { + if (dst.join(".git").isDirectory()) { + return dst + } + + // ensure the parent dir exists + dst.parent().mkdir('p') + + const tmp = Path.mktemp({}) + + const args = [ + "clone", + "--quiet", + "--depth=1" + ] + if (ref) { + args.push("--branch", ref) + } + args.push( + src.toString(), + tmp.string, + ) + + // Clone the specific ref to our temp dir + const proc = new Deno.Command("git", { + args, + // `git` uses stderr for... non errors, and --quiet + // doesn't touch them + stderr: "null", + }) + const status = await proc.spawn().status + if (!status.success) { + throw new Error(`git failed to clone ${src} to ${dst}`) + } + + return dst +} + +async function download(url: URL, dstdir: Path, pkg: Package) { + let slug = pkg.project.replace(/\//g, "∕") // this is a unicode separator + slug += `-${pkg.version}` + const tarball = dstdir.join(slug + new Path(url.pathname).extname()) + + if (tarball.isFile()) { + return tarball + } + + try { + // first try the original location + return await curl({ dst: tarball, src: url }) + } catch (err) { + try { + // then try our mirror + const stowage: Stowage = { pkg, type: 'src', extname: new Path(url.pathname).extname() } + const src = useOffLicense('s3').url(stowage) + return await curl({ dst: tarball, src }) + } catch (err2) { + err.cause = err2 + throw err + } + } +} + +async function curl({ dst, src }: { dst: Path, src: URL }) { + //NOTE we always use curl as deno’s fetch barfs on sourceforge’s SSL for some reason + dst.parent().mkdir('p') + // using cURL as deno’s fetch fails for certain sourceforge URLs + // seemingly due to SSL certificate issues. cURL basically always works ¯\_(ツ)_/¯ + const proc = new Deno.Command("curl", { + args: ["--fail", "--location", "--output", dst.string, src.toString()] + }) + const status = await proc.spawn().status + if (!status.success) { + console.error({ dst, src }) + throw new Error(`cURL failed to download ${src}`) + } + return dst +} diff --git a/lib/porcelain/fix-up.ts b/lib/porcelain/fix-up.ts new file mode 100644 index 00000000..a2eeec34 --- /dev/null +++ b/lib/porcelain/fix-up.ts @@ -0,0 +1,94 @@ +import { hooks, utils, Path, Package, Installation } from "pkgx" +import { Config } from "brewkit/config.ts" +const { usePantry } = hooks +const { host } = utils + +export default async function finish(config: Config) { + const prefix = config.path.install + await fix_rpaths(prefix, config.pkg, config.path.cache, config.deps.gas) + await fix_pc_files(prefix) + await fix_cmake_files(prefix) +} + +////////////////////////////////////////////////////////////////////////////////////// +async function fix_rpaths(pkg_prefix: Path, pkg: Package, cache: Path, deps: Installation[]) { + const bindir = new Path(new URL(import.meta.url).pathname).join("../../bin") + const yml = await usePantry().project(pkg).yaml() + + switch (host().platform) { + case 'darwin': { + if (yml.build.skip === 'fix-machos' || yml.build.skip?.includes('fix-machos')) { + console.info(`skipping rpath fixes for ${pkg.project}`) + break + } + const { success } = await Deno.run({ + cmd: [ + bindir.join('fix-machos.rb').string, + pkg_prefix.string, + ...['bin', 'sbin', 'tbin', 'lib', 'libexec'].compact(x => pkg_prefix.join(x).isDirectory()?.string) + ], + env: { + GEM_HOME: cache.join('brewkit/gem').string + } + }).status() + if (!success) throw new Error("failed to fix machos") + } break + + case 'linux': { + if (yml.build.skip === 'fix-patchelf' || yml.build.skip?.includes('fix-patchelf')) { + console.info(`skipping rpath fixes for ${pkg.project}`) + break + } + + const { success } = await Deno.run({ + cmd: [ + bindir.join('fix-elf.ts').string, + pkg_prefix.string, + ...deps.map(({ path }) => path.string) + ] + }).status() + if (!success) Deno.exit(1) + break + }} +} + +async function fix_pc_files(pkg_prefix: Path) { + //NOTE currently we only support pc files in lib/pkgconfig + // we aim to standardize on this but will relent if a package is found + // that uses share and other tools that build against it only accept that + for (const part of ["share", "lib"]) { + const d = pkg_prefix.join(part, "pkgconfig").isDirectory() + if (!d) continue + for await (const [path, { isFile }] of d.ls()) { + if (isFile && path.extname() == ".pc") { + const orig = await path.read() + const relative_path = pkg_prefix.relative({ to: path.parent() }) + const text = orig.replaceAll(pkg_prefix.string, `\${pcfiledir}/${relative_path}`) + if (orig !== text) { + console.log({ fixing: path }) + path.write({text, force: true}) + } + } + } + } +} + +async function fix_cmake_files(pkg_prefix: Path) { + // Facebook and others who use CMake sometimes rely on a libary's .cmake files + // being shipped with it. This would be fine, except they have hardcoded paths. + // But a simple solution has been found. + const cmake = pkg_prefix.join("lib", "cmake") + if (cmake.isDirectory()) { + for await (const [path, { isFile }] of cmake.walk()) { + if (isFile && path.extname() == ".cmake") { + const orig = await path.read() + const relative_path = pkg_prefix.relative({ to: path.parent() }) + const text = orig.replaceAll(pkg_prefix.string, `\${CMAKE_CURRENT_LIST_DIR}/${relative_path}`) + if (orig !== text) { + console.log({ fixing: path }) + path.write({text, force: true}) + } + } + } + } +} diff --git a/lib/repair.ts b/lib/repair.ts deleted file mode 100644 index e7c27c2e..00000000 --- a/lib/repair.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { hooks, semver, Installation, plumbing } from "pkgx" -const { useCellar } = hooks -const { link } = plumbing - -export default async function(project: string) { - const cellar = useCellar() - const installed = await cellar.ls(project) - const shelf = cellar.shelf(project) - - for await (const [path, {isSymlink}] of shelf.ls()) { - //FIXME shouldn't delete things we may not have created - if (isSymlink) path.rm() - } - - const majors: {[key: number]: Installation[]} = {} - const minors: {[key: number]: Installation[]} = {} - - for (const installation of installed) { - const {pkg: {version: v}} = installation - majors[v.major] ??= [] - majors[v.major].push(installation) - minors[v.minor] ??= [] - minors[v.minor].push(installation) - } - - for (const arr of [minors, majors]) { - for (const installations of Object.values(arr)) { - const version = installations - .map(({pkg: {version}}) => version) - .sort(semver.compare) - .slice(-1)[0] // safe bang since we have no empty arrays in above logic - - link({project, version}) //TODO link lvl2 is possible here - } - } -} diff --git a/lib/run/parse-pkg-str.ts b/lib/run/parse-pkg-str.ts deleted file mode 100644 index a45dc846..00000000 --- a/lib/run/parse-pkg-str.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { utils, hooks, PackageRequirement } from "pkgx" - -export default async function(input: string, opts?: { latest: 'ok' }): Promise { - let update = false - - if (opts?.latest && input.endsWith("@latest")) { - input = input.slice(0, -7) - update = true - } - - const rawpkg = utils.pkg.parse(input) - - const projects = await hooks.usePantry().find(rawpkg.project) - if (projects.length <= 0) throw new Error(`nothing provides: ${input}`) - if (projects.length > 1) throw new Error(`ambiguous pkg: ${input}: ${projects}`) - - const project = projects[0].project //FIXME libpkgx forgets to correctly assign type - const constraint = rawpkg.constraint - - return { project, constraint, update } -} diff --git a/lib/usePantry.getScript.ts b/lib/usePantry.getScript.ts deleted file mode 100644 index 3e7c4417..00000000 --- a/lib/usePantry.getScript.ts +++ /dev/null @@ -1,131 +0,0 @@ -import useConfig from "libpkgx/hooks/useConfig.ts" -import { SupportedPlatforms, SupportedArchitectures } from "libpkgx/utils/host.ts" -import { isArray, isString, isPlainObject, PlainObject } from "is-what" -import { Package, Installation, hooks, utils, semver } from "libpkgx" -import undent from "outdent" - -const { validate, host } = utils -const { useMoustaches } = hooks - -export const getScript = async (pkg: Package, key: 'build' | 'test', deps: Installation[]) => { - const yml = await hooks.usePantry().project(pkg).yaml() - const node = yml[key] - - const mm = useMoustaches() - const script = (input: unknown) => { - const tokens = mm.tokenize.all(pkg, deps) - tokens.push({ - from: "build.root", to: useConfig().prefix.string - }) - if (isArray(input)) input = input.map(obj => { - if (isPlainObject(obj)) { - - const condition = obj["if"] - if (condition) { - if (SupportedPlatforms.includes(condition) && host().platform != condition) return '' - if (SupportedArchitectures.includes(condition) && host().arch != condition) return '' - if (condition.includes("/")) { - const [platform, arch] = condition.split("/") - if (SupportedPlatforms.includes(platform) && - SupportedArchitectures.includes(arch) && - (host().platform != platform || - host().arch != arch)) return '' - } - - const range = semver.Range.parse(condition) - if (range && !range.satisfies(pkg.version)) return '' - } - - let run = obj['run'] - if (isArray(run)) { - run = run.join("\n") - } else if (!isString(run)) { - throw new Error('every node in a script YAML array must contain a `run` key') - } - - let cd = obj['working-directory'] - if (cd) { - cd = mm.apply(validate.str(cd), tokens) - run = undent` - OLDWD="$PWD" - mkdir -p "${cd}" - cd "${cd}" - ${run.trim()} - cd "$OLDWD" - unset OLDWD - ` - } - - let fixture_key = key == 'build' ? 'prop' : 'fixture' - let fixture = obj[fixture_key] - if (fixture) { - fixture_key = fixture_key.toUpperCase() - fixture = mm.apply(validate.str(fixture), tokens) - run = undent` - OLD_${fixture_key}=$${fixture_key} - ${fixture_key}=$(mktemp) - - cat < $${fixture_key} - ${fixture} - DEV_PKGX_EOF - - ${run} - - rm -f $${fixture_key} - - if test -n "$${fixture_key}"; then - ${fixture_key}=$OLD_${fixture_key} - else - unset ${fixture_key} - fi - ` - } - - return run.trim() - } else { - return `${obj}`.trim() - } - }).join("\n\n") - return mm.apply(validate.str(input), tokens) - } - - if (isPlainObject(node)) { - let raw = script(node.script) - - let wd = node["working-directory"] - if (wd) { - wd = mm.apply(wd, [ - ...mm.tokenize.version(pkg.version), - ...mm.tokenize.host(), - ...mm.tokenize.pkg(pkg) - ]) - raw = undent` - mkdir -p ${wd} - cd ${wd} - - ${raw} - ` - } - - const env = node.env - if (isPlainObject(env)) { - raw = `${expand_env(env, pkg, deps)}\n\n${raw}` - } - return raw - } else { - return script(node) - } -} - -function expand_env(env: PlainObject, pkg: Package, deps: Installation[]): string { - const { expand_env_obj } = hooks.usePantry() - return Object.entries(expand_env_obj(env, pkg, deps)).map(([key,value]) => { - // weird POSIX string escaping/concat stuff - // eg. export FOO="bar ""$baz"" bun" - value = `"${value.trim().replace(/"/g, '""')}"` - while (value.startsWith('""')) value = value.slice(1) //FIXME lol better pls - while (value.endsWith('""')) value = value.slice(0,-1) //FIXME lol better pls - - return `export ${key}=${value}` - }).join("\n") -} diff --git a/lib/utils.ts b/lib/utils.ts index 10fa2384..44000301 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -28,3 +28,29 @@ Array.prototype.uniq = function(): Array { export function swallow(fn: () => T) { try { return fn() } catch { /*noop*/ } } + +export async function gum(title: string) { + await new Deno.Command("gum", {args: ['format', `# ${title}`]}).spawn().status +} + +export function find_pkgx() { + return find_in_PATH('pkgx') +} +export function find_in_PATH(program: string) { + for (const part of Deno.env.get("PATH")?.split(":") ?? []) { + const path = (Path.abs(part) || Path.cwd().join(part)).join(program) + if (path.isExecutableFile()) { + return path + } + } + throw new Error(`couldn't find \`${program}\` binary`) +} + +export async function rsync(from: Path, to: Path, additional_args: string[] = []) { + console.log(`rsync ${from.string} ${to.string}`) + to.parent().mkdir('p') + const v = Deno.env.get("VERBOSE") ? 'v' : '' + const args = [`-a${v}`, '--delete', ...additional_args, `${from.string}/`, to.string] + const {success} = await new Deno.Command("rsync", {args}).spawn().status + if (!success) throw new Error("rsync failed") +} diff --git a/lib/Unarchiver.ts b/lib/utils/Unarchiver.ts similarity index 100% rename from lib/Unarchiver.ts rename to lib/utils/Unarchiver.ts diff --git a/libexec/available.ts b/libexec/available.ts deleted file mode 100755 index a29c7bc5..00000000 --- a/libexec/available.ts +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env -S pkgx deno run --allow-env --allow-read - -import { parseFlags } from "cliffy/flags/mod.ts" -import { hooks, utils } from "pkgx" - -const { parse } = utils.pkg -const { usePantry } = hooks -const { unknown: pkgnames } = parseFlags(Deno.args) - -for(const pkg of pkgnames.map(parse)) { - if (!await usePantry().project(pkg).available()) { - Deno.exit(1) - } -} diff --git a/libexec/deps.ts b/libexec/deps.ts deleted file mode 100755 index 97fb83f7..00000000 --- a/libexec/deps.ts +++ /dev/null @@ -1,37 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-net --allow-read --allow-env - -import { parseFlags } from "cliffy/flags/mod.ts" -import { utils, plumbing, hooks } from "pkgx" - -const { usePantry } = hooks -const { parse, str } = utils.pkg -const { hydrate } = plumbing - -const { flags: { build, test }, unknown: [pkgname] } = parseFlags(Deno.args, { - flags: [{ - name: "build" - }, { - name: "test" - }], -}) - -const pkg = parse(pkgname) -const pantry = usePantry() - -/// when building we don’t incorporate the target into the hydration graph -/// because we don’t want it yet: we”re about to build it - -const dry = build - ? (({ runtime, build }) => [...runtime, ...build])(await pantry.getDeps(pkg)) - : [pkg] - -const { pkgs: wet } = await hydrate(dry, async (pkg, dry) => { - const deps = await pantry.getDeps(pkg) - if (dry && test) { - return [...deps.runtime, ...deps.test] - } else { - return deps.runtime - } -}) - -console.info(wet.map(str).join("\n")) diff --git a/libexec/extract.ts b/libexec/extract.ts deleted file mode 100755 index c2d79245..00000000 --- a/libexec/extract.ts +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env -S pkgx +curl +git +tar +xz +unzip +bzip2 deno run -A - -import { Package, PackageRequirement, Path, utils, hooks } from "pkgx" -const { useSourceUnarchiver, usePantry } = hooks -import { parseFlags } from "cliffy/flags/mod.ts" -const { panic } = utils - -const { flags: { outputDir, pkg: pkgname }, unknown } = parseFlags(Deno.args, { - flags: [{ - name: "output-dir", - type: "string", - required: true - }, { - name: "pkg", - type: "string", - required: true - }], -}) - -const pantry = usePantry() - -let pkg: Package | PackageRequirement = utils.pkg.parse(pkgname) -pkg = { project: pkg.project, version: pkg.constraint.single() ?? panic() } - -const dstdir = Path.cwd().join(outputDir) - -if (!dstdir.isDirectory() || dstdir.exists()?.isEmpty()) { - const zipfile = new Path(unknown[0]) - const { stripComponents } = await pantry.getDistributable(pkg) ?? {} - await useSourceUnarchiver().unarchive({ dstdir, zipfile, stripComponents }) -} else { - console.error("notice: already extracted: skipping") -} diff --git a/libexec/fetch.ts b/libexec/fetch.ts deleted file mode 100755 index 2f1cd833..00000000 --- a/libexec/fetch.ts +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env -S pkgx +curl.se +git-scm.org +gnu.org/tar +tukaani.org/xz deno run -A - -import { hooks, utils, Stowage, Path } from "pkgx" -import { parseFlags } from "cliffy/flags/mod.ts" -import usePantry from "../lib/usePantry.ts" - -const { useOffLicense, useCache, useDownload } = hooks -const pantry = usePantry() -const { panic } = utils - -const { flags, unknown: [pkgname] } = parseFlags(Deno.args, { - flags: [{ - name: "output-dir", - type: "string" - }, { - name: "o", - type: "string" - }] -}) - -const pkg = await pantry.resolve(utils.pkg.parse(pkgname)) -const { url, ref, type } = await pantry.getDistributable(pkg) ?? {} - -if (!url) { - console.error("warn: pkg has no srcs: ", pkg) - Deno.exit(0) -} - -try { - const zipfile = await (async () => { - const stowage: Stowage = (() => { - if (type === "git") { - return { pkg, type: 'src', extname: `.tar.xz` } - } else { - return { pkg, type: 'src', extname: new Path(url.pathname).extname() } - } - })() - - const dst = (() => { - if (flags.outputDir) { - const filename = useCache().path(stowage).basename() - return Path.cwd().join(flags.outputDir, filename) - } else { - return Path.cwd().join(flags.o ?? panic()) - } - })() - - if (type === "git") return clone({ dst, src: url, ref }) - - try { - // first try the original location - return await download({ dst, src: url }) - } catch (err) { - try { - // then try our mirror - const src = useOffLicense('s3').url(stowage) - return await download({ dst, src }) - } catch (err2) { - console.error("mirror:", err2.message) - throw err - } - } - })() - - console.info(zipfile.string) -} catch (err) { - console.error(err.message) - console.error("pkgx expands the full semantic version, which may mean the URL you are") - console.error("fetching is now incorrect. Try `version.raw`?") - Deno.exit(1) -} - -async function download({ dst, src }: { dst: Path, src: URL }) { - if (Deno.env.get("GITHUB_ACTIONS")) { - dst.parent().mkdir('p') - // using cURL as deno’s fetch fails for certain sourceforge URLs - // seemingly due to SSL certificate issues. cURL basically always works ¯\_(ツ)_/¯ - const proc = new Deno.Command("curl", { - args: ["--fail", "--location", "--output", dst.string, src.toString()] - }) - const status = await proc.spawn().status - if (!status.success) { - console.error({ dst, src }) - throw new Error(`cURL failed to download ${src}`) - } - } else { - // locally using our download function as it knows how to cache properly - await useDownload().download({ dst, src }) - } - return dst -} - -// Clones a git repo, then builds a src tarball from it -// This allows our system to treat git repos as if they were -// tarballs, improving internal consistency -async function clone({ dst, src, ref }: { dst: Path, src: URL, ref?: string }) { - if (dst.isFile()) { - console.info("using cached tarball") - return dst - } - - // ensure the parent dir exists - dst.parent().mkdir('p') - - const tmp = Path.mktemp({}) - - const args = [ - "clone", - "--quiet", - "--depth=1" - ] - if (ref) { - args.push("--branch", ref) - } - args.push( - src.toString(), - tmp.string, - ) - - // Clone the specific ref to our temp dir - const proc = new Deno.Command("git", { - args, - // `git` uses stderr for... non errors, and --quiet - // doesn't touch them - stderr: "null", - }) - const status = await proc.spawn().status - if (!status.success) { - console.error({ dst, src }) - throw new Error(`git failed to clone ${src}`) - } - - // Create a tarball from the temp dir - const proc2 = new Deno.Command("tar", { - args: [ - "-C", - tmp.string, - // Prevents `tar: Ignoring unknown extended header keyword 'LIBARCHIVE.xattr.com.apple.provenance'` - // when unpacking on darwin - "--no-xattrs", - "-czf", - dst.string, - ".", - ] - }) - const status2 = await proc2.spawn().status - if (!status2.success) { - console.error({ dst, src }) - throw new Error(`tar failed to create ${dst}`) - } - return dst -} \ No newline at end of file diff --git a/libexec/find.ts b/libexec/find.ts deleted file mode 100755 index 4f4c7f4b..00000000 --- a/libexec/find.ts +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env -S pkgx deno run --allow-read --allow-env - -import { hooks, utils } from "pkgx" -const { find } = hooks.usePantry() - -for (const arg of Deno.args) { - - const {project, constraint} = utils.pkg.parse(arg) - - const rv = await find(project) - - if (rv.length > 1) { - console.error("multiple matches: " + rv.map(({project}) => project).join(' ')) - Deno.exit(1) - } - if (rv.length == 0) { - console.error("no matches for: " + arg) - Deno.exit(2) - } - - if (Deno.env.get("_PATHS")) { - console.info(rv[0].path.string) - } else { - const pkg = {project: rv[0].project, constraint} - console.info(utils.pkg.str(pkg)) - } -} diff --git a/libexec/fixup.ts b/libexec/fixup.ts deleted file mode 100755 index c4411dcc..00000000 --- a/libexec/fixup.ts +++ /dev/null @@ -1,97 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-run --allow-read --allow-write --allow-env - -import { parseFlags } from "cliffy/flags/mod.ts" -import { hooks, utils, Path } from "pkgx" - -const { useCellar, useConfig, usePantry } = hooks -const { pkg: { str, parse }, host } = utils - -const { flags, unknown } = parseFlags(Deno.args, { - flags: [{ - name: "deps", - type: "string", - required: true, - optionalValue: true - }], -}) - -const cellar = useCellar() -const pkg_prefix = new Path(unknown[1]) -const pkg = parse(unknown[0]) -const yml = await usePantry().project(pkg).yaml() - -switch (host().platform) { -case 'darwin': { - if (yml.build.skip === 'fix-machos' || yml.build.skip?.includes('fix-machos')) { - console.info(`skipping rpath fixes for ${pkg.project}`) - break - } - const { success } = await Deno.run({ - cmd: [ - 'fix-machos.rb', - pkg_prefix.string, - ...['bin', 'sbin', 'tbin', 'lib', 'libexec'].compact(x => pkg_prefix.join(x).isDirectory()?.string) - ], - env: { - GEM_HOME: useConfig().prefix.join('.local/share/ruby/gem').string - } - }).status() - if (!success) throw new Error("failed to fix machos") -} break - -case 'linux': { - if (yml.build.skip === 'fix-patchelf' || yml.build.skip?.includes('fix-patchelf')) { - console.info(`skipping rpath fixes for ${pkg.project}`) - break - } - - const raw = flags.deps == true ? '' : flags.deps as string - const installs = await Promise.all(raw.split(/\s+/).map(path => cellar.resolve(new Path(path)))) - const deps = installs.map(({ pkg }) => str(pkg)) - const { success } = await Deno.run({ - cmd: [ - 'fix-elf.ts', - pkg_prefix.string, - ...deps - ] - }).status() - if (!success) Deno.exit(1) - break -}} - -//NOTE currently we only support pc files in lib/pkgconfig -// we aim to standardize on this but will relent if a package is found -// that uses share and other tools that build against it only accept that -for (const part of ["share", "lib"]) { - const d = pkg_prefix.join(part, "pkgconfig").isDirectory() - if (!d) continue - for await (const [path, { isFile }] of d.ls()) { - if (isFile && path.extname() == ".pc") { - const orig = await path.read() - const relative_path = pkg_prefix.relative({ to: path.parent() }) - const text = orig.replaceAll(pkg_prefix.string, `\${pcfiledir}/${relative_path}`) - if (orig !== text) { - console.log({ fixing: path }) - path.write({text, force: true}) - } - } - } -} - -// Facebook and others who use CMake sometimes rely on a libary's .cmake files -// being shipped with it. This would be fine, except they have hardcoded paths. -// But a simple solution has been found. -const cmake = pkg_prefix.join("lib", "cmake") -if (cmake.isDirectory()) { - for await (const [path, { isFile }] of cmake.walk()) { - if (isFile && path.extname() == ".cmake") { - const orig = await path.read() - const relative_path = pkg_prefix.relative({ to: path.parent() }) - const text = orig.replaceAll(pkg_prefix.string, `\${CMAKE_CURRENT_LIST_DIR}/${relative_path}`) - if (orig !== text) { - console.log({ fixing: path }) - path.write({text, force: true}) - } - } - } -} diff --git a/libexec/init.ts b/libexec/init.ts deleted file mode 100755 index 140710ef..00000000 --- a/libexec/init.ts +++ /dev/null @@ -1,20 +0,0 @@ -#!//usr/bin/env -S pkgx +ruby deno run -A - -import { swallow } from "../lib/utils.ts" - -const arg = Deno.args[0] - -let project: string - -const url = swallow(() => new URL(arg)) -if (url) { - project = url.hostname + url.pathname -} else if (arg) { - project = arg -} else { - const { stdout } = await new Deno.Command("sample-blend.rb").output() - project = new TextDecoder().decode(stdout).trim() - project = `wip/${project}` -} - -console.info(project) diff --git a/libexec/install.ts b/libexec/install.ts deleted file mode 100755 index 848af7a3..00000000 --- a/libexec/install.ts +++ /dev/null @@ -1,22 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-read --allow-write --allow-run=tar,/bin/ln --allow-net --allow-env --unstable - -import { utils, hooks, plumbing, Installation } from "pkgx" -const { pkg: { parse, str }, panic } = utils -const { useCellar, useInventory } = hooks -const { install, link } = plumbing - -const cellar = useCellar() -const inventory = useInventory() - -let installation: Installation | undefined -const rv: Installation[] = [] -for (const pkg of Deno.args.map(parse)) { - if (!(installation = await cellar.has(pkg))) { - const version = await inventory.select(pkg) ?? panic(`${str(pkg)} not found`) - installation = await install({ project: pkg.project, version }) - await link(installation) - } - rv.push(installation) -} - -console.info(rv.map(({path})=>path).join("\n")) diff --git a/libexec/inventory.ts b/libexec/inventory.ts deleted file mode 100755 index fe9cd2a9..00000000 --- a/libexec/inventory.ts +++ /dev/null @@ -1,18 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-net --allow-env --allow-read - -import { parseFlags } from "cliffy/flags/mod.ts" -import { hooks, utils, SemVer } from "pkgx" -const { useInventory } = hooks - -const { unknown: pkgnames } = parseFlags(Deno.args) - -const rv: Record = {} -for (const pkg of pkgnames.map(utils.pkg.parse)) { - rv[pkg.project] = await useInventory().get(pkg) -} - -if (pkgnames.length == 1) { - console.info(Object.values(rv)[0].join("\n")) -} else { - console.info(JSON.stringify(rv, null, 2)) -} diff --git a/libexec/link.ts b/libexec/link.ts deleted file mode 100755 index f11c0dd3..00000000 --- a/libexec/link.ts +++ /dev/null @@ -1,11 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-read --allow-write --allow-env --allow-run=/bin/ln - -import { PackageRequirement, Package, Path, utils, plumbing } from "pkgx" -const { link } = plumbing - -let pkg: Package | PackageRequirement = utils.pkg.parse(Deno.args[1]) -pkg = { project: pkg.project, version: pkg.constraint.single() ?? utils.panic() } - -const path = new Path(Deno.args[0]) - -await link({ pkg, path }) diff --git a/libexec/peek.sh b/libexec/peek.sh deleted file mode 100755 index 2a029e7f..00000000 --- a/libexec/peek.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -set -e - -if ! command -v git >/dev/null; then - GIT="pkgx git" -else - GIT=git -fi - -if ! d="$($GIT rev-parse --show-toplevel 2>/dev/null)"; then - echo "brewkit: error: cwd is not inside a git repo" >&2 - exit 1 -fi - -if ! test -d "$d"/projects; then - echo "brewkit: error: cwd is not a pantry" >&2 - exit 2 -fi - -# sadly we seemingly need to reference origin/main -DIVERGENCE_SHA="$($GIT merge-base HEAD origin/main)" -CHANGED_FILES="$($GIT diff --name-only "$DIVERGENCE_SHA") $($GIT status --untracked-files --porcelain)" - -OUTPUT="" - -for CHANGED_FILE in $CHANGED_FILES; do - PROJECT=$(echo "$CHANGED_FILE" | sed -n 's#projects/\(.*\)/package\.yml$#\1#p') - if test -z "$PROJECT" - then - true # noop - elif test "$1" = "--print-paths"; then - OUTPUT="$OUTPUT $CHANGED_FILE" - else - OUTPUT="$OUTPUT $PROJECT" - fi -done - -echo "$OUTPUT" | tr ' ' '\n' | sort -u | tr '\n' ' ' diff --git a/libexec/query.ts b/libexec/query.ts deleted file mode 100755 index bc4bc424..00000000 --- a/libexec/query.ts +++ /dev/null @@ -1,106 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-read --allow-env --allow-write --allow-net - -import { utils, hooks, Package, PackageRequirement, Stowage, Path } from "pkgx" -const { host, flatmap, pkg: { parse, str } } = utils -import { parseFlags } from "cliffy/flags/mod.ts" -const { useCache, useCellar, usePantry } = hooks -const { panic } = utils - -const { flags: { prefix, srcdir, src, testdir, blddir, versions, ...flags }, unknown: [pkgname] } = parseFlags(Deno.args, { - flags: [{ - name: "prefix", - standalone: true - }, { - name: "srcdir", - standalone: true - }, { - name: "src", - standalone: true - }, { - name: "blddir", - aliases: ["build-dir"], - standalone: true - }, { - name: "testdir", - standalone: true - }, { - name: "versions", - standalone: true - }, { - name: "url", - standalone: true - }] -}) - -let pkg: PackageRequirement | Package = parse(pkgname) - -if (versions) { - const versions = await usePantry().getVersions(pkg) - console.info(versions.sort().join("\n")) - Deno.exit(0) -} - -const version = pkg.constraint.single() ?? panic() -pkg = {project: pkg.project, version } - -if (src) { - const { url, type } = await usePantry().getDistributable(pkg) ?? {} - if (!url) { - console.error("warn: pkg has no srcs: ", str(pkg)) - Deno.exit(0) // NOT AN ERROR EXIT CODE THO - } - const stowage: Stowage = (() => { - if (type === 'git') - return { pkg, type: 'src', extname: '.tar.xz' } - return { pkg, type: 'src', extname: new Path(url.pathname).extname() } - })() - const path = flatmap(Deno.env.get("SRCROOT"), x => new Path(x)) - const cache_path = useCache().path(stowage) - if (path?.join("projects").isDirectory()) { - console.info(`${path.join("srcs", cache_path.basename())}`) - } else { - console.info(cache_path.string) - } -} else if (prefix) { - const path = useCellar().keg(pkg) - console.info(path.string) -} else if (blddir) { - let path = flatmap(Deno.env.get("SRCROOT"), x => new Path(x)) - if (path?.join("projects").isDirectory()) { - const project = pkg.project.replaceAll("/", "∕") - const platform = host().platform - path = path.join("builds").join(`${project}-${pkg.version}+${platform}`) - } else { - path = new Path(Deno.makeTempDirSync()) - } - console.info(path.string) -} else if (testdir) { - let path = flatmap(Deno.env.get("SRCROOT"), x => new Path(x)) - if (path?.join("projects").isDirectory()) { - const project = pkg.project.replaceAll("/", "∕") - const platform = host().platform - path = path.join("testbeds").join(`${project}-${pkg.version}+${platform}`) - } else { - path = new Path(Deno.makeTempDirSync()) - } - console.info(path.string) -} else if (flags.url) { - const { url } = await usePantry().getDistributable(pkg) ?? {} - if (url) { - console.info(url.toString()) - } else { - console.error("null") - Deno.exit(2) - } -} else if (srcdir) { - let path = flatmap(Deno.env.get("SRCROOT"), x => new Path(x)) - if (path?.join("projects").isDirectory()) { - const project = pkg.project.replaceAll("/", "∕") - path = path.join("srcs").join(`${project}-${pkg.version}`) - } else { - path = new Path(Deno.makeTempDirSync()) - } - console.info(path.string) -} else { - Deno.exit(1) -} diff --git a/libexec/resolve.ts b/libexec/resolve.ts deleted file mode 100755 index ee9bdb5e..00000000 --- a/libexec/resolve.ts +++ /dev/null @@ -1,23 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-read --allow-net --allow-env - -const { useCellar, usePantry } = hooks -import { parseFlags } from "cliffy/flags/mod.ts" -import { utils, hooks } from "pkgx" -const { parse, str } = utils.pkg - -const { flags, unknown: [pkgname] } = parseFlags(Deno.args, { - flags: [{ - name: "cellar", - }] -}) - -if (!flags.cellar) { - const pkg = await usePantry().resolve(parse(pkgname)) - console.info(str(pkg)) -} else { - const entry = await useCellar().has(parse(pkgname)) - if (!entry) { - throw new Error(`${pkgname} not installed in $PKGX_DIR`) - } - console.info(str(entry.pkg)) -} diff --git a/libexec/sample-blend.rb b/libexec/sample-blend.rb deleted file mode 100755 index ed56e780..00000000 --- a/libexec/sample-blend.rb +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env ruby - -# thank you ChatGPT -# PRs welcome! - -puts %w{ - oolong - darjeeling - earl-grey - lady-grey - english-breakfast - chai - pu-erh - jasmine - hibiscus - rooibos - matcha - genmaicha - sencha - gyokuro - yerba-mate - chrysanthemum - dragonwell - keemun - lapsang-souchong - assam - ceylon - kenyan - nilgiri - mao-jian - bai-hao-yinzhen - tie-guan-yin - longjing - luan-guapian - bi-luo-chun - junshan-yinzhen - huangshan-maofeng - fenghuang-dancong - bai-mudan - shoumei - gongmei - ya’an-baozhong - tai-ping-houkui - qimen-hongcha - huangshan-hongcha - guapian - yuezhan - herbal-tea - tisane - spiced-tea -}.sample diff --git a/libexec/search.ts b/libexec/search.ts deleted file mode 100755 index 9e60179d..00000000 --- a/libexec/search.ts +++ /dev/null @@ -1,40 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-env --allow-read --allow-net - -import { parseFlags } from "cliffy/flags/mod.ts" -const { usePantry, useInventory } = hooks -import { hooks, semver } from "pkgx" - -const { unknown: [query] } = parseFlags(Deno.args) -const pantry = usePantry() - -const encoder = new TextEncoder() -const print = (x: string) => Deno.stdout.write(encoder.encode(x)) - -const outputted: Set = new Set() - -for await (const pkg of pantry.ls()) { - // often we are dealing with two pantries that are mostly the same during dev/debug - if (outputted.has(pkg.project)) continue - - let output = false - const bins: string[] = [] - for (const bin of await pantry.project(pkg).provides().swallow() ?? []) { - if (bin.includes(query)) { - output = true - bins.push(bin) - } - } - if (output) { - print(`${pkg.project}: ${bins.join(', ')}`) - } else if (pkg.project.includes(query)) { - print(pkg.project.trim()) - output = true - } - if (output) { - const rq = { project: pkg.project, constraint: new semver.Range('*') } - const got = await useInventory().select(rq) - print(`: ${got}\n`) - } - - outputted.add(pkg.project) -} diff --git a/libexec/sort.ts b/libexec/sort.ts deleted file mode 100755 index d7bc4f63..00000000 --- a/libexec/sort.ts +++ /dev/null @@ -1,28 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-read --allow-env - -import { parseFlags } from "cliffy/flags/mod.ts" -import { utils, plumbing, hooks } from "pkgx" -const { parse, str } = utils.pkg -const { usePantry } = hooks -const { hydrate } = plumbing - -const { flags: { delimiter: separator }, unknown: args } = parseFlags(Deno.args, { - flags: [{ - name: "delimiter", - aliases: ["d"], - type: "string", - default: "\n", - required: false - }], -}) - - -const pantry = usePantry() - -const dry = args.map(parse) -const wet = await hydrate(dry, async (pkg, dry) => { - const deps = await pantry.getDeps(pkg) - return dry ? [...deps.build, ...deps.runtime] : deps.runtime -}) - -console.info(wet.dry.map(str).join(separator)) diff --git a/libexec/stage-test.ts b/libexec/stage-test.ts deleted file mode 100755 index b5fb1e20..00000000 --- a/libexec/stage-test.ts +++ /dev/null @@ -1,100 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-read --allow-write --allow-env - -import { parseFlags } from "cliffy/flags/mod.ts" -import { hooks, utils, Path } from "pkgx" -import undent from "outdent" -import host from "libpkgx/utils/host.ts"; - -const { usePantry, useCellar, useConfig, useShellEnv } = hooks -const { pkg: pkgutils, panic } = utils - -const { flags, unknown: [pkgname] } = parseFlags(Deno.args, { - flags: [{ - name: "deps", - type: "string", - optionalValue: true - }, { - name: "dstdir", - type: "string", - required: true - }] -}) - -const pantry = usePantry() -const rq = pkgutils.parse(pkgname) -const cellar = useCellar() -const self = await cellar.has(rq) ?? panic() -const { pkg } = self -const deps = await (() => { - if (typeof flags.deps != 'string' || !flags.deps) return Promise.resolve([]) - const parts = flags.deps.split(/\s+/) - const pp = parts.map(x => cellar.resolve(new Path(x))) - return Promise.all(pp) -})() -const dstdir = new Path(flags.dstdir) -const project = pantry.project(pkg) -const yml = await project.yaml() -const installations = [...deps] -if (deps.find(x => x.pkg.project == self.pkg.project) === undefined) installations.push(self) - -/// try to find `pkgx` since we deliberately withold it from the PATH for tests -/// since it needs to be an explicit dependency -const pkgx = (PATH => { - for (const path of PATH.split(":")) { - const f = Path.abs(path)?.join("pkgx").isExecutableFile() - if (f) return f.string - } -})(Deno.env.get("PATH") ?? '') ?? 'pkgx' - -Deno.env.set("HOME", dstdir.string) //lol side-effects beware! -const env = await useShellEnv().map({ installations }) - -if (!yml.test) throw "no `test` node in package.yml" - -env['PATH'] ??= [] -env['PATH'].push("/usr/bin:/bin") - -let text = undent` - #!/usr/bin/env -S pkgx bash - - set -exo pipefail - - command_not_found_handle() { - echo "::warning::\\\`$1\\\` is not an explicit dependency!" - case $1 in - cc|c++|ld) - ${pkgx} +llvm.org -- "$@";; - *) - ${pkgx} "$@";; - esac - } - - export PKGX_DIR="${useConfig().prefix}" - export HOME="${dstdir}" - - ${useShellEnv().expand(env)} - - ` - -if (yml.test.fixture) { - const fixture = dstdir.join("dev.pkgx.fixture").write({ text: yml.test.fixture.toString() }) - text += `export FIXTURE="${fixture}"\n\n` -} - -text += `cd "${dstdir}"\n\n` - -text += await pantry.getScript(pkg, 'test', deps) -text += "\n" - -for await (const [path, {name, isFile}] of (await pantry.filepath(pkg.project)).parent().ls()) { - if (isFile && name != 'package.yml') { - path.cp({ into: dstdir }) - } -} - -const sh = dstdir - .join("dev.pkgx.test.sh") - .write({ text, force: true }) - .chmod(0o500) - -console.info(sh.string) diff --git a/libexec/stage.ts b/libexec/stage.ts deleted file mode 100755 index cc2a7f68..00000000 --- a/libexec/stage.ts +++ /dev/null @@ -1,168 +0,0 @@ -#!//usr/bin/env -S pkgx deno run --allow-net --allow-read --allow-env --allow-write --allow-run=cp - -import { parseFlags } from "cliffy/flags/mod.ts" -import { hooks, utils, Path } from "pkgx" -import undent from "outdent" - -const { useShellEnv, useCellar, useConfig, usePantry } = hooks -const { host, pkg: { parse } } = utils -const { flags, unknown: [pkgname] } = parseFlags(Deno.args, { - flags: [{ - name: "srcdir", - type: "string", - required: true - }, { - name: "prefix", - type: "string", - required: true - }, { - name: "deps", - type: "string", - optionalValue: true - }, { - name: "blddir", - aliases: ["build-dir"], - type: "string", - required: true - }], -}) - -const pantry = usePantry() -const cellar = useCellar() -const prefix = useConfig().prefix -const srcdir = Path.cwd().join(flags.srcdir) -const blddir = Path.cwd().join(flags.blddir) -const deps = await (() => { - if (typeof flags.deps != 'string' || !flags.deps) return Promise.resolve([]) - const parts = flags.deps.split(/\s+/) - const pp = parts.map(x => cellar.resolve(new Path(x))) - return Promise.all(pp) -})() -if (blddir.string.includes(" ")) { - console.error("warning: build directory contains spaces. build tools *may choke*") -} - -if (!blddir.isDirectory() || blddir.exists()?.isEmpty()) { - blddir.rm().parent().mkdir('p') - // NOTE we use cp -a to preserve symlinks - // We'd love to use deno/sd/copy.ts but it fails on symlinks - // https://github.com/denoland/deno_std/issues/3454 - await Deno.run({ cmd: ["cp", "-a", srcdir.string, blddir.string] }).status() -} - -//FIXME this goes to GitHub, and we already did this once -// NOTE we have to in order to get `version.raw` which many pkgymls need -const pkg = await pantry.resolve(parse(pkgname)) - -/// assemble build script -const pantry_sh = await pantry.getScript(pkg, 'build', deps) -const sup_PATH = [new Path(new URL(import.meta.url).pathname).parent().parent().join("share/brewkit")] - -const use_cc_shims = !deps.find(({pkg}) => pkg.project == 'llvm.org' || pkg.project == 'gnu.org/gcc') - -if (use_cc_shims) { - /// add our helper cc toolchain unless the package has picked its own toolchain - sup_PATH.push(new Path(new URL(import.meta.url).pathname).parent().parent().join("share/toolchain/bin")) - - if (host().platform != "darwin") { - const symlink = (names: string[], {to}: {to: string}) => { - const d = blddir.join('dev.pkgx.bin').mkdir() - for (const name of names) { - const path = d.join(name) - if (path.exists()) continue - const target = prefix.join('llvm.org/v*/bin', to) - path.ln('s', { target }) - } - } - - symlink(["cc", "gcc", "clang"], {to: "clang"}) - symlink(["c++", "g++", "clang++"], {to: "clang++"}) - symlink(["cpp"], {to: "clang-cpp"}) - - if (host().platform == "linux") { - symlink(["ld"], {to: "ld.lld"}) - } else if (host().platform == "windows") { - symlink(["ld"], {to: "lld-link"}) - } - - symlink(["ld.lld"], {to: "ld.lld"}) - symlink(["lld-link"], {to: "lld-link"}) - - symlink(["ar"], {to: "llvm-ar"}) - symlink(["as"], {to: "llvm-as"}) - symlink(["nm"], {to: "llvm-nm"}) - symlink(["objcopy"], {to: "llvm-objcopy"}) - symlink(["ranlib"], {to: "llvm-ranlib"}) - symlink(["readelf"], {to: "llvm-readelf"}) - symlink(["strings"], {to: "llvm-strings"}) - symlink(["strip"], {to: "llvm-strip"}) - } -} - -/// calc env -const sh = useShellEnv() -const old_home = Deno.env.get("HOME") -Deno.env.set("HOME", blddir.string) //lol side-effects beware! -const env = await sh.map({ installations: deps }) -Deno.env.set("HOME", old_home!) - -if (host().platform == 'darwin') env['MACOSX_DEPLOYMENT_TARGET'] = ['11.0'] - -env['PATH'] ??= [] -env['PATH'].push("/usr/bin", "/bin", "/usr/sbin", "/sbin", useConfig().prefix.join('pkgx.sh/v*/bin').string) - -if (host().platform == 'linux' && host().arch == 'x86-64') { - env['LDFLAGS'] = [`${env['LDFLAGS']?.[0] ?? ''} -pie`.trim()] - env['CFLAGS'] = [`${env['CFLAGS']?.[0] ?? ''} -fPIC`.trim()] - env['CXXFLAGS'] = [`${env['CXXFLAGS']?.[0] ?? ''} -fPIC`.trim()] -} -if (host().platform == 'darwin' && !use_cc_shims) { - // our shims inject this but if we aren’t using them we need to add it manually or everything breaks - env['LDFLAGS'] = [`${env['LDFLAGS']?.[0] ?? ''} -Wl,-rpath,${useConfig().prefix}`.trim()] -} - -const text = undent` - #!/usr/bin/env -S pkgx bash - - set -exo pipefail - - cd "${blddir}" - - export HOME="${blddir.parent().parent().join("homes", blddir.basename()).mkdir('p')}" - export SRCROOT="${blddir}" - export PREFIX=${flags.prefix} - export PKGX_DIR=${prefix.string} - ${sh.expand(env)} - - mkdir -p "$HOME" - - export PATH=${sup_PATH.map(x => x.string).join(':')}:"$PATH" - export CFLAGS="-w $CFLAGS" # warnings are noise - - ${pantry_sh} - ` - -/// write out build script -const script = blddir.join("dev.pkgx.build.sh").write({ text, force: true }).chmod(0o755) - -/// write out pkgx.yaml so dev-env works -import * as YAML from "deno/yaml/stringify.ts" - -blddir.join("pkgx.yaml").write({ text: YAML.stringify({ - env: sh.flatten(env), - dependencies: deps.reduce((acc, {pkg}) => { - acc[pkg.project] = `=${pkg.version}` - return acc - }, {} as Record) -}), force: true }) - -/// copy in auxillary files from pantry directory -for await (const [path, {isFile}] of (await pantry.filepath(pkg.project)).parent().ls()) { - if (isFile) { - path.cp({ into: blddir.join("props").mkdir() }) - } -} - - -/// done -console.info(script.string) diff --git a/projects/pyapp.com/1/myapp/__init__.py b/projects/pyapp.com/1/myapp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/projects/pyapp.com/1/myapp/main.py b/projects/pyapp.com/1/myapp/main.py new file mode 100644 index 00000000..97afe981 --- /dev/null +++ b/projects/pyapp.com/1/myapp/main.py @@ -0,0 +1,9 @@ +import requests + +def get_website_content(url): + response = requests.get(url) + return response.text + +def main(): + content = get_website_content("https://example.com") + print(content) diff --git a/projects/pyapp.com/1/package.yml b/projects/pyapp.com/1/package.yml new file mode 100644 index 00000000..4d290314 --- /dev/null +++ b/projects/pyapp.com/1/package.yml @@ -0,0 +1,18 @@ +distributable: null + +versions: + - 1.0.0 + +dependencies: + python.org: ^3.11 + +build: | + mv props/* . + rmdir props + python-venv.py {{prefix}}/bin/myapp + +provides: + - bin/myapp + +test: + myapp diff --git a/projects/pyapp.com/1/requirements.txt b/projects/pyapp.com/1/requirements.txt new file mode 100644 index 00000000..663bd1f6 --- /dev/null +++ b/projects/pyapp.com/1/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/projects/pyapp.com/1/setup.py b/projects/pyapp.com/1/setup.py new file mode 100644 index 00000000..4d94c22e --- /dev/null +++ b/projects/pyapp.com/1/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +setup( + name='myapp', + version='1.0.0', + packages=find_packages(), + install_requires=[ + 'requests', + ], + entry_points={ + 'console_scripts': [ + 'myapp=myapp.main:main', + ], + }, +) diff --git a/projects/pyapp.com/2/myapp/__init__.py b/projects/pyapp.com/2/myapp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/projects/pyapp.com/2/myapp/main.py b/projects/pyapp.com/2/myapp/main.py new file mode 100644 index 00000000..97afe981 --- /dev/null +++ b/projects/pyapp.com/2/myapp/main.py @@ -0,0 +1,9 @@ +import requests + +def get_website_content(url): + response = requests.get(url) + return response.text + +def main(): + content = get_website_content("https://example.com") + print(content) diff --git a/projects/pyapp.com/2/package.yml b/projects/pyapp.com/2/package.yml new file mode 100644 index 00000000..18135f72 --- /dev/null +++ b/projects/pyapp.com/2/package.yml @@ -0,0 +1,18 @@ +distributable: null + +versions: + - 1.0.0 + +dependencies: + python.org: ^3.11 + +build: | + mv props/* . + rmdir props + python-venv.sh {{prefix}}/bin/myapp + +provides: + - bin/myapp + +test: + myapp diff --git a/projects/pyapp.com/2/requirements.txt b/projects/pyapp.com/2/requirements.txt new file mode 100644 index 00000000..663bd1f6 --- /dev/null +++ b/projects/pyapp.com/2/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/projects/pyapp.com/2/setup.py b/projects/pyapp.com/2/setup.py new file mode 100644 index 00000000..4d94c22e --- /dev/null +++ b/projects/pyapp.com/2/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +setup( + name='myapp', + version='1.0.0', + packages=find_packages(), + install_requires=[ + 'requests', + ], + entry_points={ + 'console_scripts': [ + 'myapp=myapp.main:main', + ], + }, +) diff --git a/projects/stark.com/foo/package-mod.yml b/projects/stark.com/foo/package-mod.yml new file mode 100644 index 00000000..4ff68b0a --- /dev/null +++ b/projects/stark.com/foo/package-mod.yml @@ -0,0 +1,22 @@ +distributable: + url: https://github.com/pkgxdev/brewkit/archive/refs/tags/v{{ version }}.tar.gz + strip-components: 1 + +versions: + github: pkgxdev/brewkit + +dependencies: + pkgx.sh: ^1 + +build: + - working-directory: ${{prefix}}/bin + run: | + echo '#!/bin/sh' >> stark + echo 'echo not_much_u' >> stark + chmod +x stark + +provides: + - bin/stark + +test: + test $(stark) = not_much_u diff --git a/projects/stark.com/foo/package.yml b/projects/stark.com/foo/package.yml new file mode 100644 index 00000000..938ad358 --- /dev/null +++ b/projects/stark.com/foo/package.yml @@ -0,0 +1,42 @@ +distributable: ~ + +versions: + - 1.2.3 + - 2.3.4 + +build: + script: + - working-directory: ${{prefix}}/bin + run: | + echo '#!/bin/sh' >> stark + echo 'echo sup_bro' >> stark + chmod +x stark + + # verify these are set + - test -n "$PKGX_DIR" + - test "$SRCROOT" = "$PWD" + + # verify GNU sed (doesn’t require a backup file) + - sed -i s/sup_bro/sup_unc/g props/package.yml + + # verify pkgx is not here + - | + if ! command which pkgx + then exit 1 + fi + + # check for regression + - test $PREFIX = {{prefix}} + env: + PREFIX: ${{prefix}} + +provides: + - bin/stark + +test: + - test $(stark) = sup_bro + - | + if command which pkgx + then exit 1 + fi + - pkgx --version # however the command not found handler does work \ No newline at end of file diff --git a/projects/toolchain.com/foo.c b/projects/toolchain.com/foo.c new file mode 100644 index 00000000..50820691 --- /dev/null +++ b/projects/toolchain.com/foo.c @@ -0,0 +1,9 @@ +#include +#include "zlib.h" + +int main() { + z_stream strm = {0}; + inflateInit(&strm); + printf("sup_bro\n"); + return 0; +} diff --git a/projects/toolchain.com/package.yml b/projects/toolchain.com/package.yml new file mode 100644 index 00000000..2007f864 --- /dev/null +++ b/projects/toolchain.com/package.yml @@ -0,0 +1,17 @@ +distributable: + url: https://github.com/pkgxdev/brewkit/archive/refs/tags/v{{ version }}.tar.gz + strip-components: 1 + +versions: + github: pkgxdev/brewkit + +dependencies: + zlib.net: ^1.1 + +build: + - cc props/foo.c -lz + - install -D a.out {{prefix}}/bin/foo + +test: | + test $(foo) = sup_bro + test -z "$SRCROOT" diff --git a/projects/unavailable.com/package.yml b/projects/unavailable.com/package.yml new file mode 100644 index 00000000..c72b2625 --- /dev/null +++ b/projects/unavailable.com/package.yml @@ -0,0 +1,5 @@ +platforms: + - darwin + +versions: + - 1.0.0 \ No newline at end of file diff --git a/share/.DS_Store b/share/.DS_Store deleted file mode 100644 index 2d42a62b9271957c68be709b5bed2105a88e09c2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKUrPc(5T8}cJ0$R-pxzdG6;cZ;dO6I#Ko|8;VRtEcblr-c`5?l**B9!G^m&@u zT_e$;r-+=9+28E^xn+MRJGTHJnxjq)pbP*GDq+sWVTn+jbU|{~Ln!n!?x6<(M398z zg=}{GMF!~IxsZawUkqQ~->@IXD#Ykp*pH)OTCKi|T)wcjUUZ61$+`7=YV4;$IvO^E z!8Oe;l#0Wd9fX%rcib*-o~k4bqNF>}1yQGqDK}S9(oy548YP{fuJuj8aUHi^-kD75 z)keJ{5BCqI6*)PsS1Yn{RGUs+XKQ!w8;&FaylMiZEc0Kc}=J`{ivi1I)nB8KCn)q7r%*GlTl*z=1v=DPABXL7VOp zgwmpCF*Aq}6k$pcO{uU~3}MRAFKwP@F*9h&LFkq7Id)}XZzw{qj((}rL3jpvWCoal zc?R-kTBiPg_Wk{TzKCbc05kBf7!ZY)-)iEPY;RrI9Q9g>dWT9vd6~h_5;Sxx#$0N} c+o)R5FUdgkEM^ALgTfyIng$-2fnR0d1Nq5Ki~s-t diff --git a/share/brewkit/install b/share/brewkit/install new file mode 100755 index 00000000..d0829ce3 --- /dev/null +++ b/share/brewkit/install @@ -0,0 +1,11 @@ +#!/bin/sh + +if [ -x /usr/local/bin/pkgx ]; then + # removed from PATH deliberately + pkgx=/usr/local/bin/pkgx +else + # probs this is running in pkgx CI/CD + pkgx="${PKGX_DIR:-$HOME/.pkgx}/pkgx.sh/v*/bin/pkgx" +fi + +exec "$pkgx" +gnu.org/coreutils -- install "$@" diff --git a/share/brewkit/make b/share/brewkit/make index ddd05038..e83c9efc 100755 --- a/share/brewkit/make +++ b/share/brewkit/make @@ -10,4 +10,4 @@ else pkgx="${PKGX_DIR:-$HOME/.pkgx}/pkgx.sh/v*/bin/pkgx" fi -exec "$pkgx" +gnu.org/make make "$@" +exec "$pkgx" +gnu.org/make -- make "$@" diff --git a/share/brewkit/patch b/share/brewkit/patch index 534cd17c..efb6a939 100755 --- a/share/brewkit/patch +++ b/share/brewkit/patch @@ -8,4 +8,4 @@ else pkgx="${PKGX_DIR:-$HOME/.pkgx}/pkgx.sh/v*/bin/pkgx" fi -exec "$pkgx" +gnu.org/patch patch "$@" +exec "$pkgx" +gnu.org/patch -- patch "$@" diff --git a/share/brewkit/pkgx b/share/brewkit/pkgx new file mode 100755 index 00000000..6c91be8c --- /dev/null +++ b/share/brewkit/pkgx @@ -0,0 +1,11 @@ +#!/bin/sh + +if [ -x /usr/local/bin/pkgx ]; then + # removed from PATH deliberately + pkgx=/usr/local/bin/pkgx +else + # probs this is running in pkgx CI/CD + pkgx="${PKGX_DIR:-$HOME/.pkgx}/pkgx.sh/v*/bin/pkgx" +fi + +exec "$pkgx" "$@" diff --git a/share/brewkit/sed b/share/brewkit/sed index 4bedb18c..23bc416d 100755 --- a/share/brewkit/sed +++ b/share/brewkit/sed @@ -8,4 +8,4 @@ else pkgx="${PKGX_DIR:-$HOME/.pkgx}/pkgx.sh/v*/bin/pkgx" fi -exec "$pkgx" +gnu.org/sed sed "$@" +exec "$pkgx" +gnu.org/sed -- sed "$@" diff --git a/share/toolchain/.DS_Store b/share/toolchain/.DS_Store deleted file mode 100644 index 24516ca813ca6490546ad7c18ea4cf8f913a8817..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKUrPc(5T8}c6$yMOsJDe)h14LEUJkP_&_z8|n1@n_=dE~}gb4RuU#KtA=V@kl zjl_bUB60?1f3x%Fmi^taa|-~XIq6mbN&p~H2{Q#8z7UF&u1Lmt2!(k@2r={_38u+# zC7T_8kpX&l5)ycV0YvcW{RNQ@F!mY@qc9m)DsLi_&24PvWnLELt^c5>{Ui7<)U`V$U?icH=9+D ztM#&~*J|^!nl);bvN~!U&gTWWvwv`M-tI?(SU+1f1%7x&wj3_t1&x)BJ^G_K((ygU z7Ilk!MrME+UtSJm4&;Z2(vo+l}QKTS>%x!U{ a8bQAz1JQGsT0{>D{|Fcwcwh#8m4SD`Jx!PZ diff --git a/share/toolchain/shim b/share/toolchain/shim index bfd89629..51ce3866 100755 --- a/share/toolchain/shim +++ b/share/toolchain/shim @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash tool=$(basename $0) @@ -14,10 +14,13 @@ if [ $(uname) != Darwin ]; then # prevent fork bombs (shouldn't be possible but who knows) export PATH="/usr/bin:/bin:/usr/sbin:/sbin" + # NOTE this slows down configure scripts a shit tonne + # 1. a fix is speeding up pkgx resolution by caching the pantry + # 2. or do this once and store the env to a file that we can then source set -a - eval "$("$pkgx" +llvm.org 2>/dev/null)" + eval "$("$pkgx" +llvm.org)" - exec "$SRCROOT/dev.pkgx.bin/$tool" "$@" + exec "$HOME/toolchain/$tool" "$@" fi case $tool in diff --git a/test/action.yml b/test/action.yml new file mode 100644 index 00000000..8b0afc3c --- /dev/null +++ b/test/action.yml @@ -0,0 +1,35 @@ +name: pkgx/brewkit/build + +inputs: + pkg: + description: > + eg. pkgx.sh@1.1 + required: true + token: + default: ${{github.token}} + required: true + +runs: + using: composite + steps: + - run: | + if ! pkgx --sync; then + echo "::error::you must use: pkgxdev/setup before using this action" + exit 1 + fi + shell: bash + + - name: fetch deno deps + shell: bash + run: | + echo "::group::fetch deno deps" + tmpdir=$(mktemp -d) + ln -s ${GITHUB_ACTION_PATH}/../bin/cmd/test $tmpdir/test.ts + pkgx deno cache $tmpdir/test.ts --config=${GITHUB_ACTION_PATH}/../deno.jsonc + echo "::endgroup::" + + - name: test + run: ${GITHUB_ACTION_PATH}/../bin/cmd/test ${{ inputs.pkg }} + shell: bash + env: + GITHUB_TOKEN: ${{inputs.token}} diff --git a/upload-build-artifact/action.yml b/upload-build-artifact/action.yml new file mode 100644 index 00000000..1ae6b07b --- /dev/null +++ b/upload-build-artifact/action.yml @@ -0,0 +1,45 @@ +name: pkgx/brewkit//upload-build-artifacts + +inputs: + pkg: + description: > + eg. pkgx.sh@1.1.1, if unspecified defaults to `$BREWKIT_PKGSPEC` which + is set by the build action (or set it yourself if you like). + + This action moves the built keg away from `$PKGX_DIR`. Sorry about that. + required: false + +runs: + using: composite + steps: + - run: | + if ! pkgx --sync; then + echo "::error::you must use: pkgxdev/setup before using this action" + exit 1 + fi + shell: bash + + - name: fetch deno deps + shell: bash + run: | + echo "::group::fetch deno deps" + pkgx deno cache ${GITHUB_ACTION_PATH}/../lib/actions/*.ts --config=${GITHUB_ACTION_PATH}/../deno.jsonc + echo "::endgroup::" + + # we must tar because the upload action loses file permissions 😕 + - name: tar + run: | + tmpdir=$(mktemp -d) + key=$(${GITHUB_ACTION_PATH}/../lib/actions/platform-key.ts) + ${GITHUB_ACTION_PATH}/../lib/actions/stage.ts $tmpdir ${{inputs.pkg}} + cd $tmpdir + tar cf $key.tar * + echo "path=$tmpdir/$key.tar" >> $GITHUB_OUTPUT + echo "name=$key" >> $GITHUB_OUTPUT + id: tar + shell: bash + + - uses: actions/upload-artifact@v3 + with: + path: ${{ steps.tar.outputs.path }} + name: ${{ steps.tar.outputs.name }} From 0c5852a2b752e481017ce71140dd3efb794a61e8 Mon Sep 17 00:00:00 2001 From: Max Howell Date: Tue, 12 Dec 2023 13:03:23 -0500 Subject: [PATCH 2/2] Allow `bk docker --pull foo` --- bin/cmd/docker | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bin/cmd/docker b/bin/cmd/docker index fc4a028d..e0540fb2 100755 --- a/bin/cmd/docker +++ b/bin/cmd/docker @@ -13,6 +13,16 @@ if [ -z "$GITHUB_TOKEN" ]; then GITHUB_TOKEN=$(pkgx gh auth token) fi +if [ "$1" = --pull ]; then + docker image pull pkgxdev/pkgx + shift +fi + +if [ -z "$1" ]; then + echo "error: missing command" >&2 + exit 64 +fi + CMD=$1 shift