Skip to content

Commit c1ad5ef

Browse files
DilumAluthgeDilumAluthge
committed
CI (Buildkite): Add package_linux and tester_linux for more Linux architectures
1 parent ed866b5 commit c1ad5ef

File tree

8 files changed

+184
-114
lines changed

8 files changed

+184
-114
lines changed

‎.buildkite/pipelines/experimental/0_webui.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,4 @@ steps:
2121
# verifies the treehash of the pipeline itself and the inputs listed in `inputs`
2222
# signed_pipelines:
2323
# - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml
24-
# signature: "my_signature"
24+
# signature_file: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml.signature

‎.buildkite/pipelines/main/launch_unsigned_builders.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ steps:
2323
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
2424
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
2525
26-
# Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order.
27-
buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml
26+
# Finally, we launch all of the platform builders.
27+
bash .buildkite/pipelines/main/platforms/platforms.sh linux
2828
agents:
2929
queue: julia

‎.buildkite/pipelines/main/platforms/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
linux_rr.yml

‎.buildkite/pipelines/main/platforms/linux.arches‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# ARCH ROOTFS_ARCH ROOTFS_PACK_TAG ROOTFS_PACK_TREE ROOTFS_TEST_TAG ROOTFS_TEST_TREE
2+
# aarch64 aarch64 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff v3.2 d38f34db72317ffa798b18b0c8e193795bb254db
3+
# armv7l armv7l v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25 v3.2 11c66f1c3a094e031fae6a9de0fa7986459466b6
4+
32 i686 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32 v3.2 b63e7f1002f093dc51390cd8f4a6368ea929560f
5+
# ppc64le powerpc64le v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7 v3.2 0b23a48bb5ed4b3226ebceea18375a2b09caf34d
6+
64 x86_64 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7 v3.2 b1b1fce0b76b2dbe41222093f2f9f3f8e226018d

‎.buildkite/pipelines/main/platforms/linux.yml‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
agents:
2+
queue: "julia"
3+
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
4+
sandbox.jl: "true"
5+
os: "linux"
6+
7+
steps:
8+
- label: "package_linux$ARCH"
9+
key: package_linux$ARCH
10+
plugins:
11+
- JuliaCI/julia#v1:
12+
version: 1.6
13+
- staticfloat/sandbox#v1:
14+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_PACK_TAG/package_linux.${ROOTFS_ARCH}.tar.gz
15+
rootfs_treehash: "$ROOTFS_PACK_TREE"
16+
uid: 1000
17+
gid: 1000
18+
workspaces:
19+
# Include `/cache/repos` so that our `git` version introspection works.
20+
- "/cache/repos:/cache/repos"
21+
commands: |
22+
echo "--- Print the full and short commit hashes"
23+
SHORT_COMMIT_LENGTH=10
24+
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
25+
JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
26+
ARTIFACT_FILE_EXTENSION="tar.gz"
27+
ARTIFACT_FILENAME="$$JULIA_BINARYDIST_FILENAME.$$ARTIFACT_FILE_EXTENSION"
28+
echo "The full commit is: $$BUILDKITE_COMMIT"
29+
echo "The short commit is: $$SHORT_COMMIT"
30+
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
31+
32+
echo "--- Build Julia from source"
33+
rm -rf $$ARTIFACT_FILENAME
34+
make -j 8
35+
36+
echo "--- Make sure that the working directory is clean"
37+
if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi
38+
39+
echo "--- Print Julia version info"
40+
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
41+
42+
echo "--- Create build artifacts"
43+
make -j 8 binary-dist
44+
ls -l $$ARTIFACT_FILENAME
45+
46+
echo "--- Upload build artifacts"
47+
buildkite-agent artifact upload $$ARTIFACT_FILENAME
48+
timeout_in_minutes: 60
49+
notify:
50+
- github_commit_status:
51+
context: "package_linux$ARCH"
52+
53+
- label: "tester_linux$ARCH"
54+
key: tester_linux$ARCH
55+
depends_on: package_linux$ARCH
56+
if: false # TODO: delete this line
57+
plugins:
58+
- JuliaCI/julia#v1:
59+
version: 1.6
60+
- staticfloat/sandbox#v1:
61+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_TEST_TREE/package_linux.${ROOTFS_ARCH}.tar.gz
62+
rootfs_treehash: "$ROOTFS_TEST_TREE"
63+
uid: 1000
64+
gid: 1000
65+
workspaces:
66+
# Include `/cache/repos` so that our `git` version introspection works.
67+
- "/cache/repos:/cache/repos"
68+
env:
69+
JULIA_SHELL: "/bin/bash"
70+
commands: |
71+
echo "--- Print the full and short commit hashes"
72+
SHORT_COMMIT_LENGTH=10
73+
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
74+
JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT"
75+
ARTIFACT_FILE_EXTENSION="tar.gz"
76+
ARTIFACT_FILENAME="julia-$$SHORT_COMMIT-linux$ARCH.$$ARTIFACT_FILE_EXTENSION"
77+
echo "The full commit is: $$BUILDKITE_COMMIT"
78+
echo "The short commit is: $$SHORT_COMMIT"
79+
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
80+
echo "The Julia directory name will be: $$JULIA_DIRECTORY_NAME"
81+
82+
echo "--- Download build artifacts"
83+
rm -rf $$ARTIFACT_FILENAME
84+
buildkite-agent artifact download $$ARTIFACT_FILENAME .
85+
86+
echo "--- Extract build artifacts"
87+
rm -rf $$JULIA_DIRECTORY_NAME/
88+
tar xzf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/
89+
90+
echo "--- Print Julia version info"
91+
$$JULIA_DIRECTORY_NAME/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
92+
93+
echo "--- Run the Julia test suite"
94+
unset JULIA_DEPOT_PATH
95+
$$JULIA_DIRECTORY_NAME/bin/julia .buildkite/utilities/rr/rr_capture.jl $$JULIA_DIRECTORY_NAME/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
96+
timeout_in_minutes: 150
97+
notify:
98+
- github_commit_status:
99+
context: "tester_linux$ARCH"
100+
101+
- label: "tester2_linux$ARCH"
102+
key: tester2_linux$ARCH
103+
depends_on: package_linux$ARCH
104+
plugins:
105+
- JuliaCI/julia#v1:
106+
version: 1.6
107+
- staticfloat/sandbox#v1:
108+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_PACK_TAG/package_linux.${ROOTFS_ARCH}.tar.gz
109+
rootfs_treehash: "$ROOTFS_PACK_TREE"
110+
uid: 1000
111+
gid: 1000
112+
workspaces:
113+
# Include `/cache/repos` so that our `git` version introspection works.
114+
- "/cache/repos:/cache/repos"
115+
env:
116+
JULIA_SHELL: "/bin/bash"
117+
commands: |
118+
echo "--- Print the full and short commit hashes"
119+
SHORT_COMMIT_LENGTH=10
120+
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
121+
JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT"
122+
ARTIFACT_FILE_EXTENSION="tar.gz"
123+
ARTIFACT_FILENAME="julia-$$SHORT_COMMIT-linux$ARCH.$$ARTIFACT_FILE_EXTENSION"
124+
echo "The full commit is: $$BUILDKITE_COMMIT"
125+
echo "The short commit is: $$SHORT_COMMIT"
126+
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
127+
echo "The Julia directory name will be: $$JULIA_DIRECTORY_NAME"
128+
129+
echo "--- Download build artifacts"
130+
rm -rf $$ARTIFACT_FILENAME
131+
buildkite-agent artifact download $$ARTIFACT_FILENAME .
132+
133+
echo "--- Extract build artifacts"
134+
rm -rf $$JULIA_DIRECTORY_NAME/
135+
tar xzf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/
136+
137+
echo "--- Print Julia version info"
138+
$$JULIA_DIRECTORY_NAME/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
139+
140+
echo "--- Run the Julia test suite"
141+
unset JULIA_DEPOT_PATH
142+
$$JULIA_DIRECTORY_NAME/bin/julia .buildkite/utilities/rr/rr_capture.jl $$JULIA_DIRECTORY_NAME/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
143+
timeout_in_minutes: 150
144+
notify:
145+
- github_commit_status:
146+
context: "tester2_linux$ARCH"

‎.buildkite/pipelines/main/platforms/linux64.yml‎

Lines changed: 0 additions & 93 deletions
This file was deleted.

‎.buildkite/pipelines/main/platforms/platforms.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
4+
5+
PLATFORM="$1"
6+
7+
cat "$SCRIPT_DIR/$PLATFORM.arches" | tr -s ' ' | while read line; do
8+
if [[ $line == \#* ]]; then
9+
continue
10+
fi
11+
export ARCH=`echo $line | cut -d ' ' -f 1`
12+
export ROOTFS_ARCH=`echo $line | cut -d ' ' -f 2`
13+
export ROOTFS_PACK_TAG=`echo $line | cut -d ' ' -f 3`
14+
export ROOTFS_PACK_TREE=`echo $line | cut -d ' ' -f 4`
15+
export ROOTFS_TEST_TAG=`echo $line | cut -d ' ' -f 5`
16+
export ROOTFS_TEST_TREE=`echo $line | cut -d ' ' -f 6`
17+
echo "Launching: PLATFORM=$PLATFORM ARCH=$ARCH ROOTFS_ARCH=$ROOTFS_ARCH"
18+
buildkite-agent pipeline upload "$SCRIPT_DIR/$PLATFORM.yml"
19+
done

‎.buildkite/utilities/rr/rr_capture.jl‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,35 +5,26 @@ using Tar
55
if Base.VERSION < v"1.6"
66
throw(ErrorException("The `rr_capture.jl` script requires Julia 1.6 or greater"))
77
end
8-
98
if length(ARGS) < 1
109
throw(ErrorException("Usage: rr_capture.jl [command...]"))
1110
end
1211

13-
const TIMEOUT = 2 * 60 * 60 # timeout in seconds
14-
15-
# We only use `rr` on the `tester_linux64` builder
16-
const use_rr_if_builder_is = "tester_linux64"
17-
18-
const run_id = get(ENV, "BUILDKITE_JOB_ID", "unknown")
19-
const shortcommit = get(ENV, "BUILDKITE_COMMIT", "unknown")
20-
const builder = get(ENV, "BUILDKITE_STEP_KEY", use_rr_if_builder_is)
21-
const use_rr = builder == use_rr_if_builder_is
22-
23-
@info "" run_id shortcommit builder use_rr
24-
@info "" ARGS
25-
26-
# if !use_rr # TODO: uncomment this line
27-
if true # TODO: delete this line
28-
@info "We will not run the tests under rr"
12+
# We only use `rr` on certain builders
13+
const rr_builder_list = ["tester_linux64", "tester2_linux64"]
14+
const this_builder = ENV["BUILDKITE_STEP_KEY"]
15+
if !(this_builder in rr_builder_list)
16+
@info "We will not run the tests under rr" this_builder rr_builder_list
2917
p = run(`$ARGS`)
3018
exit(p.exitcode)
3119
end
3220

3321
@info "We will run the tests under rr"
3422

23+
const MINUTE = 60 # number of seconds in one minute
24+
const TIMEOUT = 120 * MINUTE
25+
const run_id = ENV["BUILDKITE_JOB_ID"]
26+
const shortcommit = ENV["BUILDKITE_COMMIT"]
3527
const num_cores = min(Sys.CPU_THREADS, 8, parse(Int, get(ENV, "JULIA_TEST_NUM_CORES", "8")) + 1)
36-
@info "" num_cores
3728

3829
proc = nothing
3930

0 commit comments

Comments
 (0)