Skip to content

Commit 9e73d3c

Browse files
DilumAluthgeDilumAluthge
committed
CI (Buildkite): Add package_linux and tester_linux for more Linux architectures
1 parent f7f46af commit 9e73d3c

File tree

6 files changed

+207
-116
lines changed

6 files changed

+207
-116
lines changed

‎.buildkite/pipelines/main/launch_unsigned_builders.yml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,18 @@ steps:
2323
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
2424
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
2525
26-
# Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order.
27-
buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml
26+
# Now, we launch the `linux64` platform builders.
27+
rm -rf .buildkite/pipelines/main/platforms/linux_rr.yml
28+
cat .buildkite/pipelines/main/platforms/linux.yml >> .buildkite/pipelines/main/platforms/linux_rr.yml
29+
cat .buildkite/pipelines/main/platforms/rr.yml >> .buildkite/pipelines/main/platforms/linux_rr.yml
30+
ARCH=64 ROOTFS_ARCH=x86_64 ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=474bf61a926b2d7fcf202284d59d4b11a04601d7 ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=b1b1fce0b76b2dbe41222093f2f9f3f8e226018d buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux_rr.yml
31+
32+
# Finally, we launch all of the remaining the platform builders.
33+
# ARCH=aarch64 ROOTFS_ARCH=aarch64 ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=0566841e29f0f9880541c26a6595fd5ce0beb5ff ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=d38f34db72317ffa798b18b0c8e193795bb254db buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux.yml
34+
# ARCH=armv7l ROOTFS_ARCH=armv7l ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=fb359370b052a47ce5c84cc6b4a7a03ed7053b25 ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=11c66f1c3a094e031fae6a9de0fa7986459466b6 buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux.yml
35+
ARCH=32 ROOTFS_ARCH=i686 ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=209c4db679a515befd7fb50ecc6bfbecf7ec3d32 ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=b63e7f1002f093dc51390cd8f4a6368ea929560f buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux.yml
36+
# ARCH=ppc64le ROOTFS_ARCH=powerpc64le ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=c03a0158b19d48ac84b426834fce0d3584cdd0c7 ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=0b23a48bb5ed4b3226ebceea18375a2b09caf34d buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux.yml
37+
38+
2839
agents:
2940
queue: julia

‎.buildkite/pipelines/main/platforms/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
linux_rr.yml

‎.buildkite/pipelines/main/platforms/linux.yml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
agents:
2+
queue: "julia"
3+
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
4+
sandbox.jl: "true"
5+
os: "linux"
6+
7+
steps:
8+
- label: "package_linux$ARCH"
9+
key: package_linux$ARCH
10+
plugins:
11+
- JuliaCI/julia#v1:
12+
version: 1.6
13+
- staticfloat/sandbox#v1:
14+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_TAG_PACK/package_linux.${ROOTFS_ARCH}.tar.gz
15+
rootfs_treehash: "$ROOTFS_TREE_PACK"
16+
uid: 1000
17+
gid: 1000
18+
workspaces:
19+
# Include `/cache/repos` so that our `git` version introspection works.
20+
- "/cache/repos:/cache/repos"
21+
commands: |
22+
echo "--- Print the full and short commit hashes"
23+
SHORT_COMMIT_LENGTH=10
24+
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
25+
JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
26+
ARTIFACT_FILE_EXTENSION="tar.gz"
27+
ARTIFACT_FILENAME="$$JULIA_BINARYDIST_FILENAME.$$ARTIFACT_FILE_EXTENSION"
28+
echo "The full commit is: $$BUILDKITE_COMMIT"
29+
echo "The short commit is: $$SHORT_COMMIT"
30+
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
31+
32+
echo "--- Build Julia from source"
33+
rm -rf $$ARTIFACT_FILENAME
34+
make -j 8
35+
36+
echo "--- Print Julia version info"
37+
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
38+
39+
echo "--- Create build artifacts"
40+
make -j 8 binary-dist
41+
ls -l $$ARTIFACT_FILENAME
42+
43+
echo "--- Upload build artifacts"
44+
buildkite-agent artifact upload $$ARTIFACT_FILENAME
45+
timeout_in_minutes: 60
46+
notify:
47+
- github_commit_status:
48+
context: "package_linux$ARCH"
49+
50+
- label: "tester_linux$ARCH"
51+
key: tester_linux$ARCH
52+
depends_on: package_linux$ARCH
53+
plugins:
54+
- JuliaCI/julia#v1:
55+
version: 1.6
56+
- staticfloat/sandbox#v1:
57+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_TAG_TEST/tester_linux.${ROOTFS_ARCH}.tar.gz
58+
rootfs_treehash: "$ROOTFS_TREE_TEST"
59+
uid: 1000
60+
gid: 1000
61+
workspaces:
62+
# Include `/cache/repos` so that our `git` version introspection works.
63+
- "/cache/repos:/cache/repos"
64+
env:
65+
JULIA_SHELL: "/bin/bash"
66+
commands: |
67+
echo "--- Print the full and short commit hashes"
68+
SHORT_COMMIT_LENGTH=10
69+
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
70+
JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT"
71+
ARTIFACT_FILE_EXTENSION="tar.gz"
72+
ARTIFACT_FILENAME="julia-$$SHORT_COMMIT-linux$ARCH.$$ARTIFACT_FILE_EXTENSION"
73+
echo "The full commit is: $$BUILDKITE_COMMIT"
74+
echo "The short commit is: $$SHORT_COMMIT"
75+
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
76+
echo "The Julia directory name will be: $$JULIA_DIRECTORY_NAME"
77+
78+
echo "--- Download build artifacts"
79+
rm -rf $$ARTIFACT_FILENAME
80+
buildkite-agent artifact download $$ARTIFACT_FILENAME .
81+
82+
echo "--- Extract build artifacts"
83+
rm -rf $$JULIA_DIRECTORY_NAME/
84+
tar xzf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/
85+
86+
echo "--- Print Julia version info"
87+
$$JULIA_DIRECTORY_NAME/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
88+
89+
echo "--- Run the Julia test suite"
90+
unset JULIA_DEPOT_PATH
91+
$$JULIA_DIRECTORY_NAME/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
92+
timeout_in_minutes: 120
93+
notify:
94+
- github_commit_status:
95+
context: "tester_linux$ARCH"
96+
97+
- label: "tester2_linux$ARCH"
98+
key: tester2_linux$ARCH
99+
depends_on: package_linux$ARCH
100+
plugins:
101+
- JuliaCI/julia#v1:
102+
version: 1.6
103+
- staticfloat/sandbox#v1:
104+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_TAG_PACK/package_linux.${ROOTFS_ARCH}.tar.gz
105+
rootfs_treehash: "$ROOTFS_TREE_PACK"
106+
uid: 1000
107+
gid: 1000
108+
workspaces:
109+
# Include `/cache/repos` so that our `git` version introspection works.
110+
- "/cache/repos:/cache/repos"
111+
env:
112+
JULIA_SHELL: "/bin/bash"
113+
commands: |
114+
echo "--- Print the full and short commit hashes"
115+
SHORT_COMMIT_LENGTH=10
116+
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
117+
JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT"
118+
ARTIFACT_FILE_EXTENSION="tar.gz"
119+
ARTIFACT_FILENAME="julia-$$SHORT_COMMIT-linux$ARCH.$$ARTIFACT_FILE_EXTENSION"
120+
echo "The full commit is: $$BUILDKITE_COMMIT"
121+
echo "The short commit is: $$SHORT_COMMIT"
122+
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
123+
echo "The Julia directory name will be: $$JULIA_DIRECTORY_NAME"
124+
125+
echo "--- Download build artifacts"
126+
rm -rf $$ARTIFACT_FILENAME
127+
buildkite-agent artifact download $$ARTIFACT_FILENAME .
128+
129+
echo "--- Extract build artifacts"
130+
rm -rf $$JULIA_DIRECTORY_NAME/
131+
tar xzf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/
132+
133+
echo "--- Print Julia version info"
134+
$$JULIA_DIRECTORY_NAME/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
135+
136+
echo "--- Run the Julia test suite"
137+
unset JULIA_DEPOT_PATH
138+
$$JULIA_DIRECTORY_NAME/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
139+
timeout_in_minutes: 120
140+
notify:
141+
- github_commit_status:
142+
context: "tester2_linux$ARCH"

‎.buildkite/pipelines/main/platforms/linux64.yml‎

Lines changed: 0 additions & 90 deletions
This file was deleted.

‎.buildkite/pipelines/main/platforms/rr.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
2+
- label: "tester2_rr_linux$ARCH"
3+
key: tester2_rr_linux$ARCH
4+
depends_on: package_linux$ARCH
5+
plugins:
6+
- JuliaCI/julia#v1:
7+
version: 1.6
8+
- staticfloat/sandbox#v1:
9+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_TAG_PACK/package_linux.${ROOTFS_ARCH}.tar.gz
10+
rootfs_treehash: "$ROOTFS_TREE_PACK"
11+
uid: 1000
12+
gid: 1000
13+
workspaces:
14+
# Include `/cache/repos` so that our `git` version introspection works.
15+
- "/cache/repos:/cache/repos"
16+
env:
17+
JULIA_SHELL: "/bin/bash"
18+
commands: |
19+
echo "--- Print the full and short commit hashes"
20+
SHORT_COMMIT_LENGTH=10
21+
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
22+
JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT"
23+
ARTIFACT_FILE_EXTENSION="tar.gz"
24+
ARTIFACT_FILENAME="julia-$$SHORT_COMMIT-linux$ARCH.$$ARTIFACT_FILE_EXTENSION"
25+
echo "The full commit is: $$BUILDKITE_COMMIT"
26+
echo "The short commit is: $$SHORT_COMMIT"
27+
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
28+
echo "The Julia directory name will be: $$JULIA_DIRECTORY_NAME"
29+
30+
echo "--- Download build artifacts"
31+
rm -rf $$ARTIFACT_FILENAME
32+
buildkite-agent artifact download $$ARTIFACT_FILENAME .
33+
34+
echo "--- Extract build artifacts"
35+
rm -rf $$JULIA_DIRECTORY_NAME/
36+
tar xzf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/
37+
38+
echo "--- Print Julia version info"
39+
$$JULIA_DIRECTORY_NAME/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
40+
41+
echo "--- Run the Julia test suite inside rr"
42+
unset JULIA_DEPOT_PATH
43+
$$JULIA_DIRECTORY_NAME/bin/julia .buildkite/utilities/rr/rr_capture.jl 120 $$JULIA_DIRECTORY_NAME/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
44+
timeout_in_minutes: 150
45+
notify:
46+
- github_commit_status:
47+
context: "tester2_rr_linux$ARCH"

‎.buildkite/utilities/rr/rr_capture.jl‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,11 @@ if Base.VERSION < v"1.6"
66
throw(ErrorException("The `rr_capture.jl` script requires Julia 1.6 or greater"))
77
end
88

9-
if length(ARGS) < 1
10-
throw(ErrorException("Usage: rr_capture.jl [command...]"))
9+
if length(ARGS) < 2
10+
throw(ErrorException("Usage: rr_capture.jl <timeout_in_minutes> <command...>"))
1111
end
1212

13-
const TIMEOUT = 2 * 60 * 60 # timeout in seconds
14-
15-
# We only use `rr` on the `tester_linux64` builder
16-
const use_rr_if_builder_is = "tester_linux64"
17-
18-
const run_id = get(ENV, "BUILDKITE_JOB_ID", "unknown")
19-
const shortcommit = get(ENV, "BUILDKITE_COMMIT", "unknown")
20-
const builder = get(ENV, "BUILDKITE_STEP_KEY", use_rr_if_builder_is)
21-
const use_rr = builder == use_rr_if_builder_is
22-
23-
@info "" run_id shortcommit builder use_rr
24-
@info "" ARGS
25-
26-
# if !use_rr # TODO: uncomment this line
27-
if true # TODO: delete this line
28-
@info "We will not run the tests under rr"
29-
p = run(`$ARGS`)
30-
exit(p.exitcode)
31-
end
32-
33-
@info "We will run the tests under rr"
13+
const TIMEOUT_IN_MINUTES = parse(Int, popfirst!(ARGS))::Int
3414

3515
const num_cores = min(Sys.CPU_THREADS, 8, parse(Int, get(ENV, "JULIA_TEST_NUM_CORES", "8")) + 1)
3616
@info "" num_cores
@@ -68,7 +48,7 @@ mktempdir() do dir
6848

6949
# Start asynchronous timer that will kill `rr`
7050
@async begin
71-
sleep(TIMEOUT)
51+
sleep(60 * TIMEOUT_IN_MINUTES)
7252

7353
# If we've exceeded the timeout and `rr` is still running, kill it.
7454
if isopen(proc)

0 commit comments

Comments
 (0)