Skip to content

Commit f5e4586

Browse files
tenzen-ytenzen-y
authored
Add the CI to build multi-platform container images (#1956)
Signed-off-by: tenzen-y <[email protected]> Signed-off-by: tenzen-y <[email protected]>
1 parent e02eb6e commit f5e4586

File tree

8 files changed

+129
-78
lines changed

8 files changed

+129
-78
lines changed

‎.github/workflows/build-and-publish-images.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Reusable workflows for publishing Katib images.
2+
name: Build And Publish Images
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
component-name:
8+
required: true
9+
type: string
10+
platforms:
11+
required: true
12+
type: string
13+
dockerfile:
14+
required: true
15+
type: string
16+
secrets:
17+
DOCKERHUB_USERNAME:
18+
required: false
19+
DOCKERHUB_TOKEN:
20+
required: false
21+
22+
jobs:
23+
build-and-publish:
24+
name: Publish Image
25+
runs-on: ubuntu-latest
26+
steps:
27+
- name: Checkout
28+
uses: actions/checkout@v2
29+
30+
- name: Docker Login
31+
# Trigger workflow only for kubeflow/katib repository with specific branch (master, release-.*) or tag (v.*).
32+
if: >-
33+
github.repository == 'kubeflow/katib' &&
34+
(github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
35+
uses: docker/login-action@v2
36+
with:
37+
username: ${{ secrets.DOCKERHUB_USERNAME }}
38+
password: ${{ secrets.DOCKERHUB_TOKEN }}
39+
40+
- name: Publish Component ${{ inputs.component-name }}
41+
# Trigger workflow only for kubeflow/katib repository with specific branch (master, release-.*) or tag (v.*).
42+
if: >-
43+
github.repository == 'kubeflow/katib' &&
44+
(github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
45+
id: publish
46+
uses: ./.github/workflows/template-publish-image
47+
with:
48+
image: docker.io/kubeflowkatib/${{ inputs.component-name }}
49+
dockerfile: ${{ inputs.dockerfile }}
50+
platforms: ${{ inputs.platforms }}
51+
push: true
52+
53+
- name: Test Build For Component ${{ inputs.component-name }}
54+
if: steps.publish.outcome == 'skipped'
55+
uses: ./.github/workflows/template-publish-image
56+
with:
57+
image: docker.io/kubeflowkatib/${{ inputs.component-name }}
58+
dockerfile: ${{ inputs.dockerfile }}
59+
platforms: ${{ inputs.platforms }}
60+
push: false

‎.github/workflows/lint.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,3 @@ jobs:
2020

2121
- name: Check YAML
2222
run: make yamllint
23-

‎.github/workflows/publish-algorithm-images.yaml‎

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,20 @@
11
name: Publish AutoML Algorithm Images
22

33
on:
4-
push:
5-
branches:
6-
- master
7-
8-
env:
9-
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
10-
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
4+
- push
5+
- pull_request
116

127
jobs:
138
algorithm:
149
name: Publish Image
15-
# Trigger workflow only for kubeflow/katib repository.
16-
if: github.repository == 'kubeflow/katib'
17-
runs-on: ubuntu-latest
18-
steps:
19-
- name: Checkout
20-
uses: actions/checkout@v2
21-
22-
- name: Publish Component ${{ matrix.component-name }}
23-
uses: ./.github/workflows/template-publish-image
24-
with:
25-
image: docker.io/kubeflowkatib/${{ matrix.component-name }}
26-
dockerfile: ${{ matrix.dockerfile }}
10+
uses: ./.github/workflows/build-and-publish-images.yaml
11+
with:
12+
component-name: ${{ matrix.component-name }}
13+
platforms: linux/amd64,linux/arm64
14+
dockerfile: ${{ matrix.dockerfile }}
15+
secrets:
16+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
17+
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
2718

2819
strategy:
2920
fail-fast: false

‎.github/workflows/publish-core-images.yaml‎

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,21 @@
11
name: Publish Katib Core Images
22

33
on:
4-
push:
5-
branches:
6-
- master
7-
8-
env:
9-
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
10-
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
4+
- push
5+
- pull_request
116

127
jobs:
138
core:
149
name: Publish Image
15-
# Trigger workflow only for kubeflow/katib repository.
16-
if: github.repository == 'kubeflow/katib'
17-
runs-on: ubuntu-latest
18-
steps:
19-
- name: Checkout
20-
uses: actions/checkout@v2
10+
uses: ./.github/workflows/build-and-publish-images.yaml
11+
with:
12+
component-name: ${{ matrix.component-name }}
13+
platforms: linux/amd64,linux/arm64
14+
dockerfile: ${{ matrix.dockerfile }}
15+
secrets:
16+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
17+
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
2118

22-
- name: Publish Component ${{ matrix.component-name }}
23-
uses: ./.github/workflows/template-publish-image
24-
with:
25-
image: docker.io/kubeflowkatib/${{ matrix.component-name }}
26-
dockerfile: ${{ matrix.dockerfile }}
2719

2820
strategy:
2921
fail-fast: false

‎.github/workflows/publish-trial-images.yaml‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,49 @@
11
name: Publish Trial Images
22

33
on:
4-
push:
5-
branches:
6-
- master
7-
8-
env:
9-
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
10-
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
4+
- push
5+
- pull_request
116

127
jobs:
138
trial:
149
name: Publish Image
15-
# Trigger workflow only for kubeflow/katib repository.
16-
if: github.repository == 'kubeflow/katib'
17-
runs-on: ubuntu-latest
18-
steps:
19-
- name: Checkout
20-
uses: actions/checkout@v2
21-
22-
- name: Publish Trial ${{ matrix.trial-name }}
23-
uses: ./.github/workflows/template-publish-image
24-
with:
25-
image: docker.io/kubeflowkatib/${{ matrix.trial-name }}
26-
dockerfile: ${{ matrix.dockerfile }}
10+
uses: ./.github/workflows/build-and-publish-images.yaml
11+
with:
12+
component-name: ${{ matrix.trial-name }}
13+
platforms: ${{ matrix.platforms }}
14+
dockerfile: ${{ matrix.dockerfile }}
15+
secrets:
16+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
17+
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
2718

2819
strategy:
2920
fail-fast: false
3021
matrix:
3122
include:
3223
- trial-name: mxnet-mnist
24+
platforms: linux/amd64,linux/arm64
3325
dockerfile: examples/v1beta1/trial-images/mxnet-mnist/Dockerfile
3426
- trial-name: pytorch-mnist-cpu
27+
platforms: linux/amd64,linux/arm64
3528
dockerfile: examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu
3629
- trial-name: pytorch-mnist-gpu
30+
platforms: linux/amd64
3731
dockerfile: examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.gpu
3832
- trial-name: tf-mnist-with-summaries
33+
platforms: linux/amd64,linux/arm64
3934
dockerfile: examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile
4035
- trial-name: enas-cnn-cifar10-gpu
36+
platforms: linux/amd64
4137
dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu
4238
- trial-name: enas-cnn-cifar10-cpu
39+
platforms: linux/amd64,linux/arm64
4340
dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu
4441
- trial-name: darts-cnn-cifar10-cpu
42+
platforms: linux/amd64,linux/arm64
4543
dockerfile: examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu
4644
- trial-name: darts-cnn-cifar10-gpu
45+
platforms: linux/amd64
4746
dockerfile: examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.gpu
4847
- trial-name: simple-pbt
48+
platforms: linux/amd64,linux/arm64
4949
dockerfile: examples/v1beta1/trial-images/simple-pbt/Dockerfile

‎.github/workflows/template-e2e-test/action.yaml‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,25 @@
1-
# Template for e2e tests.
1+
# Composite action for e2e tests.
2+
name: Run E2E Test
3+
description: Run e2e test using the minikube cluster
24

35
inputs:
46
experiments:
57
required: true
6-
type: string
8+
description: comma delimited experiment name
79
training-operator:
810
required: false
9-
type: boolean
10-
default: false
11+
description: whether to deploy training-operator or not
12+
default: "false"
1113
trial-images:
1214
required: true
13-
type: string
15+
description: comma delimited trial image name
1416
katib-ui:
1517
required: true
16-
type: boolean
17-
default: false
18+
description: whether to deploy katib-ui or not
19+
default: "false"
1820
database-type:
1921
required: false
20-
type: string
22+
description: mysql or postgres
2123
default: mysql
2224

2325
runs:

‎.github/workflows/template-publish-image/action.yaml‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,30 @@
1-
# Template run for publishing Katib images.
1+
# Composite action for publishing Katib images.
2+
name: Build And Publish Container Images
3+
description: Build MultiPlatform Supporting Container Images
24

35
inputs:
46
image:
57
required: true
6-
type: string
8+
description: image tag
79
dockerfile:
810
required: true
9-
type: string
11+
description: path for dockerfile
12+
platforms:
13+
required: true
14+
description: linux/amd64 or linux/amd64,linux/arm64
15+
push:
16+
required: true
17+
description: whether to push container images or not
1018

1119
runs:
1220
using: composite
1321
steps:
22+
- name: Set up QEMU
23+
uses: docker/setup-qemu-action@v2
24+
1425
- name: Set Up Docker Buildx
1526
uses: docker/setup-buildx-action@v2
1627

17-
- name: Docker Login
18-
uses: docker/login-action@v1
19-
with:
20-
username: ${{ env.DOCKERHUB_USERNAME }}
21-
password: ${{ env.DOCKERHUB_TOKEN }}
22-
2328
- name: Add Docker Tags
2429
id: meta
2530
uses: docker/metadata-action@v3
@@ -34,8 +39,8 @@ runs:
3439
with:
3540
context: .
3641
file: ${{ inputs.dockerfile }}
37-
push: true
42+
push: ${{ inputs.push }}
3843
tags: ${{ steps.meta.outputs.tags }}
3944
cache-from: type=gha
4045
cache-to: type=gha,mode=max
41-
platforms: linux/amd64
46+
platforms: ${{ inputs.platforms }}

‎.github/workflows/template-setup-e2e-test/action.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1-
# Template for e2e tests.
1+
# Composite action to setup e2e tests.
2+
name: Setup E2E Test
3+
description: setup env for e2e test using the minikube cluster
24

35
inputs:
46
kubernetes-version:
57
required: true
6-
type: string
8+
description: kubernetes version
79

810
runs:
911
using: composite

0 commit comments

Comments
 (0)