Sgx runner

.github/workflows/build_and_test.yml

@@ -21,27 +21,35 @@ env:
 jobs:
   cancel_previous_runs:
     name: Cancel Previous Runs
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: styfle/cancel-workflow-action@0.11.0
         with:
           access_token: ${{ secrets.GITHUB_TOKEN }}
 
   build-test:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.host }}
     strategy:
       fail-fast: false
       matrix:
         include:
           - flavor_id: sidechain
             mode: sidechain
+            host: integritee-builder-sgx
+            sgx_mode: HW
           - flavor_id: offchain-worker
             mode: offchain-worker
+            host: integritee-builder-sgx
+            sgx_mode: HW
           - flavor_id: teeracle
             mode: teeracle
+            host: integritee-builder-sgx
+            sgx_mode: HW
           - flavor_id: sidechain-evm
             mode: sidechain
             additional_features: evm
+            host: integritee-builder-sgx
+            sgx_mode: HW
 
     steps:
       - uses: actions/checkout@v3
@@ -58,7 +66,7 @@ jobs:
         run: >
           docker build -t integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }}
           --target deployed-worker
-          --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }}
+          --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }}
           -f build.Dockerfile .
 
       - name: Build CLI client
@@ -73,7 +81,7 @@ jobs:
       - run: docker images --all
 
       - name: Test Enclave # cargo test is not supported in the enclave, see: https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232
-        run: docker run --name ${{ env.BUILD_CONTAINER_NAME }} integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} test --all
+        run: docker run --device=/dev/sgx/enclave --device=/dev/sgx/provision -v /var/run/aesmd:/var/run/aesmd integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} test --all
 
       - name: Export worker image(s)
         run: |
@@ -94,7 +102,7 @@ jobs:
 
   clippy:
     runs-on: ubuntu-22.04
-    container: "integritee/integritee-dev:0.1.13"
+    container: "integritee/integritee-dev:0.2.1"
     steps:
       - uses: actions/checkout@v3
       - name: init rust
@@ -152,7 +160,7 @@ jobs:
         uses: andymckay/cancel-action@0.3
 
   integration-tests:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.host }}
     if: ${{ always() }}
     needs: build-test
     env:
@@ -168,34 +176,60 @@ jobs:
           - test: M6
             flavor_id: sidechain
             demo_name: demo-indirect-invocation
+            host: test-runner-sgx
+            sgx_mode: HW
           - test: M8
             flavor_id: sidechain
             demo_name: demo-direct-call
+            host: test-runner-sgx
+            sgx_mode: HW
           - test: Sidechain
             flavor_id: sidechain
             demo_name: demo-sidechain
+            host: test-runner-sgx
+            sgx_mode: HW
           - test: M6
             flavor_id: offchain-worker
             demo_name: demo-indirect-invocation
+            host: test-runner-sgx
+            sgx_mode: HW
           - test: Teeracle
             flavor_id: teeracle
             demo_name: demo-teeracle
+            host: test-runner-sgx
+            sgx_mode: HW
           - test: Teeracle
             flavor_id: teeracle
             demo_name: demo-teeracle-generic
+            host: test-runner-sgx
+            sgx_mode: HW
           - test: Benchmark
             flavor_id: sidechain
             demo_name: sidechain-benchmark
+            host: test-runner-sgx
+            sgx_mode: HW
           - test: EVM
             flavor_id: sidechain-evm
             demo_name: demo-smart-contract
+            host: test-runner-sgx
+            sgx_mode: HW
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Set env
         run: |
+          version=$RANDOM
           echo "FLAVOR_ID=${{ matrix.flavor_id }}" >> $GITHUB_ENV
+          echo "PROJECT=${{ matrix.flavor_id }}-${{ matrix.demo_name }}" >> $GITHUB_ENV
+          echo "VERSION=dev.$version" >> $GITHUB_ENV
+          echo "WORKER_IMAGE_TAG=integritee-worker:dev.$version" >> $GITHUB_ENV
+          echo "CLIENT_IMAGE_TAG=integritee-cli:dev.$version" >> $GITHUB_ENV
+          if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then
+             echo "SGX_PROVISION=/dev/sgx/provision" >> $GITHUB_ENV
+             echo "SGX_ENCLAVE=/dev/sgx/enclave" >> $GITHUB_ENV
+             echo "AESMD=/var/run/aesmd" >> $GITHUB_ENV
+          fi
 
       - name: Download Worker Image
         uses: actions/download-artifact@v3
@@ -217,42 +251,82 @@ jobs:
           docker image load --input integritee-cli-client-${{ matrix.flavor_id }}-${{ github.sha }}.tar.gz
           docker images --all
 
+      ##
+      # Before tagging, delete the old "stuck" ones to be sure that the newly created ones are the latest
+      # Without if the docker image rmi throws an error if the image doesn't exist.
+      ##
       - name: Re-name Image Tags
         run: |
+          if [[ "$(docker images -q ${{ env.WORKER_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then
+             docker image rmi --force ${{ env.WORKER_IMAGE_TAG }} 2>/dev/null
+          fi
+          if [[ "$(docker images -q ${{ env.CLIENT_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then
+             docker image rmi --force ${{ env.CLIENT_IMAGE_TAG }} 2>/dev/null
+          fi
           docker tag integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} ${{ env.WORKER_IMAGE_TAG }}
           docker tag integritee-cli-client-${{ matrix.flavor_id }}-${{ github.sha }} ${{ env.CLIENT_IMAGE_TAG }}
           docker images --all
 
-      - name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }}
+      ##
+      # Stop any stucked/running compose projects
+      ##
+      - name: Stop docker containers
+        if: always()
+        continue-on-error: true
         run: |
           cd docker
-          docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }}
+          docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} stop
 
-      - name: Stop docker containers
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }}
         run: |
           cd docker
-          docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml stop
+          docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }} --remove-orphans
+
 
       - name: Collect Docker Logs
         continue-on-error: true
         if: always()
         uses: jwalton/gh-docker-logs@v2
         with:
-          #images: '${{ env.WORKER_IMAGE_TAG }},${{ env.CLIENT_IMAGE_TAG }}'
+          images: 'integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }},integritee-cli-client-${{ matrix.flavor_id }}-${{ github.sha }}'
           tail: all
-          dest: ./${{ env.LOG_DIR }}
+          dest: ./${LOG_DIR}-${{ matrix.flavor_id }}-${{ github.sha }}
 
       - name: Upload logs
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: logs-${{ matrix.test }}-${{ matrix.flavor_id }}
-          path: ./${{ env.LOG_DIR }}
+          path: ./${LOG_DIR}-${{ matrix.flavor_id }}-${{ github.sha }}
+
+      - name: Stop docker containers
+        if: always()
+        continue-on-error: true
+        run: |
+          cd docker
+          docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} stop
+
+      - name: Delete images
+        run: |
+          if [[ "$(docker images -q ${{ env.WORKER_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then
+             docker image rmi --force ${{ env.WORKER_IMAGE_TAG }} 2>/dev/null
+          fi
+          if [[ "$(docker images -q ${{ env.CLIENT_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then
+             docker image rmi --force ${{ env.CLIENT_IMAGE_TAG }} 2>/dev/null
+          fi
+          docker images --all
+
 
   release:
+    runs-on: ubuntu-22.04
     name: Draft Release
     if: startsWith(github.ref, 'refs/tags/')
-    runs-on: ubuntu-22.04
     needs: [build-test, integration-tests]
     outputs:
       release_url: ${{ steps.create-release.outputs.html_url }}

.github/workflows/delete-release.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   purge-image:
     name: Delete image from ghcr.io
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         binary: ["integritee-client", "integritee-demo-validateer"]

.github/workflows/label-checker.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
     check_for_matching_labels:
-      runs-on: ubuntu-22.04
+      runs-on: ubuntu-latest
       if: github.base_ref == 'master' && github.event.pull_request.draft == false
       steps:
         - name: Label check

.github/workflows/publish-docker-release.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   main:
     name: Push Integritee Services to Dockerhub
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         binary: ["integritee-demo-validateer", "integritee-client"]

.github/workflows/publish-draft-release.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   publish-draft-release:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3

build.Dockerfile

@@ -17,18 +17,21 @@
 
 ### Builder Stage
 ##################################################
-FROM integritee/integritee-dev:0.1.13 AS builder
+FROM integritee/integritee-dev:0.2.1 AS builder
 LABEL maintainer="zoltan@integritee.network"
 
 # set environment variables
 ENV SGX_SDK /opt/sgxsdk
-ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/root/.cargo/bin"
+ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin//bin"
 ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig"
 ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs"
 ENV CARGO_NET_GIT_FETCH_WITH_CLI true
-ENV SGX_MODE SW
 
-ENV HOME=/root/work
+# Default SGX MODE is software mode
+ARG SGX_MODE=SW
+ENV SGX_MODE=$SGX_MODE
+
+ENV HOME=/home/ubuntu/work
 
 ARG WORKER_MODE_ARG
 ENV WORKER_MODE=$WORKER_MODE_ARG
@@ -49,42 +52,42 @@ RUN cargo test --release
 # A builder stage that uses sccache to speed up local builds with docker
 # Installation and setup of sccache should be moved to the integritee-dev image, so we don't
 # always need to compile and install sccache on CI (where we have no caching so far).
-FROM integritee/integritee-dev:0.1.13 AS cached-builder
+FROM integritee/integritee-dev:0.2.1 AS cached-builder
 LABEL maintainer="zoltan@integritee.network"
 
 # set environment variables
 ENV SGX_SDK /opt/sgxsdk
-ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/root/.cargo/bin"
+ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin/bin"
 ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig"
 ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs"
 ENV CARGO_NET_GIT_FETCH_WITH_CLI true
-ENV SGX_MODE SW
 
-ENV HOME=/root/work
+# Default SGX MODE is software mode
+ARG SGX_MODE=SW
+ENV SGX_MODE=$SGX_MODE
+
+ENV HOME=/home/ubuntu/work
 
-RUN rustup default stable && cargo install sccache --root /usr/local/cargo
-ENV PATH "$PATH:/usr/local/cargo/bin"
+RUN rustup default stable && cargo install sccache
 ENV SCCACHE_CACHE_SIZE="3G"
 ENV SCCACHE_DIR=$HOME/.cache/sccache
-ENV RUSTC_WRAPPER="/usr/local/cargo/bin/sccache"
+ENV RUSTC_WRAPPER="/opt/rust/bin/sccache"
 
 ARG WORKER_MODE_ARG
 ENV WORKER_MODE=$WORKER_MODE_ARG
 
 WORKDIR $HOME/worker
 COPY . .
 
-RUN --mount=type=cache,id=cargo,target=/root/work/.cache/sccache make && sccache --show-stats
+RUN --mount=type=cache,id=cargo,target=${HOME}/.cache/sccache make && sccache --show-stats
 
-RUN --mount=type=cache,id=cargo,target=/root/work/.cache/sccache cargo test --release && sccache --show-stats
+RUN --mount=type=cache,id=cargo,target=${HOME}/.cache/sccache cargo test --release && sccache --show-stats
 
 
 ### Base Runner Stage
-##################################################
-FROM ubuntu:22.04 AS runner
-
-RUN apt update && apt install -y libssl-dev iproute2 curl
-
+### The runner needs the aesmd service for the `SGX_MODE=HW`.
+######################################################
+FROM oasisprotocol/aesmd:master AS runner
 
 ### Deployed CLI client
 ##################################################
@@ -97,7 +100,7 @@ ARG LOG_DIR=/usr/local/log
 ENV SCRIPT_DIR ${SCRIPT_DIR}
 ENV LOG_DIR ${LOG_DIR}
 
-COPY --from=builder /root/work/worker/bin/integritee-cli /usr/local/bin
+COPY --from=builder /home/ubuntu/work/worker/bin/integritee-cli /usr/local/bin
 COPY ./cli/*.sh /usr/local/worker-cli/
 
 RUN chmod +x /usr/local/bin/integritee-cli ${SCRIPT_DIR}/*.sh
@@ -114,13 +117,10 @@ ENTRYPOINT ["/usr/local/bin/integritee-cli"]
 FROM runner AS deployed-worker
 LABEL maintainer="zoltan@integritee.network"
 
-ENV SGX_SDK /opt/sgxsdk
-ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/lib64"
-
 WORKDIR /usr/local/bin
 
-COPY --from=builder /opt/sgxsdk/lib64 /opt/sgxsdk/lib64
-COPY --from=builder /root/work/worker/bin/* ./
+COPY --from=builder /opt/sgxsdk /opt/sgxsdk
+COPY --from=builder /home/ubuntu/work/worker/bin/* ./
 COPY --from=builder /lib/x86_64-linux-gnu/libsgx* /lib/x86_64-linux-gnu/
 COPY --from=builder /lib/x86_64-linux-gnu/libdcap* /lib/x86_64-linux-gnu/
 
@@ -129,6 +129,8 @@ RUN chmod +x /usr/local/bin/integritee-service
 RUN ls -al /usr/local/bin
 
 # checks
+ENV SGX_SDK /opt/sgxsdk
+ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:$SGX_SDK/sdk_libs
 RUN ldd /usr/local/bin/integritee-service && \
 	/usr/local/bin/integritee-service --version