diff --git a/Makefile b/Makefile index fff1f2fc30..abe9958885 100644 --- a/Makefile +++ b/Makefile @@ -91,8 +91,20 @@ push-images: push-image push-test-image build-image build-test-image: build%-image: docker build --pull --build-arg CACHEBUST=$(BUILD_IMAGE_ID) --build-arg BIN_SUFFIX=$(findstring -test,$*) $(BUILD_ARGS) -t $(IMAGE_TAG) -f ./Dockerfile . --label revision=$(VERSION) PUSH_IMAGE_DEP = build%-image -push-image push-test-image: push%-image: $(PUSH_IMAGE_DEP) - docker push $(IMAGE_TAG) +# "docker push" has been seen to fail temporarily with "error creating overlay mount to /var/lib/docker/overlay2/xxx/merged: device or resource busy". +# Here we simply try three times before giving up. +push-image push-test-image: push%-image: # $(PUSH_IMAGE_DEP) + @ i=0; while true; do \ + if (set -x; docker push $(IMAGE_TAG)); then \ + exit 0; \ + elif [ $$i -ge 2 ]; then \ + echo "'docker push' failed repeatedly, giving up"; \ + exit; \ + else \ + echo "attempt #$$i: 'docker push' failed, will try again"; \ + i=$$(($$i + 1)); \ + fi; \ + done .PHONY: print-image-version print-image-version: diff --git a/test/start-kubernetes.sh b/test/start-kubernetes.sh index 93742b25b3..dd3fc87c68 100755 --- a/test/start-kubernetes.sh +++ b/test/start-kubernetes.sh @@ -373,7 +373,19 @@ EOF docker save "$image" | ssh $SSH_ARGS ${CLOUD_USER}@${master_ip} sudo docker load || die "failed to copy $image" echo Load $image into registry ssh $SSH_ARGS ${CLOUD_USER}@${master_ip} sudo docker tag "$image" "$remoteimage" || die "failed to tag $image as $remoteimage" - ssh $SSH_ARGS ${CLOUD_USER}@${master_ip} sudo docker push "$remoteimage" || die "failed to push $remoteimage" + # "docker push" has been seen to fail temporarily with "error creating overlay mount to /var/lib/docker/overlay2/xxx/merged: device or resource busy". + # Here we simply try three times before giving up. + local i=0 + while true; do + if (set -x; ssh $SSH_ARGS ${CLOUD_USER}@${master_ip} sudo docker push "$remoteimage"); then + break + elif [ $i -ge 2 ]; then + die "'docker push' failed repeatedly, giving up" + else + echo "attempt #$i: 'docker push' failed, will try again" + i=$(($i + 1)) + fi + done done # TEST_PMEM_REGISTRY in test-config.sh uses this machine name as registry,