build: retry 'docker push'

Makefile

@@ -91,8 +91,20 @@ push-images: push-image push-test-image
 build-image build-test-image: build%-image:
 	docker build --pull --build-arg CACHEBUST=$(BUILD_IMAGE_ID) --build-arg BIN_SUFFIX=$(findstring -test,$*) $(BUILD_ARGS) -t $(IMAGE_TAG) -f ./Dockerfile . --label revision=$(VERSION)
 PUSH_IMAGE_DEP = build%-image
-push-image push-test-image: push%-image: $(PUSH_IMAGE_DEP)
-	docker push $(IMAGE_TAG)
+# "docker push" has been seen to fail temporarily with "error creating overlay mount to /var/lib/docker/overlay2/xxx/merged: device or resource busy".
+# Here we simply try three times before giving up.
+push-image push-test-image: push%-image: # $(PUSH_IMAGE_DEP)
+	@ i=0; while true; do \
+		if (set -x; docker push $(IMAGE_TAG)); then \
+			exit 0; \
+		elif [ $$i -ge 2 ]; then \
+			echo "'docker push' failed repeatedly, giving up"; \
+			exit; \
+		else \
+			echo "attempt #$$i: 'docker push' failed, will try again"; \
+			i=$$(($$i + 1)); \
+		fi; \
+	done
 
 .PHONY: print-image-version
 print-image-version:

test/start-kubernetes.sh

@@ -373,7 +373,19 @@ EOF
             docker save "$image" | ssh $SSH_ARGS ${CLOUD_USER}@${master_ip} sudo docker load || die "failed to copy $image"
             echo Load $image into registry
             ssh $SSH_ARGS ${CLOUD_USER}@${master_ip} sudo docker tag "$image" "$remoteimage" || die "failed to tag $image as $remoteimage"
-            ssh $SSH_ARGS ${CLOUD_USER}@${master_ip} sudo docker push "$remoteimage" || die "failed to push $remoteimage"
+            # "docker push" has been seen to fail temporarily with "error creating overlay mount to /var/lib/docker/overlay2/xxx/merged: device or resource busy".
+            # Here we simply try three times before giving up.
+	    local i=0
+            while true; do
+		if (set -x; ssh $SSH_ARGS ${CLOUD_USER}@${master_ip} sudo docker push "$remoteimage"); then
+		    break
+		elif [ $i -ge 2 ]; then
+		    die "'docker push' failed repeatedly, giving up"
+		else
+		    echo "attempt #$i: 'docker push' failed, will try again"
+		    i=$(($i + 1))
+		fi
+	    done
         done
 
         # TEST_PMEM_REGISTRY in test-config.sh uses this machine name as registry,