diff --git a/build/backup/backup.sh b/build/backup/backup.sh old mode 100644 new mode 100755 index 0285fb107..6e411a2f1 --- a/build/backup/backup.sh +++ b/build/backup/backup.sh @@ -46,7 +46,7 @@ function get_backup_source() { | cut -d . -f 1 } -function request_streaming() { +function request_streaming_57() { local LOCAL_IP local NODE_NAME LOCAL_IP=$(hostname -i | sed -E 's/.*\b([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})\b.*/\1/') @@ -69,40 +69,20 @@ function request_streaming() { --sst "xtrabackup-v2:$LOCAL_IP:4444/xtrabackup_sst//1" \ --recv-script="/opt/percona/backup/run_backup.sh" 2>&1 | tee /tmp/garbd.log - local sst_info_path - if [[ -n $S3_BUCKET || -n $AZURE_CONTAINER_NAME ]]; then - sst_info_path="/tmp/${SST_INFO_NAME}" - else - sst_info_path="${BACKUP_DIR}/${SST_INFO_NAME}" + if grep 'State transfer request failed' /tmp/garbd.log; then + exit 1 fi - - MYSQL_VERSION=$(parse_ini 'mysql-version' "$sst_info_path") - if ! check_for_version "$MYSQL_VERSION" '8.0.0'; then - if grep 'State transfer request failed' /tmp/garbd.log; then - exit 1 - fi - if grep 'WARN: Protocol violation. JOIN message sender ... (garb) is not in state transfer' /tmp/garbd.log; then - exit 1 - fi - if grep 'WARN: Rejecting JOIN message from ... (garb): new State Transfer required.' /tmp/garbd.log; then - exit 1 - fi - if grep 'INFO: Shifting CLOSED -> DESTROYED (TO: -1)' /tmp/garbd.log; then - exit 1 - fi - if ! grep 'INFO: Sending state transfer request' /tmp/garbd.log; then - exit 1 - fi - else - if grep 'Will never receive state. Need to abort' /tmp/garbd.log; then - exit 1 - fi - - if grep 'Donor is no longer in the cluster, interrupting script' /tmp/garbd.log; then - exit 1 - elif grep 'failed: Invalid argument' /tmp/garbd.log; then - exit 1 - fi + if grep 'WARN: Protocol violation. JOIN message sender ... (garb) is not in state transfer' /tmp/garbd.log; then + exit 1 + fi + if grep 'WARN: Rejecting JOIN message from ... (garb): new State Transfer required.' /tmp/garbd.log; then + exit 1 + fi + if grep 'INFO: Shifting CLOSED -> DESTROYED (TO: -1)' /tmp/garbd.log; then + exit 1 + fi + if ! grep 'INFO: Sending state transfer request' /tmp/garbd.log; then + exit 1 fi if [ -f '/tmp/backup-is-completed' ]; then @@ -115,6 +95,56 @@ function request_streaming() { exit 1 } +function request_streaming() { + local LOCAL_IP + local NODE_NAME + LOCAL_IP=$(hostname -i | sed -E 's/.*\b([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})\b.*/\1/') + NODE_NAME=$(get_backup_source) + + if [ -z "$NODE_NAME" ]; then + /opt/percona/peer-list -on-start=/opt/percona/backup/lib/pxc/get-pxc-state.sh -service="$PXC_SERVICE" + log 'ERROR' 'Cannot find node for backup' + log 'ERROR' 'Backup was finished unsuccessful' + exit 1 + fi + + set +o errexit + log 'INFO' 'Garbd was started' + garbd \ + --address "gcomm://$NODE_NAME.$PXC_SERVICE?gmcast.listen_addr=tcp://0.0.0.0:4567" \ + --donor "$NODE_NAME" \ + --group "$PXC_SERVICE" \ + --options "$GARBD_OPTS" \ + --sst "xtrabackup-v2:$LOCAL_IP:4444/xtrabackup_sst//1" \ + --extended-exit-codes \ + --wait-for-recv-script-exit \ + --recv-script="/opt/percona/backup/run_backup.sh" + GARBD_EXIT_CODE=$? + + case ${GARBD_EXIT_CODE} in + 0) + log 'INFO' 'Backup was finished successfully' + exit 0 + ;; + 100) + log 'ERROR' 'Backup was unsuccessful: Generic failure' + exit 1 + ;; + 101) + log 'ERROR' 'Backup was unsuccessful: Donor disappeared' + exit 1 + ;; + 102) + log 'ERROR' 'Backup was unsuccessful: SST request failure' + exit 1 + ;; + *) + log 'ERROR' "Backup was unsuccessful: garbd exited with ${GARBD_EXIT_CODE}" + exit 1 + ;; + esac +} + # TODO: should i remove it? function check_ssl() { CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -149,4 +179,9 @@ elif [ -n "$AZURE_CONTAINER_NAME" ]; then clean_backup_azure fi -request_streaming +XTRABACKUP_VERSION=$(get_xtrabackup_version) +if check_for_version "$XTRABACKUP_VERSION" '8.0.0'; then + request_streaming +else + request_streaming_57 +fi diff --git a/build/backup/recovery-cloud.sh b/build/backup/recovery-cloud.sh index ac7210252..cec45889f 100644 --- a/build/backup/recovery-cloud.sh +++ b/build/backup/recovery-cloud.sh @@ -102,7 +102,8 @@ else REMAINING_XB_ARGS="$XB_EXTRA_ARGS" fi -if ! check_for_version "$XTRABACKUP_VERSION" '8.0.0'; then +if ! check_for_version "$XTRABACKUP_VERSION" '8.0.0' \ + && ! check_for_version "$XTRABACKUP_VERSION" '8.4.0'; then # shellcheck disable=SC2086 innobackupex ${XB_USE_MEMORY+--use-memory=$XB_USE_MEMORY} --parallel="$(grep -c processor /proc/cpuinfo)" $REMAINING_XB_ARGS --decompress "$tmp" XB_EXTRA_ARGS="$XB_EXTRA_ARGS --binlog-info=ON" diff --git a/build/backup/recovery-pvc-joiner.sh b/build/backup/recovery-pvc-joiner.sh index 9c82452a8..483f3624c 100755 --- a/build/backup/recovery-pvc-joiner.sh +++ b/build/backup/recovery-pvc-joiner.sh @@ -97,7 +97,8 @@ else REMAINING_XB_ARGS="$XB_EXTRA_ARGS" fi -if ! check_for_version "$XTRABACKUP_VERSION" '8.0.0'; then +if ! check_for_version "$XTRABACKUP_VERSION" '8.0.0' \ + && ! check_for_version "$XTRABACKUP_VERSION" '8.4.0'; then # shellcheck disable=SC2086 innobackupex $DEFAULTS_FILE ${XB_USE_MEMORY+--use-memory=$XB_USE_MEMORY} --parallel="$(grep -c processor /proc/cpuinfo)" $REMAINING_XB_ARGS --decompress "$tmp" XB_EXTRA_ARGS="$XB_EXTRA_ARGS --binlog-info=ON" diff --git a/build/backup/run_backup.sh b/build/backup/run_backup.sh index b1d8267dd..ce6272318 100755 --- a/build/backup/run_backup.sh +++ b/build/backup/run_backup.sh @@ -1,7 +1,8 @@ #!/bin/bash -set -o errexit set -o xtrace +set -o errexit +set -o pipefail set -m LIB_PATH='/opt/percona/backup/lib/pxc' @@ -96,7 +97,6 @@ backup_volume() { log 'INFO' "Socat(2) returned $?" fi - trap '' 15 stat xtrabackup.stream if (($(stat -c%s xtrabackup.stream) < 5000000)); then log 'ERROR' 'Backup is empty' @@ -126,20 +126,33 @@ backup_s3() { fi vault_store /tmp/${SST_INFO_NAME} + # this xbcloud command will fail with backup is incomplete + # it's expected since we only upload sst_info + set +o pipefail # shellcheck disable=SC2086 xbstream -C /tmp -c ${SST_INFO_NAME} $XBSTREAM_EXTRA_ARGS \ - | xbcloud put --storage=s3 --parallel="$(grep -c processor /proc/cpuinfo)" --md5 $XBCLOUD_ARGS --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME" 2>&1 \ + | xbcloud put --storage=s3 \ + --md5 \ + --parallel="$(grep -c processor /proc/cpuinfo)" \ + $XBCLOUD_ARGS \ + --s3-bucket="$S3_BUCKET" \ + "$S3_BUCKET_PATH.$SST_INFO_NAME" 2>&1 \ | (grep -v "error: http request failed: Couldn't resolve host name" || exit 1) + set -o pipefail if ((SST_FAILED == 0)); then # shellcheck disable=SC2086 socat -u "$SOCAT_OPTS" stdio \ - | xbcloud put --storage=s3 --parallel="$(grep -c processor /proc/cpuinfo)" --md5 $XBCLOUD_ARGS --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH" 2>&1 \ - | (grep -v "error: http request failed: Couldn't resolve host name" || exit 1) + | xbcloud put --storage=s3 \ + --md5 \ + --parallel="$(grep -c processor /proc/cpuinfo)" \ + $XBCLOUD_ARGS \ + --s3-bucket="$S3_BUCKET" \ + "$S3_BUCKET_PATH" 2>&1 \ + | (grep -v "error: http request failed: Couldn't resolve host name" || exit 1) & + wait $! fi - trap '' 15 - # shellcheck disable=SC2086 aws $AWS_S3_NO_VERIFY_SSL s3 ls "s3://$S3_BUCKET/$S3_BUCKET_PATH.md5" # shellcheck disable=SC2086 @@ -172,16 +185,27 @@ backup_azure() { fi vault_store /tmp/${SST_INFO_NAME} + # this xbcloud command will fail with backup is incomplete + # it's expected since we only upload sst_info + set +o pipefail # shellcheck disable=SC2086 xbstream -C /tmp -c ${SST_INFO_NAME} $XBSTREAM_EXTRA_ARGS \ - | xbcloud put --storage=azure --parallel="$(grep -c processor /proc/cpuinfo)" $XBCLOUD_ARGS "$BACKUP_PATH.$SST_INFO_NAME" 2>&1 \ + | xbcloud put --storage=azure \ + --parallel="$(grep -c processor /proc/cpuinfo)" \ + $XBCLOUD_ARGS \ + "$BACKUP_PATH.$SST_INFO_NAME" 2>&1 \ | (grep -v "error: http request failed: Couldn't resolve host name" || exit 1) + set -o pipefail if ((SST_FAILED == 0)); then # shellcheck disable=SC2086 socat -u "$SOCAT_OPTS" stdio \ - | xbcloud put --storage=azure --parallel="$(grep -c processor /proc/cpuinfo)" $XBCLOUD_ARGS "$BACKUP_PATH" 2>&1 \ - | (grep -v "error: http request failed: Couldn't resolve host name" || exit 1) + | xbcloud put --storage=azure \ + --parallel="$(grep -c processor /proc/cpuinfo)" \ + $XBCLOUD_ARGS \ + "$BACKUP_PATH" 2>&1 \ + | (grep -v "error: http request failed: Couldn't resolve host name" || exit 1) & + wait $! fi } diff --git a/e2e-tests/demand-backup/run b/e2e-tests/demand-backup/run index b8a36b01a..8373a53f9 100755 --- a/e2e-tests/demand-backup/run +++ b/e2e-tests/demand-backup/run @@ -53,7 +53,7 @@ check_finalizer_for_fs() { } -check_restore_logs() { +check_logs() { local restore=$1 local search=$2 @@ -118,8 +118,8 @@ main() { fi backup_job_name=$(kubectl get pod -l "percona.com/backup-job-name=xb-on-demand-backup-minio" -o jsonpath='{.items[].metadata.name}') - kubectl logs "$backup_job_name" | grep -E "xbcloud put --storage=s3 --parallel=[0-9]+ --md5 --insecure (--curl-retriable-errors=7 )?--parallel=2 --curl-retriable-errors=8" - kubectl logs "$backup_job_name" | grep "xbstream -x -C /tmp --parallel=2" + check_logs "$backup_job_name" "xbcloud put --storage=s3 --md5 --parallel=[0-9]+ --insecure (--curl-retriable-errors=7 )?--parallel=2 --curl-retriable-errors=8" + check_logs "$backup_job_name" "xbstream -x -C /tmp --parallel=2" new_pass=$(echo -n "newpass" | base64) kubectl_bin patch secret my-cluster-secrets -p="{\"data\":{\"xtrabackup\": \"${new_pass}\", \"monitor\": \"${new_pass}\"}}" @@ -131,11 +131,11 @@ main() { compare_kubectl job/restore-job-on-demand-backup-minio-demand-backup restore_job_name=$(kubectl get pod -l "percona.com/restore-job-name=restore-job-on-demand-backup-minio-demand-backup" -o jsonpath='{.items[].metadata.name}') - check_restore_logs ${restore_job_name} "xtrabackup --defaults-group=mysqld --datadir=/datadir --move-back" - check_restore_logs ${restore_job_name} "xtrabackup --use-memory=1500000000 --prepare" - check_restore_logs ${restore_job_name} "xbcloud get --parallel=[0-9]+ --insecure (--curl-retriable-errors=7 )?--parallel=3" - check_restore_logs ${restore_job_name} "xbstream -x -C .* --parallel=[0-9]+ --parallel=3" - check_restore_logs ${restore_job_name} "(xbstream --decompress -x -C .* --parallel=[0-9]+ --parallel=3|xbstream -x -C .* --parallel=[0-9]+ --parallel=3)" + check_logs ${restore_job_name} "xtrabackup --defaults-group=mysqld --datadir=/datadir --move-back" + check_logs ${restore_job_name} "xtrabackup --use-memory=1500000000 --prepare" + check_logs ${restore_job_name} "xbcloud get --parallel=[0-9]+ --insecure (--curl-retriable-errors=7 )?--parallel=3" + check_logs ${restore_job_name} "xbstream -x -C .* --parallel=[0-9]+ --parallel=3" + check_logs ${restore_job_name} "(xbstream --decompress -x -C .* --parallel=[0-9]+ --parallel=3|xbstream -x -C .* --parallel=[0-9]+ --parallel=3)" desc "Check backup deletion" kubectl_bin delete pxc-backup --all