HDDS-11334. Improve EC xcompat acceptance test (#7492)

Author: adoroszlai

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -30,8 +30,29 @@ fi
 
 source ${_testlib_dir}/compose_v2_compatibility.sh
 
+: ${OZONE_COMPOSE_RUNNING:=false}
 : ${SCM:=scm}
 
+# create temp directory for test data; only once, even if testlib.sh is sourced again
+if [[ -z "${TEST_DATA_DIR:-}" ]] && [[ "${KEEP_RUNNING:-false}" == "false" ]]; then
+  export TEST_DATA_DIR="$(mktemp -d "${TMPDIR:-/tmp}"/robot-data-XXXXXX)"
+  chmod go+rx "${TEST_DATA_DIR}"
+  _compose_delete_test_data() {
+    rm -frv "${TEST_DATA_DIR}"
+  }
+
+  trap _compose_cleanup EXIT HUP INT TERM
+fi
+
+_compose_cleanup() {
+  if [[ "${OZONE_COMPOSE_RUNNING}" == "true" ]]; then
+    stop_docker_env || true
+  fi
+  if [[ "$(type -t _compose_delete_test_data || true)" == "function" ]]; then
+    _compose_delete_test_data
+  fi
+}
+
 ## @description create results directory, purging any prior data
 create_results_dir() {
   #delete previous results
@@ -140,13 +161,13 @@ start_docker_env(){
 
   docker-compose --ansi never down --remove-orphans
 
-  trap stop_docker_env EXIT HUP INT TERM
-
   opts=""
   if has_scalable_datanode; then
     opts="--scale datanode=${datanode_count}"
   fi
 
+  OZONE_COMPOSE_RUNNING=true
+  trap _compose_cleanup EXIT HUP INT TERM
   docker-compose --ansi never up -d $opts
 
   wait_for_safemode_exit
@@ -184,11 +205,11 @@ execute_robot_test(){
   local output_name=$(get_output_name)
 
   # find unique filename
-  declare -i i=0
-  OUTPUT_FILE="robot-${output_name}1.xml"
-  while [[ -f $RESULT_DIR/$OUTPUT_FILE ]]; do
-    let ++i
-    OUTPUT_FILE="robot-${output_name}${i}.xml"
+  for ((i=1; i<1000; i++)); do
+    OUTPUT_FILE="robot-${output_name}$(printf "%03d" ${i}).xml"
+    if [[ ! -f $RESULT_DIR/$OUTPUT_FILE ]]; then
+      break;
+    fi
   done
 
   SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest"
@@ -368,6 +389,7 @@ stop_docker_env(){
     for i in $(seq 1 $down_repeats)
     do
       if docker-compose --ansi never --profile "*" down --remove-orphans; then
+        OZONE_COMPOSE_RUNNING=false
         return
       fi
       if [[ ${i} -eq 1 ]]; then

hadoop-ozone/dist/src/main/compose/upgrade/test.sh

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#suite:compat
+#suite:upgrade
 
 
 TEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )

hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml

@@ -14,69 +14,58 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+x-common-config:
+  &common-config
+  dns_search: .
+  env_file:
+    - docker-config
+  command: ["sleep","1000000"]
+
+x-old-config:
+  &old-config
+  <<: *common-config
+  volumes:
+    - ../..:/opt/ozone
+    - ../_keytabs:/etc/security/keytabs
+    - ./krb5.conf:/etc/krb5.conf
+    - ${TEST_DATA_DIR}:/testdata
+
+x-new-config:
+  &new-config
+  image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+  <<: *common-config
+  volumes:
+    - ../..:/opt/hadoop
+    - ../_keytabs:/etc/security/keytabs
+    - ./krb5.conf:/etc/krb5.conf
+    - ${TEST_DATA_DIR}:/testdata
+
 services:
   old_client_1_0_0:
     image: ${OZONE_IMAGE}:1.0.0${OZONE_IMAGE_FLAVOR}
-    env_file:
-      - docker-config
-    volumes:
-      - ../..:/opt/ozone
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
-    command: ["sleep","1000000"]
+    <<: *old-config
+
   old_client_1_1_0:
     image: ${OZONE_IMAGE}:1.1.0${OZONE_IMAGE_FLAVOR}
-    env_file:
-      - docker-config
-    volumes:
-      - ../..:/opt/ozone
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
-    command: ["sleep","1000000"]
+    <<: *old-config
+
   old_client_1_2_1:
     image: ${OZONE_IMAGE}:1.2.1${OZONE_IMAGE_FLAVOR}
-    env_file:
-      - docker-config
-    volumes:
-      - ../..:/opt/ozone
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
-    command: ["sleep","1000000"]
+    <<: *old-config
+
   old_client_1_3_0:
     image: ${OZONE_IMAGE}:1.3.0${OZONE_IMAGE_FLAVOR}
-    env_file:
-      - docker-config
-    volumes:
-      - ../..:/opt/ozone
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
-    command: ["sleep","1000000"]
+    <<: *old-config
+
   old_client_1_4_0:
     image: ${OZONE_IMAGE}:1.4.0${OZONE_IMAGE_FLAVOR}
-    env_file:
-      - docker-config
-    volumes:
-      - ../..:/opt/ozone
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
-    command: ["sleep","1000000"]
+    <<: *old-config
+
   old_client_1_4_1:
     image: ${OZONE_IMAGE}:1.4.1${OZONE_IMAGE_FLAVOR}
-    env_file:
-      - docker-config
-    volumes:
-      - ../..:/opt/ozone
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
-    command: ["sleep","1000000"]
+    <<: *old-config
+
   new_client:
-    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
-    env_file:
-      - docker-config
-    volumes:
-      - ../..:/opt/hadoop
-      - ../_keytabs:/etc/security/keytabs
-      - ./krb5.conf:/etc/krb5.conf
+    <<: *new-config
     environment:
       OZONE_OPTS:
-    command: ["sleep","1000000"]

hadoop-ozone/dist/src/main/compose/xcompat/test.sh

@@ -32,16 +32,17 @@ source "${COMPOSE_DIR}/../testlib.sh"
 export SECURITY_ENABLED=true
 : ${OZONE_BUCKET_KEY_NAME:=key1}
 
-old_client() {
-  OZONE_DIR=/opt/ozone
-  container=${client}
-  "$@"
-}
+echo 'Compatibility Test' > "${TEST_DATA_DIR}"/small
+
+client() {
+  if [[ "${client_version}" == "${current_version}" ]]; then
+    OZONE_DIR=/opt/hadoop
+    container=new_client
+  else
+    OZONE_DIR=/opt/ozone
+    container="old_client_${client_version//./_}"
+  fi
 
-new_client() {
-  OZONE_DIR=/opt/hadoop
-  container=new_client
-  client_version=${current_version}
   "$@"
 }
 
@@ -60,7 +61,7 @@ _write() {
   execute_robot_test ${container} -N "xcompat-cluster-${cluster_version}-client-${client_version}-write" \
     -v CLIENT_VERSION:${client_version} \
     -v CLUSTER_VERSION:${cluster_version} \
-    -v SUFFIX:${client_version} \
+    -v TEST_DATA_DIR:/testdata \
     compatibility/write.robot
 }
 
@@ -71,87 +72,46 @@ _read() {
     -v CLIENT_VERSION:${client_version} \
     -v CLUSTER_VERSION:${cluster_version} \
     -v DATA_VERSION:${data_version} \
-    -v SUFFIX:${data_version} \
+    -v TEST_DATA_DIR:/testdata \
     compatibility/read.robot
 }
 
 test_cross_compatibility() {
   echo "Starting ${cluster_version} cluster with COMPOSE_FILE=${COMPOSE_FILE}"
 
-  OZONE_KEEP_RESULTS=true start_docker_env
+  OZONE_KEEP_RESULTS=true start_docker_env 5
 
   execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
 
   _init
 
-  new_client _write
-  new_client _read ${current_version}
+  # first write with client matching cluster version
+  client_version="${cluster_version}" client _write
 
   for client_version in "$@"; do
-    client="old_client_${client_version//./_}"
-
-    old_client _write
-    old_client _read ${client_version}
-
-    old_client _read ${current_version}
-    new_client _read ${client_version}
+    # skip write, since already done
+    if [[ "${client_version}" == "${cluster_version}" ]]; then
+      continue
+    fi
+    client _write
   done
 
-  KEEP_RUNNING=false stop_docker_env
-}
-
-test_ec_cross_compatibility() {
-  echo "Running Erasure Coded storage backward compatibility tests."
-  # local cluster_versions_with_ec="1.3.0 1.4.0 ${current_version}"
-  local cluster_versions_with_ec="${current_version}" # until HDDS-11334
-  # TODO: debug acceptance test failures for client versions 1.0.0 on secure clusters
-  local non_ec_client_versions="1.1.0 1.2.1"
-
-  for cluster_version in ${cluster_versions_with_ec}; do
-    export COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${cluster_version}
-    OZONE_KEEP_RESULTS=true start_docker_env 5
-
-    echo -n "Generating data locally...   "
-    dd if=/dev/urandom of=/tmp/1mb bs=1048576 count=1 >/dev/null 2>&1
-    dd if=/dev/urandom of=/tmp/2mb bs=1048576 count=2 >/dev/null 2>&1
-    dd if=/dev/urandom of=/tmp/3mb bs=1048576 count=3 >/dev/null 2>&1
-    echo "done"
-    echo -n "Copy data into client containers...   "
-    for container in $(docker ps --format '{{.Names}}' | grep client); do
-      docker cp /tmp/1mb ${container}:/tmp/1mb
-      docker cp /tmp/2mb ${container}:/tmp/2mb
-      docker cp /tmp/3mb ${container}:/tmp/3mb
-    done
-    echo "done"
-    rm -f /tmp/1mb /tmp/2mb /tmp/3mb
-
-
-    local prefix=$(LC_CTYPE=C tr -dc '[:alnum:]' < /dev/urandom | head -c 5 | tr '[:upper:]' '[:lower:]')
-    OZONE_DIR=/opt/hadoop
-    new_client _kinit
-    execute_robot_test new_client --include setup-ec-data -N "xcompat-cluster-${cluster_version}-setup-data" -v prefix:"${prefix}" ec/backward-compat.robot
-     OZONE_DIR=/opt/ozone
-
-    for client_version in ${non_ec_client_versions}; do
-      client="old_client_${client_version//./_}"
-      unset OUTPUT_PATH
-      container="${client}" _kinit
-      execute_robot_test "${client}" --include test-ec-compat -N "xcompat-cluster-${cluster_version}-client-${client_version}-read-${cluster_version}" -v prefix:"${prefix}" ec/backward-compat.robot
+  for client_version in "$@"; do
+    for data_version in $(echo "$client_version" "$cluster_version" "$current_version" | xargs -n1 | sort -u); do
+      client _read ${data_version}
     done
-
-    KEEP_RUNNING=false stop_docker_env
   done
+
+  KEEP_RUNNING=false stop_docker_env
 }
 
 create_results_dir
 
 # current cluster with various clients
-COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${current_version} test_cross_compatibility ${old_versions}
+COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${current_version} test_cross_compatibility ${old_versions} ${current_version}
 
 # old cluster with clients: same version and current version
 for cluster_version in ${old_versions}; do
   export OZONE_VERSION=${cluster_version}
-  COMPOSE_FILE=old-cluster.yaml:clients.yaml test_cross_compatibility ${cluster_version}
+  COMPOSE_FILE=old-cluster.yaml:clients.yaml test_cross_compatibility ${cluster_version} ${current_version}
 done
-
-test_ec_cross_compatibility