Skip to content

Commit 9d358b9

Browse files
alekstheodalekstheod
authored
Use specific tag_filters for tsan and asan builds (#367)
* Use specific tag_filters for tsan and asan builds * Use script to generate tag_filters
1 parent 5597c0d commit 9d358b9

File tree

5 files changed

+71
-37
lines changed

5 files changed

+71
-37
lines changed

‎build_tools/rocm/rocm_tag_filters.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
TAG_FILTERS=(
4+
-no_gpu
5+
-requires-gpu-intel
6+
-requires-gpu-nvidia
7+
-no_oss
8+
-oss_excluded
9+
-oss_serial
10+
-cuda-only
11+
-oneapi-only
12+
-requires-gpu-sm60
13+
-requires-gpu-sm60-only
14+
-requires-gpu-sm70
15+
-requires-gpu-sm70-only
16+
-requires-gpu-sm80
17+
-requires-gpu-sm80-only
18+
-requires-gpu-sm86
19+
-requires-gpu-sm86-only
20+
-requires-gpu-sm89
21+
-requires-gpu-sm89-only
22+
-requires-gpu-sm90
23+
-requires-gpu-sm90-only
24+
)
25+
26+
echo $(IFS=, ; echo "${TAG_FILTERS[*]}")

‎build_tools/rocm/rocm_xla.bazelrc‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,24 @@
11
# Test-related settings.
22

3-
test:xla_sgpu_filters --test_tag_filters=gpu,requires-gpu-amd,-multi_gpu,-multi_gpu_h100,-requires-gpu-intel,-requires-gpu-nvidia,-no_oss,-oss_excluded,-oss_serial,-no_gpu,-cuda-only,-oneapi-only,-requires-gpu-sm60,-requires-gpu-sm60-only,-requires-gpu-sm70,-requires-gpu-sm70-only,-requires-gpu-sm80,-requires-gpu-sm80-only,-requires-gpu-sm86,-requires-gpu-sm86-only,-requires-gpu-sm89,-requires-gpu-sm89-only,-requires-gpu-sm90,-requires-gpu-sm90-only
4-
test:xla_sgpu_filters --build_tag_filters=gpu,requires-gpu-amd,-multi_gpu,-multi_gpu_h100,-requires-gpu-intel,-requires-gpu-nvidia,-no_oss,-oss_excluded,-oss_serial,-no_gpu,-cuda-only,-oneapi-only,-requires-gpu-sm60,-requires-gpu-sm60-only,-requires-gpu-sm70,-requires-gpu-sm70-only,-requires-gpu-sm80,-requires-gpu-sm80-only,-requires-gpu-sm86,-requires-gpu-sm86-only,-requires-gpu-sm89,-requires-gpu-sm89-only,-requires-gpu-sm90,-requires-gpu-sm90-only
5-
test:xla_sgpu --config=xla_sgpu_filters -- \
3+
build:asan --strip=never
4+
build:asan --copt -fsanitize=address
5+
build:asan --copt -DADDRESS_SANITIZER
6+
build:asan --copt -O1
7+
build:asan --copt -g
8+
build:asan --linkopt -g
9+
build:asan --copt -fno-omit-frame-pointer
10+
build:asan --linkopt -fsanitize=address
11+
build:asan --//build_tools/rocm:sanitizer=asan
12+
13+
build:tsan --strip=never
14+
build:tsan --copt -fsanitize=thread
15+
build:tsan --copt -g
16+
build:tsan --copt -fno-omit-frame-pointer
17+
build:tsan --linkopt -fsanitize=thread
18+
build:tsan --linkopt -g
19+
build:tsan --//build_tools/rocm:sanitizer=tsan
20+
21+
test:xla_sgpu -- \
622
//xla/... \
723
-//xla/backends/gpu/collectives:gpu_clique_key_test \
824
-//xla/backends/gpu/collectives:nccl_communicator_test \
@@ -22,9 +38,7 @@ test:xla_sgpu --config=xla_sgpu_filters -- \
2238
-//xla/pjrt/distributed:topology_util_test \
2339
-//xla/pjrt/distributed:client_server_test
2440

25-
test:xla_mgpu_filters --test_tag_filters=-no_gpu,-requires-gpu-nvidia,-requires-gpu-intel,-no_oss,-oss_excluded,-oss_serial,-cuda-only,-oneapi-only,-requires-gpu-sm60,-requires-gpu-sm60-only,-requires-gpu-sm70,-requires-gpu-sm70-only,-requires-gpu-sm80,-requires-gpu-sm80-only,-requires-gpu-sm86,-requires-gpu-sm86-only,-requires-gpu-sm89,-requires-gpu-sm89-only,-requires-gpu-sm90,-requires-gpu-sm90-only
26-
test:xla_mgpu_filters --build_tag_filters=-no_gpu,-requires-gpu-nvidia,-requires-gpu-intel,-no_oss,-oss_excluded,-oss_serial,-cuda-only,-oneapi-only,-requires-gpu-sm60,-requires-gpu-sm60-only,-requires-gpu-sm70,-requires-gpu-sm70-only,-requires-gpu-sm80,-requires-gpu-sm80-only,-requires-gpu-sm86,-requires-gpu-sm86-only,-requires-gpu-sm89,-requires-gpu-sm89-only,-requires-gpu-sm90,-requires-gpu-sm90-only
27-
test:xla_mgpu --config=xla_mgpu_filters -- \
41+
test:xla_mgpu -- \
2842
//xla/tests:collective_ops_e2e_test \
2943
//xla/tests:collective_ops_test \
3044
//xla/tests:collective_pipeline_parallelism_test \
@@ -47,4 +61,4 @@ test:xla_mgpu --config=xla_mgpu_filters -- \
4761
//xla/tools/multihost_hlo_runner:functional_hlo_runner_test \
4862
//xla/pjrt/distributed:topology_util_test \
4963
//xla/pjrt/distributed:client_server_test \
50-
//xla/backends/gpu/runtime:all_reduce_test
64+
//xla/backends/gpu/runtime:all_reduce_test

‎build_tools/rocm/run_xla.sh‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,26 +68,32 @@ BAZEL_DISK_CACHE_SIZE=100G
6868
BAZEL_DISK_CACHE_DIR="/tf/disk_cache/rocm-jaxlib-v0.6.0"
6969
mkdir -p ${BAZEL_DISK_CACHE_DIR}
7070

71+
SCRIPT_DIR=$(realpath $(dirname $0))
72+
TAG_FILTERS=$($SCRIPT_DIR/rocm_tag_filters.sh),gpu,-multigpu,-multi_gpu_h100,requires-gpu-amd
73+
7174
SANITIZER_ARGS=()
7275
if [[ $1 == "asan" ]]; then
73-
SANITIZER_ARGS+=("--test_env=ASAN_OPTIONS=suppressions=$(realpath $(dirname $0))/asan_ignore_list.txt:use_sigaltstack=0")
74-
SANITIZER_ARGS+=("--test_env=LSAN_OPTIONS=suppressions=$(realpath $(dirname $0))/lsan_ignore_list.txt:use_sigaltstack=0")
76+
SANITIZER_ARGS+=("--test_env=ASAN_OPTIONS=suppressions=${SCRIPT_DIR}/asan_ignore_list.txt:use_sigaltstack=0")
77+
SANITIZER_ARGS+=("--test_env=LSAN_OPTIONS=suppressions=${SCRIPT_DIR}/lsan_ignore_list.txt:use_sigaltstack=0")
7578
SANITIZER_ARGS+=("--config=asan")
79+
TAG_FILTERS=$TAG_FILTERS,-noasan
7680
shift
7781
elif [[ $1 == "tsan" ]]; then
78-
SANITIZER_ARGS+=("--test_env=TSAN_OPTIONS=suppressions=$(realpath $(dirname $0))/tsan_ignore_list.txt::history_size=7:ignore_noninstrumented_modules=1")
82+
SANITIZER_ARGS+=("--test_env=TSAN_OPTIONS=suppressions=${SCRIPT_DIR}/tsan_ignore_list.txt::history_size=7:ignore_noninstrumented_modules=1")
7983
SANITIZER_ARGS+=("--config=tsan")
84+
TAG_FILTERS=$TAG_FILTERS,-notsan
8085
shift
8186
fi
8287

8388
bazel --bazelrc=build_tools/rocm/rocm_xla.bazelrc test \
89+
--config=rocm_ci \
90+
--config=xla_sgpu \
8491
--disk_cache=${BAZEL_DISK_CACHE_DIR} \
8592
--profile=/tf/pkg/profile.json.gz \
8693
--experimental_disk_cache_gc_max_size=${BAZEL_DISK_CACHE_SIZE} \
8794
--experimental_guard_against_concurrent_changes \
88-
"${SANITIZER_ARGS[@]}" \
89-
--config=rocm_ci \
90-
--config=xla_sgpu \
95+
--build_tag_filters=$TAG_FILTERS \
96+
--test_tag_filters=$TAG_FILTERS \
9197
--test_timeout=920,2400,7200,9600 \
9298
--test_sharding_strategy=disabled \
9399
--test_output=errors \
@@ -102,6 +108,7 @@ bazel --bazelrc=build_tools/rocm/rocm_xla.bazelrc test \
102108
--test_env=MIOPEN_FIND_ENFORCE=5 \
103109
--test_env=MIOPEN_FIND_MODE=1 \
104110
--test_filter=-$(IFS=: ; echo "${EXCLUDED_TESTS[*]}") \
111+
"${SANITIZER_ARGS[@]}" \
105112
"$@"
106113

107114
# clean up bazel disk_cache

‎build_tools/rocm/run_xla_multi_gpu.sh‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,30 +64,34 @@ EXCLUDED_TESTS=(
6464
RaggedAllToAllTest/RaggedAllToAllTest.RaggedAllToAll_8GPUs_2ReplicasPerGroups/async_decomposer
6565
)
6666

67+
SCRIPT_DIR=$(realpath $(dirname $0))
68+
TAG_FILTERS="$($SCRIPT_DIR/rocm_tag_filters.sh)"
69+
6770
SANITIZER_ARGS=()
6871
if [[ $1 == "asan" ]]; then
69-
SANITIZER_ARGS+=("--test_env=ASAN_OPTIONS=suppressions=$(realpath $(dirname $0))/asan_ignore_list.txt:use_sigaltstack=0")
70-
SANITIZER_ARGS+=("--test_env=LSAN_OPTIONS=suppressions=$(realpath $(dirname $0))/lsan_ignore_list.txt:use_sigaltstack=0")
72+
SANITIZER_ARGS+=("--test_env=ASAN_OPTIONS=suppressions=${SCRIPT_DIR}/asan_ignore_list.txt:use_sigaltstack=0")
73+
SANITIZER_ARGS+=("--test_env=LSAN_OPTIONS=suppressions=${SCRIPT_DIR}/lsan_ignore_list.txt:use_sigaltstack=0")
7174
SANITIZER_ARGS+=("--run_under=//build_tools/rocm:sanitizer_wrapper")
7275
SANITIZER_ARGS+=("--config=asan")
76+
TAG_FILTERS="$TAG_FILTERS,-noasan"
7377
shift
7478
elif [[ $1 == "tsan" ]]; then
75-
SANITIZER_ARGS+=("--test_env=TSAN_OPTIONS=suppressions=$(realpath $(dirname $0))/tsan_ignore_list.txt::history_size=7:ignore_noninstrumented_modules=1")
79+
SANITIZER_ARGS+=("--test_env=TSAN_OPTIONS=suppressions=${SCRIPT_DIR}/tsan_ignore_list.txt::history_size=7:ignore_noninstrumented_modules=1")
7680
SANITIZER_ARGS+=("--run_under=//build_tools/rocm:sanitizer_wrapper")
7781
SANITIZER_ARGS+=("--config=tsan")
82+
TAG_FILTERS="$TAG_FILTERS,-notsan"
7883
shift
7984
fi
8085

8186
bazel --bazelrc=build_tools/rocm/rocm_xla.bazelrc test \
82-
"${SANITIZER_ARGS[@]}" \
8387
--config=rocm_ci \
8488
--config=xla_mgpu \
89+
--build_tag_filters=${TAG_FILTERS} \
90+
--test_tag_filters=${TAG_FILTERS} \
8591
--profile=/tf/pkg/profile.json.gz \
8692
--disk_cache=${BAZEL_DISK_CACHE_DIR} \
8793
--experimental_disk_cache_gc_max_size=${BAZEL_DISK_CACHE_SIZE} \
8894
--experimental_guard_against_concurrent_changes \
89-
--build_tag_filters=${TAGS_FILTER} \
90-
--test_tag_filters=${TAGS_FILTER} \
9195
--test_timeout=920,2400,7200,9600 \
9296
--test_sharding_strategy=disabled \
9397
--test_output=errors \
@@ -99,6 +103,7 @@ bazel --bazelrc=build_tools/rocm/rocm_xla.bazelrc test \
99103
--action_env=XLA_FLAGS=--xla_gpu_enable_llvm_module_compilation_parallelism=true \
100104
--action_env=NCCL_MAX_NCHANNELS=1 \
101105
--test_filter=-$(IFS=: ; echo "${EXCLUDED_TESTS[*]}") \
106+
"${SANITIZER_ARGS[@]}" \
102107
"$@"
103108

104109
# clean up bazel disk_cache

‎tensorflow.bazelrc‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -216,24 +216,6 @@ build:dbg --per_file_copt=+.*,-xla.*@-g0
216216
# AWS SDK must be compiled in release mode. see: https://github.com/tensorflow/tensorflow/issues/37498
217217
build:dbg --copt -DDEBUG_BUILD
218218

219-
build:asan --strip=never
220-
build:asan --copt -fsanitize=address
221-
build:asan --copt -DADDRESS_SANITIZER
222-
build:asan --copt -O1
223-
build:asan --copt -g
224-
build:tsan --linkopt -g
225-
build:asan --copt -fno-omit-frame-pointer
226-
build:asan --linkopt -fsanitize=address
227-
build:asan --//build_tools/rocm:sanitizer=asan
228-
229-
build:tsan --strip=never
230-
build:tsan --copt -fsanitize=thread
231-
build:tsan --copt -g
232-
build:tsan --copt -fno-omit-frame-pointer
233-
build:tsan --linkopt -fsanitize=thread
234-
build:tsan --linkopt -g
235-
build:tsan --//build_tools/rocm:sanitizer=tsan
236-
237219
build:rocm_base --copt=-Wno-gnu-offsetof-extensions
238220
build:rocm_base --crosstool_top=@local_config_rocm//crosstool:toolchain
239221
build:rocm_base --define=using_rocm_hipcc=true

0 commit comments

Comments
 (0)