From 4c8c417ff4e7297dfd77d2703f0b7b866c8d8f42 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 22 Oct 2020 14:45:46 -0400 Subject: [PATCH 01/19] Test TF GPU CI --- .github/workflows/self-push.yml | 112 +++++++++++++++++-- .github/workflows/self-scheduled.yml | 157 ++++++++++++++++++++++----- 2 files changed, 233 insertions(+), 36 deletions(-) diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index 8a9a25bc0aa1..e5a517ba4deb 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -4,7 +4,8 @@ on: push: branches: - master - paths: + - ci_* + paths: - "src/**" - "tests/**" - ".github/**" @@ -13,8 +14,8 @@ on: jobs: - run_tests_torch_and_tf_gpu: - runs-on: [self-hosted, single-gpu] + run_tests_torch_and_gpu: + runs-on: [self-hosted, gpu-tests] steps: - uses: actions/checkout@v2 - name: Python version @@ -58,16 +59,62 @@ jobs: - name: Run all non-slow tests on GPU env: - TF_FORCE_GPU_ALLOW_GROWTH: "true" - # TF_GPU_MEMORY_LIMIT: 4096 OMP_NUM_THREADS: 1 + CUDA_VISIBLE_DEVICES: 0 run: | source .env/bin/activate - python -m pytest -n 2 --dist=loadfile -s tests + python -m pytest -n 2 --dist=loadfile -s ./tests/ + + run_tests_tf_gpu: + runs-on: [self-hosted, gpu-tests] + steps: + - uses: actions/checkout@v2 + - name: Python version + run: | + which python + python --version + pip --version + - name: Current dir + run: pwd + - run: nvidia-smi + + - name: Loading cache. + uses: actions/cache@v2 + id: cache + with: + path: .env + key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }} + + - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) + run: | + python -m venv .env + source .env/bin/activate + which python + python --version + pip --version + - name: Install dependencies + run: | + source .env/bin/activate + pip install --upgrade pip + pip install .[tf,sklearn,testing,onnxruntime] + pip install git+https://github.com/huggingface/datasets + - name: Are GPUs recognized by our DL frameworks + run: | + source .env/bin/activate + TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))" + TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))" + + - name: Run all non-slow tests on GPU + env: + OMP_NUM_THREADS: 1 + CUDA_VISIBLE_DEVICES: 0 + run: | + source .env/bin/activate + python -m pytest -n 2 --dist=loadfile -s ./tests/ - run_tests_torch_and_tf_multiple_gpu: - runs-on: [self-hosted, multi-gpu] + run_tests_torch_multiple_gpu: + runs-on: [self-hosted, multi-gpu-tests] steps: - uses: actions/checkout@v2 - name: Python version @@ -109,8 +156,53 @@ jobs: - name: Run all non-slow tests on GPU env: - TF_FORCE_GPU_ALLOW_GROWTH: "true" - # TF_GPU_MEMORY_LIMIT: 4096 + OMP_NUM_THREADS: 1 + run: | + source .env/bin/activate + python -m pytest -n 2 --dist=loadfile -s ./tests/ + + run_tests_tf_multiple_gpu: + runs-on: [self-hosted, multi-gpu-tests] + steps: + - uses: actions/checkout@v2 + - name: Python version + run: | + which python + python --version + pip --version + - name: Current dir + run: pwd + - run: nvidia-smi + + - name: Loading cache. + uses: actions/cache@v2 + id: cache + with: + path: .env + key: v0-tests_tf_torch_multiple_gpu-${{ hashFiles('setup.py') }} + + - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) + run: | + python -m venv .env + source .env/bin/activate + which python + python --version + pip --version + - name: Install dependencies + run: | + source .env/bin/activate + pip install --upgrade pip + pip install .[tf,sklearn,testing,onnxruntime] + pip install git+https://github.com/huggingface/datasets + + - name: Are GPUs recognized by our DL frameworks + run: | + source .env/bin/activate + TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))" + TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))" + + - name: Run all non-slow tests on GPU + env: OMP_NUM_THREADS: 1 run: | source .env/bin/activate diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 451a5c0032a4..7bbe80905cd2 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -9,8 +9,8 @@ on: - cron: "0 0 * * *" jobs: - run_all_tests_torch_and_tf_gpu: - runs-on: [self-hosted, single-gpu] + run_all_tests_torch_gpu: + runs-on: [self-hosted, gpu-tests] steps: - uses: actions/checkout@v2 @@ -54,44 +54,152 @@ jobs: python -c "import torch; print('Cuda available:', torch.cuda.is_available())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" + - name: Run all tests on GPU + env: + OMP_NUM_THREADS: 1 + RUN_SLOW: yes + run: | + source .env/bin/activate + python -m pytest -n 1 --dist=loadfile -s --make_reports=tests tests + + - name: Run examples tests on GPU + env: + OMP_NUM_THREADS: 1 + RUN_SLOW: yes + CUDA_VISIBLE_DEVICES: 0 + run: | + source .env/bin/activate + pip install -r examples/requirements.txt + python -m pytest -n 1 --dist=loadfile -s examples --durations=0 + python -m pytest -n 1 --dist=loadfile -s --make_reports=examples examples + + - name: test suite reports artifacts + uses: actions/upload-artifact@v2 + with: + name: test_reports + path: reports + + + run_all_tests_tf_gpu: + runs-on: [self-hosted, gpu-tests] + steps: + - uses: actions/checkout@v2 + + - name: Loading cache. + uses: actions/cache@v2 + id: cache + with: + path: .env + key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }} + + - name: Python version + run: | + which python + python --version + pip --version + - name: Current dir + run: pwd + - run: nvidia-smi + - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) + if: steps.cache.outputs.cache-hit != 'true' + run: | + python -m venv .env + source .env/bin/activate + which python + python --version + pip --version + - name: Install dependencies + run: | + source .env/bin/activate + pip install --upgrade pip + pip install .[tf,sklearn,testing,onnxruntime] + pip install git+https://github.com/huggingface/datasets + + - name: Are GPUs recognized by our DL frameworks + run: | + source .env/bin/activate + TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))" + TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))" + - name: Run all tests on GPU env: - TF_FORCE_GPU_ALLOW_GROWTH: "true" OMP_NUM_THREADS: 1 RUN_SLOW: yes run: | source .env/bin/activate - python -m pytest -n 1 --dist=loadfile -s --make_reports=tests tests + python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0 - name: Run examples tests on GPU env: - TF_FORCE_GPU_ALLOW_GROWTH: "true" OMP_NUM_THREADS: 1 RUN_SLOW: yes + CUDA_VISIBLE_DEVICES: 0 run: | source .env/bin/activate pip install -r examples/requirements.txt - python -m pytest -n 1 --dist=loadfile -s --make_reports=examples examples + python -m pytest -n 1 --dist=loadfile -s examples --durations=0 - - name: Run all pipeline tests on GPU + run_all_tests_torch_multiple_gpu: + runs-on: [self-hosted, multi-gpu-tests] + steps: + - uses: actions/checkout@v2 + + - name: Loading cache. + uses: actions/cache@v2 + id: cache + with: + path: .env + key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }} + + - name: Python version + run: | + which python + python --version + pip --version + - name: Current dir + run: pwd + - run: nvidia-smi + - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) + if: steps.cache.outputs.cache-hit != 'true' + run: | + python -m venv .env + source .env/bin/activate + which python + python --version + pip --version + - name: Install dependencies + run: | + source .env/bin/activate + pip install --upgrade pip + pip install torch!=1.6.0 + pip install .[sklearn,testing,onnxruntime] + pip install git+https://github.com/huggingface/datasets + + - name: Are GPUs recognized by our DL frameworks + run: | + source .env/bin/activate + python -c "import torch; print('Cuda available:', torch.cuda.is_available())" + python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" + + - name: Run all tests on GPU env: - TF_FORCE_GPU_ALLOW_GROWTH: "true" OMP_NUM_THREADS: 1 RUN_SLOW: yes - RUN_PIPELINE_TESTS: yes run: | source .env/bin/activate - python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_pipeline tests - - - name: test suite reports artifacts - uses: actions/upload-artifact@v2 - with: - name: test_reports - path: reports + python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=50 + - name: Run examples tests on GPU + env: + OMP_NUM_THREADS: 1 + RUN_SLOW: yes + run: | + source .env/bin/activate + pip install -r examples/requirements.txt + python -m pytest -n 1 --dist=loadfile -s examples --durations=50 - run_all_tests_torch_and_tf_multiple_gpu: - runs-on: [self-hosted, multi-gpu] + run_all_tests_tf_multiple_gpu: + runs-on: [self-hosted, multi-gpu-tests] steps: - uses: actions/checkout@v2 @@ -122,34 +230,31 @@ jobs: run: | source .env/bin/activate pip install --upgrade pip - pip install torch!=1.6.0 - pip install .[sklearn,testing,onnxruntime] + pip install .[tf,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets - name: Are GPUs recognized by our DL frameworks run: | source .env/bin/activate - python -c "import torch; print('Cuda available:', torch.cuda.is_available())" - python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" + TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))" + TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))" - name: Run all tests on GPU env: - TF_FORCE_GPU_ALLOW_GROWTH: "true" OMP_NUM_THREADS: 1 RUN_SLOW: yes run: | source .env/bin/activate - python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=50 + python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0 - name: Run examples tests on GPU env: - TF_FORCE_GPU_ALLOW_GROWTH: "true" OMP_NUM_THREADS: 1 RUN_SLOW: yes run: | source .env/bin/activate pip install -r examples/requirements.txt - python -m pytest -n 1 --dist=loadfile -s examples --durations=50 + python -m pytest -n 1 --dist=loadfile -s examples --durations=0 - name: Run all pipeline tests on GPU env: From 2ee3d490bdb45ff20f544656ff0a3cad9c3421b2 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 23 Oct 2020 06:18:17 -0400 Subject: [PATCH 02/19] Change cache --- .github/workflows/self-push.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index e5a517ba4deb..411e6d13d28d 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -14,7 +14,7 @@ on: jobs: - run_tests_torch_and_gpu: + run_tests_torch_gpu: runs-on: [self-hosted, gpu-tests] steps: - uses: actions/checkout@v2 @@ -28,12 +28,12 @@ jobs: run: pwd - run: nvidia-smi - - name: Loading cache. - uses: actions/cache@v2 - id: cache - with: - path: .env - key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }} + - name: Loading cache. + uses: actions/cache@v2 + id: cache + with: + path: .env + key: v0-tests_torch_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | @@ -83,7 +83,7 @@ jobs: id: cache with: path: .env - key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }} + key: v0-tests_tf_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | @@ -131,7 +131,7 @@ jobs: id: cache with: path: .env - key: v0-tests_tf_torch_multiple_gpu-${{ hashFiles('setup.py') }} + key: v0-tests_torch_multiple_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | @@ -179,7 +179,7 @@ jobs: id: cache with: path: .env - key: v0-tests_tf_torch_multiple_gpu-${{ hashFiles('setup.py') }} + key: v0-tests_tf_multiple_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | From 6b1ee3ba383ce7d7eb7b473d646648095901d420 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 23 Oct 2020 16:37:06 -0400 Subject: [PATCH 03/19] Fix missing torch requirement --- tests/test_modeling_marian.py | 1 + tests/test_modeling_prophetnet.py | 1 + tests/test_modeling_roberta.py | 1 + tests/test_modeling_squeezebert.py | 1 + tests/test_modeling_transfo_xl.py | 1 + tests/test_modeling_xlm_prophetnet.py | 4 ++-- tests/test_modeling_xlm_roberta.py | 4 ++-- 7 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_modeling_marian.py b/tests/test_modeling_marian.py index 3859f4348248..af5f73fd0233 100644 --- a/tests/test_modeling_marian.py +++ b/tests/test_modeling_marian.py @@ -69,6 +69,7 @@ def setUp(self): class ModelManagementTests(unittest.TestCase): @slow + @require_torch def test_model_names(self): model_list = HfApi().model_list() model_ids = [x.modelId for x in model_list if x.modelId.startswith(ORG_NAME)] diff --git a/tests/test_modeling_prophetnet.py b/tests/test_modeling_prophetnet.py index 55336c5d2fe6..f74123f3d133 100644 --- a/tests/test_modeling_prophetnet.py +++ b/tests/test_modeling_prophetnet.py @@ -959,6 +959,7 @@ def test_config(self): self.config_tester.run_common_tests() +@require_torch class ProphetNetModelIntegrationTest(unittest.TestCase): @slow def test_pretrained_checkpoint_hidden_states(self): diff --git a/tests/test_modeling_roberta.py b/tests/test_modeling_roberta.py index 5c0ec2709f30..5ab097b5ff48 100644 --- a/tests/test_modeling_roberta.py +++ b/tests/test_modeling_roberta.py @@ -396,6 +396,7 @@ def test_create_position_ids_from_inputs_embeds(self): @require_sentencepiece @require_tokenizers +@require_torch class RobertaModelIntegrationTest(unittest.TestCase): @slow def test_inference_masked_lm(self): diff --git a/tests/test_modeling_squeezebert.py b/tests/test_modeling_squeezebert.py index 966f4771c585..f257f1de8d30 100644 --- a/tests/test_modeling_squeezebert.py +++ b/tests/test_modeling_squeezebert.py @@ -273,6 +273,7 @@ def test_model_from_pretrained(self): @require_sentencepiece @require_tokenizers +@require_torch class SqueezeBertModelIntegrationTest(unittest.TestCase): @slow def test_inference_classification_head(self): diff --git a/tests/test_modeling_transfo_xl.py b/tests/test_modeling_transfo_xl.py index 5f16a1a4e224..24b1787c8912 100644 --- a/tests/test_modeling_transfo_xl.py +++ b/tests/test_modeling_transfo_xl.py @@ -279,6 +279,7 @@ def test_resize_tokens_embeddings(self): self.assertEqual(model_embed.emb_layers[layer].weight.shape[0], cloned_embeddings[layer].shape[0]) +@require_torch class TransfoXLModelLanguageGenerationTest(unittest.TestCase): @slow def test_lm_generate_transfo_xl_wt103(self): diff --git a/tests/test_modeling_xlm_prophetnet.py b/tests/test_modeling_xlm_prophetnet.py index 4f2c2558a92a..cb353c42cb09 100644 --- a/tests/test_modeling_xlm_prophetnet.py +++ b/tests/test_modeling_xlm_prophetnet.py @@ -17,8 +17,7 @@ import unittest from transformers import is_torch_available -from transformers.testing_utils import slow, torch_device - +from transformers.testing_utils import slow, torch_device, require_torch if is_torch_available(): import torch @@ -26,6 +25,7 @@ from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer +@require_torch class XLMProphetNetModelIntegrationTest(unittest.TestCase): @slow def test_pretrained_checkpoint_hidden_states(self): diff --git a/tests/test_modeling_xlm_roberta.py b/tests/test_modeling_xlm_roberta.py index abafeec6ef28..d5eeb829ba28 100644 --- a/tests/test_modeling_xlm_roberta.py +++ b/tests/test_modeling_xlm_roberta.py @@ -17,8 +17,7 @@ import unittest from transformers import is_torch_available -from transformers.testing_utils import require_sentencepiece, require_tokenizers, slow - +from transformers.testing_utils import require_sentencepiece, require_tokenizers, slow, require_torch if is_torch_available(): import torch @@ -28,6 +27,7 @@ @require_sentencepiece @require_tokenizers +@require_torch class XLMRobertaModelIntegrationTest(unittest.TestCase): @slow def test_xlm_roberta_base(self): From 3ed3d307ff78e22a243bd06d9ef3cd4444db91db Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 23 Oct 2020 19:35:12 -0400 Subject: [PATCH 04/19] Fix some model tests Style --- src/transformers/modeling_tf_albert.py | 43 +++++++++++++++----------- src/transformers/modeling_tf_funnel.py | 2 +- tests/test_modeling_tf_camembert.py | 2 +- tests/test_modeling_tf_common.py | 30 +++++++++--------- tests/test_modeling_tf_flaubert.py | 8 +++++ tests/test_modeling_xlm_prophetnet.py | 3 +- tests/test_modeling_xlm_roberta.py | 3 +- 7 files changed, 53 insertions(+), 38 deletions(-) diff --git a/src/transformers/modeling_tf_albert.py b/src/transformers/modeling_tf_albert.py index c30f876cd490..968b9545d3ad 100644 --- a/src/transformers/modeling_tf_albert.py +++ b/src/transformers/modeling_tf_albert.py @@ -79,25 +79,31 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super().__init__(**kwargs) - self.config = config self.vocab_size = config.vocab_size + self.embedding_size = config.embedding_size + self.initializer_range = config.initializer_range + self.max_position_embeddings = config.max_position_embeddings + self.type_vocab_size = config.type_vocab_size + self.layer_norm_eps = config.layer_norm_eps + self.hidden_dropout_prob = config.hidden_dropout_prob + self.position_embeddings = tf.keras.layers.Embedding( - config.max_position_embeddings, - config.embedding_size, - embeddings_initializer=get_initializer(self.config.initializer_range), + self.max_position_embeddings, + self.embedding_size, + embeddings_initializer=get_initializer(self.initializer_range), name="position_embeddings", ) self.token_type_embeddings = tf.keras.layers.Embedding( - config.type_vocab_size, - config.embedding_size, - embeddings_initializer=get_initializer(self.config.initializer_range), + self.type_vocab_size, + self.embedding_size, + embeddings_initializer=get_initializer(self.initializer_range), name="token_type_embeddings", ) # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load # any TensorFlow checkpoint file - self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm") - self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=self.layer_norm_eps, name="LayerNorm") + self.dropout = tf.keras.layers.Dropout(self.hidden_dropout_prob) def build(self, input_shape): """Build shared word embedding layer """ @@ -106,8 +112,8 @@ def build(self, input_shape): # arbitrarily, and works well. self.word_embeddings = self.add_weight( "weight", - shape=[self.config.vocab_size, self.config.embedding_size], - initializer=get_initializer(self.config.initializer_range), + shape=[self.vocab_size, self.embedding_size], + initializer=get_initializer(self.initializer_range), ) super().build(input_shape) @@ -182,9 +188,9 @@ def _linear(self, inputs): """ batch_size = shape_list(inputs)[0] length = shape_list(inputs)[1] - x = tf.reshape(inputs, [-1, self.config.embedding_size]) + x = tf.reshape(inputs, [-1, self.embedding_size]) logits = tf.matmul(x, self.word_embeddings, transpose_b=True) - return tf.reshape(logits, [batch_size, length, self.config.vocab_size]) + return tf.reshape(logits, [batch_size, length, self.vocab_size]) class TFAlbertSelfOutput(tf.keras.layers.Layer): @@ -207,7 +213,7 @@ class TFAlbertAttention(tf.keras.layers.Layer): """ Contains the complete attention sublayer, including both dropouts and layer norm. """ def __init__(self, config, **kwargs): - super().__init__(config, **kwargs) + super().__init__(**kwargs) self.hidden_size = config.hidden_size self.output_attentions = config.output_attentions @@ -371,7 +377,8 @@ class TFAlbertTransformer(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super().__init__(**kwargs) - self.config = config + self.num_hidden_layers = config.num_hidden_layers + self.num_hidden_groups = config.num_hidden_groups self.embedding_hidden_mapping_in = tf.keras.layers.Dense( config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), @@ -396,12 +403,12 @@ def call( all_attentions = () if output_attentions else None all_hidden_states = (hidden_states,) if output_hidden_states else None - for i in range(self.config.num_hidden_layers): + for i in range(self.num_hidden_layers): # Number of layers in a hidden group - layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups) + layers_per_group = int(self.num_hidden_layers / self.num_hidden_groups) # Index of the hidden group - group_idx = int(i / (self.config.num_hidden_layers / self.config.num_hidden_groups)) + group_idx = int(i / (self.num_hidden_layers / self.num_hidden_groups)) layer_group_output = self.albert_layer_groups[group_idx]( hidden_states, diff --git a/src/transformers/modeling_tf_funnel.py b/src/transformers/modeling_tf_funnel.py index 9cbde8190d1e..7bc0022c3bb4 100644 --- a/src/transformers/modeling_tf_funnel.py +++ b/src/transformers/modeling_tf_funnel.py @@ -307,7 +307,7 @@ def relative_pos(self, pos, stride, pooled_pos=None, shift=1): pooled_pos = pos ref_point = pooled_pos[0] - pos[0] - num_remove = shift * len(pooled_pos) + num_remove = shift * pooled_pos.shape[0] max_dist = ref_point + num_remove * stride min_dist = pooled_pos[0] - pos[-1] diff --git a/tests/test_modeling_tf_camembert.py b/tests/test_modeling_tf_camembert.py index cfd96fe56e9d..92caa29a6439 100644 --- a/tests/test_modeling_tf_camembert.py +++ b/tests/test_modeling_tf_camembert.py @@ -39,7 +39,7 @@ def test_output_embeds_base_model(self): dtype=tf.int32, ) # J'aime le camembert !" - output = model(input_ids)["last_hidden_state"] + output = model(input_ids, return_dict=True)["last_hidden_state"] expected_shape = tf.TensorShape((1, 10, 768)) self.assertEqual(output.shape, expected_shape) # compare the actual values for a slice. diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 75ff72712997..65b004dbb9c2 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -165,16 +165,16 @@ def test_saved_model_with_hidden_states_output(self): config.output_hidden_states = True for model_class in self.all_model_classes: - inputs_dict = self._prepare_for_class(inputs_dict, model_class) + class_inputs_dict = self._prepare_for_class(inputs_dict, model_class) model = model_class(config) - num_out = len(model(inputs_dict)) + num_out = len(model(class_inputs_dict)) model._saved_model_inputs_spec = None - model._set_save_spec(inputs_dict) + model._set_save_spec(class_inputs_dict) with tempfile.TemporaryDirectory() as tmpdirname: tf.saved_model.save(model, tmpdirname) model = tf.keras.models.load_model(tmpdirname) - outputs = model(inputs_dict) + outputs = model(class_inputs_dict) if self.is_encoder_decoder: output = outputs["encoder_hidden_states"] if isinstance(outputs, dict) else outputs[-1] @@ -183,7 +183,10 @@ def test_saved_model_with_hidden_states_output(self): hidden_states = [t.numpy() for t in output] self.assertEqual(len(outputs), num_out) - self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) + expected_num_layers = getattr( + self.model_tester, "expected_num_hidden_layers", self.model_tester.num_hidden_layers + 1 + ) + self.assertEqual(len(hidden_states), expected_num_layers) self.assertListEqual( list(hidden_states[0].shape[-2:]), [self.model_tester.seq_length, self.model_tester.hidden_size], @@ -193,26 +196,21 @@ def test_saved_model_with_hidden_states_output(self): def test_saved_model_with_attentions_output(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() config.output_attentions = True - encoder_seq_length = ( - self.model_tester.encoder_seq_length - if hasattr(self.model_tester, "encoder_seq_length") - else self.model_tester.seq_length - ) - encoder_key_length = ( - self.model_tester.key_length if hasattr(self.model_tester, "key_length") else encoder_seq_length - ) + + encoder_seq_length = getattr(self.model_tester, "encoder_seq_length", self.model_tester.seq_length) + encoder_key_length = getattr(self.model_tester, "key_length", encoder_seq_length) for model_class in self.all_model_classes: - inputs_dict = self._prepare_for_class(inputs_dict, model_class) + class_inputs_dict = self._prepare_for_class(inputs_dict, model_class) model = model_class(config) num_out = len(model(inputs_dict)) model._saved_model_inputs_spec = None - model._set_save_spec(inputs_dict) + model._set_save_spec(class_inputs_dict) with tempfile.TemporaryDirectory() as tmpdirname: tf.saved_model.save(model, tmpdirname) model = tf.keras.models.load_model(tmpdirname) - outputs = model(inputs_dict) + outputs = model(class_inputs_dict) if self.is_encoder_decoder: output = outputs["encoder_attentions"] if isinstance(outputs, dict) else outputs[-1] diff --git a/tests/test_modeling_tf_flaubert.py b/tests/test_modeling_tf_flaubert.py index 1d7a533067cd..77f04266662a 100644 --- a/tests/test_modeling_tf_flaubert.py +++ b/tests/test_modeling_tf_flaubert.py @@ -330,6 +330,14 @@ def test_model_from_pretrained(self): model = TFFlaubertModel.from_pretrained(model_name) self.assertIsNotNone(model) + def test_saved_model_with_hidden_states_output(self): + # Should be uncommented during patrick TF refactor + pass + + def test_saved_model_with_attentions_output(self): + # Should be uncommented during patrick TF refactor + pass + @require_tf @require_sentencepiece diff --git a/tests/test_modeling_xlm_prophetnet.py b/tests/test_modeling_xlm_prophetnet.py index cb353c42cb09..51e8502b9bd5 100644 --- a/tests/test_modeling_xlm_prophetnet.py +++ b/tests/test_modeling_xlm_prophetnet.py @@ -17,7 +17,8 @@ import unittest from transformers import is_torch_available -from transformers.testing_utils import slow, torch_device, require_torch +from transformers.testing_utils import require_torch, slow, torch_device + if is_torch_available(): import torch diff --git a/tests/test_modeling_xlm_roberta.py b/tests/test_modeling_xlm_roberta.py index d5eeb829ba28..ef22e325c95a 100644 --- a/tests/test_modeling_xlm_roberta.py +++ b/tests/test_modeling_xlm_roberta.py @@ -17,7 +17,8 @@ import unittest from transformers import is_torch_available -from transformers.testing_utils import require_sentencepiece, require_tokenizers, slow, require_torch +from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow + if is_torch_available(): import torch From ae66c17e92813dc0e68088782e6fa25aabc29e7d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 26 Oct 2020 17:39:39 -0400 Subject: [PATCH 05/19] LXMERT --- tests/test_modeling_tf_lxmert.py | 74 ++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/test_modeling_tf_lxmert.py b/tests/test_modeling_tf_lxmert.py index 89c67c929065..a9072e3dd176 100644 --- a/tests/test_modeling_tf_lxmert.py +++ b/tests/test_modeling_tf_lxmert.py @@ -678,3 +678,77 @@ def test_compile_tf_model(self): # Compile extended model extended_model = tf.keras.Model(inputs=[input_ids, visual_feats, visual_pos], outputs=[outputs]) extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) + + @slow + def test_saved_model_with_hidden_states_output(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.output_hidden_states = True + + for model_class in self.all_model_classes: + class_inputs_dict = self._prepare_for_class(inputs_dict, model_class) + model = model_class(config) + num_out = len(model(class_inputs_dict)) + model._saved_model_inputs_spec = None + model._set_save_spec(class_inputs_dict) + + with tempfile.TemporaryDirectory() as tmpdirname: + tf.saved_model.save(model, tmpdirname) + model = tf.keras.models.load_model(tmpdirname) + outputs = model(class_inputs_dict) + + language_hidden_states, vision_hidden_states = outputs[-2], outputs[-1] + + self.assertEqual(len(language_hidden_states), self.model_tester.num_hidden_layers["language"] + 1) + self.assertEqual(len(vision_hidden_states), self.model_tester.num_hidden_layers["vision"] + 1) + + seq_length = self.model_tester.seq_length + num_visual_features = self.model_tester.num_visual_features + + self.assertListEqual( + list(language_hidden_states[0].shape[-2:]), + [seq_length, self.model_tester.hidden_size], + ) + self.assertListEqual( + list(vision_hidden_states[0].shape[-2:]), + [num_visual_features, self.model_tester.hidden_size], + ) + + @slow + def test_saved_model_with_attentions_output(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.output_attentions = True + + encoder_seq_length = getattr(self.model_tester, "encoder_seq_length", self.model_tester.seq_length) + encoder_key_length = getattr(self.model_tester, "key_length", encoder_seq_length) + + for model_class in self.all_model_classes: + class_inputs_dict = self._prepare_for_class(inputs_dict, model_class) + model = model_class(config) + num_out = len(model(inputs_dict)) + model._saved_model_inputs_spec = None + model._set_save_spec(class_inputs_dict) + + with tempfile.TemporaryDirectory() as tmpdirname: + tf.saved_model.save(model, tmpdirname) + model = tf.keras.models.load_model(tmpdirname) + outputs = model(class_inputs_dict) + + language_attentions, vision_attentions, cross_encoder_attentions = (outputs[-3], outputs[-2], outputs[-1]) + + self.assertEqual(len(language_attentions), self.model_tester.num_hidden_layers["language"]) + self.assertEqual(len(vision_attentions), self.model_tester.num_hidden_layers["vision"]) + self.assertEqual(len(cross_encoder_attentions), self.model_tester.num_hidden_layers["cross_encoder"]) + + attentions = [language_attentions, vision_attentions, cross_encoder_attentions] + attention_shapes = [ + [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length], + [ + self.model_tester.num_attention_heads, + self.model_tester.num_visual_features, + self.model_tester.num_visual_features, + ], + [self.model_tester.num_attention_heads, encoder_key_length, self.model_tester.num_visual_features], + ] + + for attention, attention_shape in zip(attentions, attention_shapes): + self.assertListEqual(list(attention[0].shape[-3:]), attention_shape) From c2a74030004c776ed3ed856dbdf66a43ccdafd06 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 26 Oct 2020 17:39:49 -0400 Subject: [PATCH 06/19] MobileBERT --- tests/test_modeling_tf_mobilebert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_modeling_tf_mobilebert.py b/tests/test_modeling_tf_mobilebert.py index 61af8e32c521..d40803160c14 100644 --- a/tests/test_modeling_tf_mobilebert.py +++ b/tests/test_modeling_tf_mobilebert.py @@ -287,6 +287,6 @@ def test_for_token_classification(self): @slow def test_model_from_pretrained(self): # for model_name in TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: - for model_name in ["mobilebert-uncased"]: + for model_name in ["google/mobilebert-uncased"]: model = TFMobileBertModel.from_pretrained(model_name) self.assertIsNotNone(model) From eb6d7b500c3fcc7f95e600bf9172c0d0fe0dc0d7 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 27 Oct 2020 16:06:35 -0400 Subject: [PATCH 07/19] Longformer skip test --- tests/test_modeling_tf_common.py | 2 +- tests/test_modeling_tf_longformer.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 65b004dbb9c2..1f966922d06f 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -203,7 +203,7 @@ def test_saved_model_with_attentions_output(self): for model_class in self.all_model_classes: class_inputs_dict = self._prepare_for_class(inputs_dict, model_class) model = model_class(config) - num_out = len(model(inputs_dict)) + num_out = len(model(class_inputs_dict)) model._saved_model_inputs_spec = None model._set_save_spec(class_inputs_dict) diff --git a/tests/test_modeling_tf_longformer.py b/tests/test_modeling_tf_longformer.py index 75e9355eb565..0f07dc780f1d 100644 --- a/tests/test_modeling_tf_longformer.py +++ b/tests/test_modeling_tf_longformer.py @@ -302,6 +302,10 @@ def test_longformer_for_question_answering(self): config_and_inputs = self.model_tester.prepare_config_and_inputs_for_question_answering() self.model_tester.create_and_check_longformer_for_question_answering(*config_and_inputs) + @slow + def test_saved_model_with_attentions_output(self): + pass + @require_tf @require_sentencepiece From ba27b9b97ea4297817ff5e288ba58c536d8583ff Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 27 Oct 2020 16:44:31 -0400 Subject: [PATCH 08/19] XLNet --- src/transformers/modeling_tf_xlnet.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/transformers/modeling_tf_xlnet.py b/src/transformers/modeling_tf_xlnet.py index a48ae6b3465b..f1849342879d 100644 --- a/src/transformers/modeling_tf_xlnet.py +++ b/src/transformers/modeling_tf_xlnet.py @@ -1257,17 +1257,17 @@ def call( transformer_outputs = self.transformer( inputs, - attention_mask=None, - mems=None, - perm_mask=None, - target_mapping=None, - token_type_ids=None, - input_mask=None, - head_mask=None, - inputs_embeds=None, - use_cache=True, - output_attentions=None, - output_hidden_states=None, + attention_mask=attention_mask, + mems=mems, + perm_mask=perm_mask, + target_mapping=target_mapping, + token_type_ids=token_type_ids, + input_mask=input_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, ) From 29729e7299b18f0bea8147c3460e0fb0ad7450d3 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 27 Oct 2020 17:29:23 -0400 Subject: [PATCH 09/19] The rest of the tests --- src/transformers/modeling_tf_t5.py | 4 +--- tests/test_modeling_tf_common.py | 2 +- tests/test_modeling_tf_lxmert.py | 6 +++++- tests/test_modeling_tf_t5.py | 11 +++++++++-- tests/test_modeling_tf_xlm_roberta.py | 2 +- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/transformers/modeling_tf_t5.py b/src/transformers/modeling_tf_t5.py index dcefd5531d68..05ddf63e089d 100644 --- a/src/transformers/modeling_tf_t5.py +++ b/src/transformers/modeling_tf_t5.py @@ -1103,9 +1103,7 @@ def call( training=training, ) - past = ( - (encoder_outputs, decoder_outputs[1]) if cast_bool_to_primitive(use_cache, self.config.use_cache) else None - ) + past = (encoder_outputs, decoder_outputs[1]) if (use_cache) else None if not return_dict: if past is not None: decoder_outputs = decoder_outputs[:1] + (past,) + decoder_outputs[2:] diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 1f966922d06f..abd37e72a139 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -76,7 +76,7 @@ class TFModelTesterMixin: test_resize_embeddings = True is_encoder_decoder = False - def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): + def _prepare_for_class(self, inputs_dict, model_class, return_labels=False) -> dict: inputs_dict = copy.deepcopy(inputs_dict) if model_class in TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING.values(): diff --git a/tests/test_modeling_tf_lxmert.py b/tests/test_modeling_tf_lxmert.py index a9072e3dd176..0fee98910993 100644 --- a/tests/test_modeling_tf_lxmert.py +++ b/tests/test_modeling_tf_lxmert.py @@ -733,7 +733,11 @@ def test_saved_model_with_attentions_output(self): model = tf.keras.models.load_model(tmpdirname) outputs = model(class_inputs_dict) - language_attentions, vision_attentions, cross_encoder_attentions = (outputs[-3], outputs[-2], outputs[-1]) + language_attentions, vision_attentions, cross_encoder_attentions = ( + outputs[-3], + outputs[-2], + outputs[-1], + ) self.assertEqual(len(language_attentions), self.model_tester.num_hidden_layers["language"]) self.assertEqual(len(vision_attentions), self.model_tester.num_hidden_layers["vision"]) diff --git a/tests/test_modeling_tf_t5.py b/tests/test_modeling_tf_t5.py index 5a1e7768f283..60e6a8218e52 100644 --- a/tests/test_modeling_tf_t5.py +++ b/tests/test_modeling_tf_t5.py @@ -12,8 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - +import tempfile import unittest from transformers import T5Config, is_tf_available @@ -282,6 +281,14 @@ def test_model_from_pretrained(self): model = TFT5Model.from_pretrained("t5-small") self.assertIsNotNone(model) + @slow + def test_saved_model_with_attentions_output(self): + pass + + @slow + def test_saved_model_with_hidden_states_output(self): + pass + @require_tf @require_sentencepiece diff --git a/tests/test_modeling_tf_xlm_roberta.py b/tests/test_modeling_tf_xlm_roberta.py index b67d42db4e5f..10485abfe5cd 100644 --- a/tests/test_modeling_tf_xlm_roberta.py +++ b/tests/test_modeling_tf_xlm_roberta.py @@ -39,7 +39,7 @@ def test_output_embeds_base_model(self): "attention_mask": tf.convert_to_tensor([[1, 1, 1, 1, 1, 1]], dtype=tf.int32), } - output = model(features)["last_hidden_state"] + output = model(features, return_dict=True)["last_hidden_state"] expected_shape = tf.TensorShape((1, 6, 768)) self.assertEqual(output.shape, expected_shape) # compare the actual values for a slice. From 106f14ee1abf15f16c67809e562e2aa03025e408 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 27 Oct 2020 17:43:22 -0400 Subject: [PATCH 10/19] RAG goes OOM in multi gpu setup --- src/transformers/modeling_tf_t5.py | 4 +++- tests/test_modeling_rag.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_tf_t5.py b/src/transformers/modeling_tf_t5.py index 05ddf63e089d..dcefd5531d68 100644 --- a/src/transformers/modeling_tf_t5.py +++ b/src/transformers/modeling_tf_t5.py @@ -1103,7 +1103,9 @@ def call( training=training, ) - past = (encoder_outputs, decoder_outputs[1]) if (use_cache) else None + past = ( + (encoder_outputs, decoder_outputs[1]) if cast_bool_to_primitive(use_cache, self.config.use_cache) else None + ) if not return_dict: if past is not None: decoder_outputs = decoder_outputs[:1] + (past,) + decoder_outputs[2:] diff --git a/tests/test_modeling_rag.py b/tests/test_modeling_rag.py index 6717661739fb..a43076b20a1a 100644 --- a/tests/test_modeling_rag.py +++ b/tests/test_modeling_rag.py @@ -25,7 +25,8 @@ from transformers import BartTokenizer, T5Tokenizer from transformers.file_utils import cached_property, is_datasets_available, is_faiss_available, is_torch_available -from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device +from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device, \ + require_torch_non_multigpu from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer from transformers.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES @@ -574,6 +575,7 @@ def config_and_inputs(self): @require_retrieval @require_sentencepiece @require_tokenizers +@require_torch_non_multigpu class RagModelIntegrationTests(unittest.TestCase): @cached_property def sequence_model(self): From 1cded32c5086b5e3695a4223791bcd3a8d926bc4 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 27 Oct 2020 18:11:51 -0400 Subject: [PATCH 11/19] YAML test files --- .github/workflows/self-push.yml | 27 +++++++------ .github/workflows/self-scheduled.yml | 60 +++++++++++++++------------- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index 411e6d13d28d..ae3649813d14 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -1,6 +1,6 @@ name: Self-hosted runner (push) -on: +on: push: branches: - master @@ -23,17 +23,17 @@ jobs: which python python --version pip --version - + - name: Current dir run: pwd - run: nvidia-smi - - name: Loading cache. - uses: actions/cache@v2 - id: cache - with: - path: .env - key: v0-tests_torch_gpu-${{ hashFiles('setup.py') }} + - name: Loading cache. + uses: actions/cache@v2 + id: cache + with: + path: .env + key: v0-tests_torch_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | @@ -42,13 +42,12 @@ jobs: which python python --version pip --version - + - name: Install dependencies run: | source .env/bin/activate pip install --upgrade pip - pip install torch!=1.6.0 - pip install .[sklearn,testing,onnxruntime] + pip install .[torch,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets - name: Are GPUs recognized by our DL frameworks @@ -92,6 +91,7 @@ jobs: which python python --version pip --version + - name: Install dependencies run: | source .env/bin/activate @@ -122,6 +122,7 @@ jobs: which python python --version pip --version + - name: Current dir run: pwd - run: nvidia-smi @@ -144,8 +145,7 @@ jobs: run: | source .env/bin/activate pip install --upgrade pip - pip install torch!=1.6.0 - pip install .[sklearn,testing,onnxruntime] + pip install .[tf,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets - name: Are GPUs recognized by our DL frameworks @@ -170,6 +170,7 @@ jobs: which python python --version pip --version + - name: Current dir run: pwd - run: nvidia-smi diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 7bbe80905cd2..4d6423750318 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -26,11 +26,11 @@ jobs: which python python --version pip --version - + - name: Current dir run: pwd - run: nvidia-smi - + - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) if: steps.cache.outputs.cache-hit != 'true' run: | @@ -39,13 +39,12 @@ jobs: which python python --version pip --version - + - name: Install dependencies run: | source .env/bin/activate pip install --upgrade pip - pip install torch!=1.6.0 - pip install .[sklearn,testing,onnxruntime] + pip install .[torch,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets - name: Are GPUs recognized by our DL frameworks @@ -54,24 +53,23 @@ jobs: python -c "import torch; print('Cuda available:', torch.cuda.is_available())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" - - name: Run all tests on GPU - env: - OMP_NUM_THREADS: 1 - RUN_SLOW: yes - run: | - source .env/bin/activate - python -m pytest -n 1 --dist=loadfile -s --make_reports=tests tests - - - name: Run examples tests on GPU - env: - OMP_NUM_THREADS: 1 - RUN_SLOW: yes - CUDA_VISIBLE_DEVICES: 0 - run: | - source .env/bin/activate - pip install -r examples/requirements.txt - python -m pytest -n 1 --dist=loadfile -s examples --durations=0 - python -m pytest -n 1 --dist=loadfile -s --make_reports=examples examples + - name: Run all tests on GPU + env: + OMP_NUM_THREADS: 1 + RUN_SLOW: yes + run: | + source .env/bin/activate + python -m pytest -n 1 --dist=loadfile -s --make_reports=tests_torch tests + + - name: Run examples tests on GPU + env: + OMP_NUM_THREADS: 1 + RUN_SLOW: yes + CUDA_VISIBLE_DEVICES: 0 + run: | + source .env/bin/activate + pip install -r examples/requirements.txt + python -m pytest -n 1 --dist=loadfile -s --make_reports=examples_torch examples - name: test suite reports artifacts uses: actions/upload-artifact@v2 @@ -97,9 +95,11 @@ jobs: which python python --version pip --version + - name: Current dir run: pwd - run: nvidia-smi + - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) if: steps.cache.outputs.cache-hit != 'true' run: | @@ -108,6 +108,7 @@ jobs: which python python --version pip --version + - name: Install dependencies run: | source .env/bin/activate @@ -127,7 +128,7 @@ jobs: RUN_SLOW: yes run: | source .env/bin/activate - python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0 + python -m pytest -n 1 --dist=loadfile -s --make_reports=tests_tf tests - name: Run examples tests on GPU env: @@ -137,7 +138,7 @@ jobs: run: | source .env/bin/activate pip install -r examples/requirements.txt - python -m pytest -n 1 --dist=loadfile -s examples --durations=0 + python -m pytest -n 1 --dist=loadfile -s --make_reports=examples_tf examples run_all_tests_torch_multiple_gpu: runs-on: [self-hosted, multi-gpu-tests] @@ -156,9 +157,11 @@ jobs: which python python --version pip --version + - name: Current dir run: pwd - run: nvidia-smi + - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) if: steps.cache.outputs.cache-hit != 'true' run: | @@ -167,12 +170,12 @@ jobs: which python python --version pip --version + - name: Install dependencies run: | source .env/bin/activate pip install --upgrade pip - pip install torch!=1.6.0 - pip install .[sklearn,testing,onnxruntime] + pip install .[tf,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets - name: Are GPUs recognized by our DL frameworks @@ -215,9 +218,11 @@ jobs: which python python --version pip --version + - name: Current dir run: pwd - run: nvidia-smi + - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) if: steps.cache.outputs.cache-hit != 'true' run: | @@ -226,6 +231,7 @@ jobs: which python python --version pip --version + - name: Install dependencies run: | source .env/bin/activate From fe2bfb72611c5b1432aa13714b1f8355d1bb55a5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 28 Oct 2020 10:34:26 -0400 Subject: [PATCH 12/19] Last fixes --- .github/workflows/self-push.yml | 2 +- .github/workflows/self-scheduled.yml | 21 +-------------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index ae3649813d14..f66001ade8a4 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -145,7 +145,7 @@ jobs: run: | source .env/bin/activate pip install --upgrade pip - pip install .[tf,sklearn,testing,onnxruntime] + pip install .[torch,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets - name: Are GPUs recognized by our DL frameworks diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 4d6423750318..d68612a73de9 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -130,16 +130,6 @@ jobs: source .env/bin/activate python -m pytest -n 1 --dist=loadfile -s --make_reports=tests_tf tests - - name: Run examples tests on GPU - env: - OMP_NUM_THREADS: 1 - RUN_SLOW: yes - CUDA_VISIBLE_DEVICES: 0 - run: | - source .env/bin/activate - pip install -r examples/requirements.txt - python -m pytest -n 1 --dist=loadfile -s --make_reports=examples_tf examples - run_all_tests_torch_multiple_gpu: runs-on: [self-hosted, multi-gpu-tests] steps: @@ -175,7 +165,7 @@ jobs: run: | source .env/bin/activate pip install --upgrade pip - pip install .[tf,sklearn,testing,onnxruntime] + pip install .[torch,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets - name: Are GPUs recognized by our DL frameworks @@ -253,15 +243,6 @@ jobs: source .env/bin/activate python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0 - - name: Run examples tests on GPU - env: - OMP_NUM_THREADS: 1 - RUN_SLOW: yes - run: | - source .env/bin/activate - pip install -r examples/requirements.txt - python -m pytest -n 1 --dist=loadfile -s examples --durations=0 - - name: Run all pipeline tests on GPU env: TF_FORCE_GPU_ALLOW_GROWTH: "true" From 06ea8f863427848878589e7781945b4bd0543bd7 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 29 Oct 2020 09:29:43 -0400 Subject: [PATCH 13/19] Skip doctests --- tests/test_doc_samples.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_doc_samples.py b/tests/test_doc_samples.py index 5ce718415426..716537c95584 100644 --- a/tests/test_doc_samples.py +++ b/tests/test_doc_samples.py @@ -27,6 +27,7 @@ logger = logging.getLogger() +@unittest.skip("Temporarily disable the doc tests.") @require_torch @require_tf @slow From 0420537551d59da35c7dd7909ff5804ba5a8dc5c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 29 Oct 2020 09:31:02 -0400 Subject: [PATCH 14/19] Fill mask tests --- tests/test_pipelines_fill_mask.py | 136 +++++++++++++++++++++++------- 1 file changed, 106 insertions(+), 30 deletions(-) diff --git a/tests/test_pipelines_fill_mask.py b/tests/test_pipelines_fill_mask.py index 0bf8c629341b..72cc38e1044c 100644 --- a/tests/test_pipelines_fill_mask.py +++ b/tests/test_pipelines_fill_mask.py @@ -96,21 +96,59 @@ def test_torch_fill_mask_results(self): framework="pt", topk=2, ) - self._test_mono_column_pipeline( - nlp, - valid_inputs, - mandatory_keys, - expected_multi_result=EXPECTED_FILL_MASK_RESULT, - expected_check_keys=["sequence"], - ) - self._test_mono_column_pipeline( - nlp, - valid_inputs[:1], - mandatory_keys, - expected_multi_result=EXPECTED_FILL_MASK_TARGET_RESULT, - expected_check_keys=["sequence"], - targets=valid_targets, - ) + + mono_result = nlp(valid_inputs[0], targets=valid_targets) + self.assertIsInstance(mono_result, list) + self.assertIsInstance(mono_result[0], dict) + + for mandatory_key in mandatory_keys: + self.assertIn(mandatory_key, mono_result[0]) + + multi_result = [nlp(valid_input) for valid_input in valid_inputs] + self.assertIsInstance(multi_result, list) + self.assertIsInstance(multi_result[0], (dict, list)) + + for result, expected in zip(multi_result, EXPECTED_FILL_MASK_RESULT): + self.assertEqual( + set([o["sequence"] for o in result]), + set([o["sequence"] for o in result]) + ) + + if isinstance(multi_result[0], list): + multi_result = multi_result[0] + + for result in multi_result: + for key in mandatory_keys: + self.assertIn(key, result) + + self.assertRaises(Exception, nlp, [None]) + + valid_inputs = valid_inputs[:1] + mono_result = nlp(valid_inputs[0], targets=valid_targets) + self.assertIsInstance(mono_result, list) + self.assertIsInstance(mono_result[0], dict) + + for mandatory_key in mandatory_keys: + self.assertIn(mandatory_key, mono_result[0]) + + multi_result = [nlp(valid_input) for valid_input in valid_inputs] + self.assertIsInstance(multi_result, list) + self.assertIsInstance(multi_result[0], (dict, list)) + + for result, expected in zip(multi_result, EXPECTED_FILL_MASK_TARGET_RESULT): + self.assertEqual( + set([o["sequence"] for o in result]), + set([o["sequence"] for o in result]) + ) + + if isinstance(multi_result[0], list): + multi_result = multi_result[0] + + for result in multi_result: + for key in mandatory_keys: + self.assertIn(key, result) + + self.assertRaises(Exception, nlp, [None]) @require_tf @slow @@ -123,18 +161,56 @@ def test_tf_fill_mask_results(self): valid_targets = [" Patrick", " Clara"] for model_name in self.large_models: nlp = pipeline(task="fill-mask", model=model_name, tokenizer=model_name, framework="tf", topk=2) - self._test_mono_column_pipeline( - nlp, - valid_inputs, - mandatory_keys, - expected_multi_result=EXPECTED_FILL_MASK_RESULT, - expected_check_keys=["sequence"], - ) - self._test_mono_column_pipeline( - nlp, - valid_inputs[:1], - mandatory_keys, - expected_multi_result=EXPECTED_FILL_MASK_TARGET_RESULT, - expected_check_keys=["sequence"], - targets=valid_targets, - ) + + mono_result = nlp(valid_inputs[0], targets=valid_targets) + self.assertIsInstance(mono_result, list) + self.assertIsInstance(mono_result[0], dict) + + for mandatory_key in mandatory_keys: + self.assertIn(mandatory_key, mono_result[0]) + + multi_result = [nlp(valid_input) for valid_input in valid_inputs] + self.assertIsInstance(multi_result, list) + self.assertIsInstance(multi_result[0], (dict, list)) + + for result, expected in zip(multi_result, EXPECTED_FILL_MASK_RESULT): + self.assertEqual( + set([o["sequence"] for o in result]), + set([o["sequence"] for o in result]) + ) + + if isinstance(multi_result[0], list): + multi_result = multi_result[0] + + for result in multi_result: + for key in mandatory_keys: + self.assertIn(key, result) + + self.assertRaises(Exception, nlp, [None]) + + valid_inputs = valid_inputs[:1] + mono_result = nlp(valid_inputs[0], targets=valid_targets) + self.assertIsInstance(mono_result, list) + self.assertIsInstance(mono_result[0], dict) + + for mandatory_key in mandatory_keys: + self.assertIn(mandatory_key, mono_result[0]) + + multi_result = [nlp(valid_input) for valid_input in valid_inputs] + self.assertIsInstance(multi_result, list) + self.assertIsInstance(multi_result[0], (dict, list)) + + for result, expected in zip(multi_result, EXPECTED_FILL_MASK_TARGET_RESULT): + self.assertEqual( + set([o["sequence"] for o in result]), + set([o["sequence"] for o in result]) + ) + + if isinstance(multi_result[0], list): + multi_result = multi_result[0] + + for result in multi_result: + for key in mandatory_keys: + self.assertIn(key, result) + + self.assertRaises(Exception, nlp, [None]) From 127aa6cdae69f62af3a9879d2a2809e94016ce98 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 29 Oct 2020 09:41:31 -0400 Subject: [PATCH 15/19] Yaml files --- .github/workflows/self-push.yml | 8 ++++---- .github/workflows/self-scheduled.yml | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index f66001ade8a4..d10ee1dc2da2 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -15,7 +15,7 @@ on: jobs: run_tests_torch_gpu: - runs-on: [self-hosted, gpu-tests] + runs-on: [self-hosted, single-gpu] steps: - uses: actions/checkout@v2 - name: Python version @@ -65,7 +65,7 @@ jobs: python -m pytest -n 2 --dist=loadfile -s ./tests/ run_tests_tf_gpu: - runs-on: [self-hosted, gpu-tests] + runs-on: [self-hosted, single-gpu] steps: - uses: actions/checkout@v2 - name: Python version @@ -114,7 +114,7 @@ jobs: python -m pytest -n 2 --dist=loadfile -s ./tests/ run_tests_torch_multiple_gpu: - runs-on: [self-hosted, multi-gpu-tests] + runs-on: [self-hosted, multi-gpu] steps: - uses: actions/checkout@v2 - name: Python version @@ -162,7 +162,7 @@ jobs: python -m pytest -n 2 --dist=loadfile -s ./tests/ run_tests_tf_multiple_gpu: - runs-on: [self-hosted, multi-gpu-tests] + runs-on: [self-hosted, multi-gpu] steps: - uses: actions/checkout@v2 - name: Python version diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index d68612a73de9..6a18d9e463bf 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -19,7 +19,7 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }} + key: v0-slow_tests_torch_gpu-${{ hashFiles('setup.py') }} - name: Python version run: | @@ -46,6 +46,7 @@ jobs: pip install --upgrade pip pip install .[torch,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets + pip list - name: Are GPUs recognized by our DL frameworks run: | @@ -88,7 +89,7 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }} + key: v0-slow_tests_tf_gpu-${{ hashFiles('setup.py') }} - name: Python version run: | @@ -115,6 +116,7 @@ jobs: pip install --upgrade pip pip install .[tf,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets + pip list - name: Are GPUs recognized by our DL frameworks run: | @@ -140,7 +142,7 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }} + key: v0-slow_tests_torch_multi_gpu-${{ hashFiles('setup.py') }} - name: Python version run: | @@ -167,6 +169,7 @@ jobs: pip install --upgrade pip pip install .[torch,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets + pip list - name: Are GPUs recognized by our DL frameworks run: | @@ -201,7 +204,7 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }} + key: v0-slow_tests_tf_multi_gpu-${{ hashFiles('setup.py') }} - name: Python version run: | @@ -228,6 +231,7 @@ jobs: pip install --upgrade pip pip install .[tf,sklearn,testing,onnxruntime] pip install git+https://github.com/huggingface/datasets + pip list - name: Are GPUs recognized by our DL frameworks run: | From d07013439051cb3ea32802503d6b6de52db438ad Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 29 Oct 2020 09:44:29 -0400 Subject: [PATCH 16/19] Last test fix --- tests/test_modeling_common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 552651faa3b5..0029c0f06f9d 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -1087,6 +1087,10 @@ def test_model_from_pretrained(self): self.assertEqual(len(value), 0) config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) + + # Not sure this is the intended behavior. TODO fix Lysandre & Thom + config.name_or_path = model_name + model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) self.assertEqual(model.config.output_hidden_states, True) self.assertEqual(model.config, config) From 3eab0986fd146869322ea1daf2396550f746f404 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 29 Oct 2020 09:44:33 -0400 Subject: [PATCH 17/19] Style --- tests/test_modeling_rag.py | 10 ++++++++-- tests/test_modeling_tf_lxmert.py | 2 -- tests/test_modeling_tf_t5.py | 1 - tests/test_pipelines_fill_mask.py | 20 ++++---------------- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/tests/test_modeling_rag.py b/tests/test_modeling_rag.py index a43076b20a1a..fe8fefde4d84 100644 --- a/tests/test_modeling_rag.py +++ b/tests/test_modeling_rag.py @@ -25,8 +25,14 @@ from transformers import BartTokenizer, T5Tokenizer from transformers.file_utils import cached_property, is_datasets_available, is_faiss_available, is_torch_available -from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device, \ - require_torch_non_multigpu +from transformers.testing_utils import ( + require_sentencepiece, + require_tokenizers, + require_torch, + require_torch_non_multigpu, + slow, + torch_device, +) from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer from transformers.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES diff --git a/tests/test_modeling_tf_lxmert.py b/tests/test_modeling_tf_lxmert.py index 0fee98910993..0bb17adcb508 100644 --- a/tests/test_modeling_tf_lxmert.py +++ b/tests/test_modeling_tf_lxmert.py @@ -687,7 +687,6 @@ def test_saved_model_with_hidden_states_output(self): for model_class in self.all_model_classes: class_inputs_dict = self._prepare_for_class(inputs_dict, model_class) model = model_class(config) - num_out = len(model(class_inputs_dict)) model._saved_model_inputs_spec = None model._set_save_spec(class_inputs_dict) @@ -724,7 +723,6 @@ def test_saved_model_with_attentions_output(self): for model_class in self.all_model_classes: class_inputs_dict = self._prepare_for_class(inputs_dict, model_class) model = model_class(config) - num_out = len(model(inputs_dict)) model._saved_model_inputs_spec = None model._set_save_spec(class_inputs_dict) diff --git a/tests/test_modeling_tf_t5.py b/tests/test_modeling_tf_t5.py index 60e6a8218e52..f42e03ca1681 100644 --- a/tests/test_modeling_tf_t5.py +++ b/tests/test_modeling_tf_t5.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import tempfile import unittest from transformers import T5Config, is_tf_available diff --git a/tests/test_pipelines_fill_mask.py b/tests/test_pipelines_fill_mask.py index 72cc38e1044c..e49454d74168 100644 --- a/tests/test_pipelines_fill_mask.py +++ b/tests/test_pipelines_fill_mask.py @@ -109,10 +109,7 @@ def test_torch_fill_mask_results(self): self.assertIsInstance(multi_result[0], (dict, list)) for result, expected in zip(multi_result, EXPECTED_FILL_MASK_RESULT): - self.assertEqual( - set([o["sequence"] for o in result]), - set([o["sequence"] for o in result]) - ) + self.assertEqual(set([o["sequence"] for o in result]), set([o["sequence"] for o in result])) if isinstance(multi_result[0], list): multi_result = multi_result[0] @@ -136,10 +133,7 @@ def test_torch_fill_mask_results(self): self.assertIsInstance(multi_result[0], (dict, list)) for result, expected in zip(multi_result, EXPECTED_FILL_MASK_TARGET_RESULT): - self.assertEqual( - set([o["sequence"] for o in result]), - set([o["sequence"] for o in result]) - ) + self.assertEqual(set([o["sequence"] for o in result]), set([o["sequence"] for o in result])) if isinstance(multi_result[0], list): multi_result = multi_result[0] @@ -174,10 +168,7 @@ def test_tf_fill_mask_results(self): self.assertIsInstance(multi_result[0], (dict, list)) for result, expected in zip(multi_result, EXPECTED_FILL_MASK_RESULT): - self.assertEqual( - set([o["sequence"] for o in result]), - set([o["sequence"] for o in result]) - ) + self.assertEqual(set([o["sequence"] for o in result]), set([o["sequence"] for o in result])) if isinstance(multi_result[0], list): multi_result = multi_result[0] @@ -201,10 +192,7 @@ def test_tf_fill_mask_results(self): self.assertIsInstance(multi_result[0], (dict, list)) for result, expected in zip(multi_result, EXPECTED_FILL_MASK_TARGET_RESULT): - self.assertEqual( - set([o["sequence"] for o in result]), - set([o["sequence"] for o in result]) - ) + self.assertEqual(set([o["sequence"] for o in result]), set([o["sequence"] for o in result])) if isinstance(multi_result[0], list): multi_result = multi_result[0] From 41641c7e3dc3e281780108fc4291a65d7c98eee7 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 29 Oct 2020 10:29:49 -0400 Subject: [PATCH 18/19] Update cache --- .github/workflows/self-scheduled.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 6a18d9e463bf..029a58ea37cb 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -19,7 +19,7 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_torch_gpu-${{ hashFiles('setup.py') }} + key: v0.1-slow_tests_torch_gpu-${{ hashFiles('setup.py') }} - name: Python version run: | @@ -89,7 +89,7 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_tf_gpu-${{ hashFiles('setup.py') }} + key: v0.1-slow_tests_tf_gpu-${{ hashFiles('setup.py') }} - name: Python version run: | @@ -142,7 +142,7 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_torch_multi_gpu-${{ hashFiles('setup.py') }} + key: v0.1-slow_tests_torch_multi_gpu-${{ hashFiles('setup.py') }} - name: Python version run: | @@ -204,7 +204,7 @@ jobs: id: cache with: path: .env - key: v0-slow_tests_tf_multi_gpu-${{ hashFiles('setup.py') }} + key: v0.1-slow_tests_tf_multi_gpu-${{ hashFiles('setup.py') }} - name: Python version run: | From f5660ac3f4de79ea9d7ec9f65ee8f9241260352d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 29 Oct 2020 14:03:19 -0400 Subject: [PATCH 19/19] Change ONNX tests to slow + use tiny model --- .github/workflows/self-push.yml | 8 ++++---- tests/test_onnx.py | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index d10ee1dc2da2..b7294fbbc34d 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -33,7 +33,7 @@ jobs: id: cache with: path: .env - key: v0-tests_torch_gpu-${{ hashFiles('setup.py') }} + key: v0.1-tests_torch_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | @@ -82,7 +82,7 @@ jobs: id: cache with: path: .env - key: v0-tests_tf_gpu-${{ hashFiles('setup.py') }} + key: v0.1-tests_tf_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | @@ -132,7 +132,7 @@ jobs: id: cache with: path: .env - key: v0-tests_torch_multiple_gpu-${{ hashFiles('setup.py') }} + key: v0.1-tests_torch_multiple_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | @@ -180,7 +180,7 @@ jobs: id: cache with: path: .env - key: v0-tests_tf_multiple_gpu-${{ hashFiles('setup.py') }} + key: v0.1-tests_tf_multiple_gpu-${{ hashFiles('setup.py') }} - name: Create new python env (on self-hosted runners we have to handle isolation ourselves) run: | diff --git a/tests/test_onnx.py b/tests/test_onnx.py index 17578ed670d2..c13ce6d90b66 100644 --- a/tests/test_onnx.py +++ b/tests/test_onnx.py @@ -95,26 +95,28 @@ def _test_export(self, model, framework, opset, tokenizer=None): @require_torch @require_tokenizers + @slow def test_infer_dynamic_axis_pytorch(self): """ Validate the dynamic axis generated for each parameters are correct """ from transformers import BertModel - model = BertModel(BertConfig.from_pretrained("bert-base-cased")) - tokenizer = BertTokenizerFast.from_pretrained("bert-base-cased") + model = BertModel(BertConfig.from_pretrained("lysandre/tiny-bert-random")) + tokenizer = BertTokenizerFast.from_pretrained("lysandre/tiny-bert-random") self._test_infer_dynamic_axis(model, tokenizer, "pt") @require_tf @require_tokenizers + @slow def test_infer_dynamic_axis_tf(self): """ Validate the dynamic axis generated for each parameters are correct """ from transformers import TFBertModel - model = TFBertModel(BertConfig.from_pretrained("bert-base-cased")) - tokenizer = BertTokenizerFast.from_pretrained("bert-base-cased") + model = TFBertModel(BertConfig.from_pretrained("lysandre/tiny-bert-random")) + tokenizer = BertTokenizerFast.from_pretrained("lysandre/tiny-bert-random") self._test_infer_dynamic_axis(model, tokenizer, "tf") def _test_infer_dynamic_axis(self, model, tokenizer, framework):