lint

Apply isort and black reformatting

Signed-off-by: soluwalana <[email protected]>
Signed-off-by: Sam Oluwalana <[email protected]>

Author: soluwalana

nemo/collections/llm/gpt/data/core.py

@@ -25,12 +25,7 @@
 from datasets import load_dataset
 
 from nemo.collections.common.tokenizers import TokenizerSpec
-from nemo.collections.llm.gpt.data.utils import (
-    _get_samples_mapping,
-    _JSONLMemMapDataset,
-    _OnlineSampleMapping,
-    _preprocess,
-)
+from nemo.collections.llm.gpt.data.utils import _get_samples_mapping, _JSONLMemMapDataset, _OnlineSampleMapping
 from nemo.core.classes import Dataset
 from nemo.lightning.base import NEMO_DATASETS_CACHE
 

nemo/collections/llm/gpt/data/packed_sequence.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import json
-import os
 
 from dataclasses import dataclass
 from pathlib import Path

nemo/collections/nlp/models/language_modeling/megatron_gpt_sft_model.py

@@ -211,7 +211,7 @@ def _build_dataset(self, data_cfg, is_train=True):
         # Determine if we are using a single dataset or a list of datasets.
         is_list_config = isinstance(data_cfg.file_names, ListConfig)
         if not is_list_config:
-            raise ValueError(f"SFT train/validation datasets must be provided as a list of individual JSONL files.")
+            raise ValueError("SFT train/validation datasets must be provided as a list of individual JSONL files.")
 
         if is_train:
             # Construct the data prefix list for `get_datasets_weights_and_num_samples()`
@@ -221,15 +221,15 @@ def _build_dataset(self, data_cfg, is_train=True):
             ):
                 raise ValueError(
                     (
-                        f"concat_sampling_probabilities must be a ListConfig with the same number of files in file_names."
+                        "concat_sampling_probabilities must be a ListConfig with the same number of files in file_names."
                         f"Found: {data_cfg.concat_sampling_probabilities}"
                     )
                 )
 
             if len(data_cfg.get('concat_sampling_probabilities', None)) != len(data_cfg.file_names):
                 raise ValueError(
                     (
-                        f"concat_sampling_probabilities must be of the same size as file_names.",
+                        "concat_sampling_probabilities must be of the same size as file_names.",
                         f"Provided size {len(data_cfg.concat_sampling_probabilities)}, number of datasets {len(data_cfg.file_names)}",
                     )
                 )

nemo/collections/nlp/modules/common/tokenizer_utils.py

@@ -217,12 +217,12 @@ def get_nmt_tokenizer(
     elif library == 'byte-level':
         from nemo.collections.common.tokenizers.bytelevel_tokenizers import ByteLevelTokenizer
 
-        logging.info(f'Using byte-level tokenization')
+        logging.info('Using byte-level tokenization')
         return ByteLevelTokenizer(special_tokens_dict)
     elif library == 'regex':
         from nemo.collections.common.tokenizers.regex_tokenizer import RegExTokenizer
 
-        logging.info(f'Using regex tokenization')
+        logging.info('Using regex tokenization')
         return RegExTokenizer().load_tokenizer(regex_file=tokenizer_model, vocab_file=vocab_file)
     elif library == 'megatron':