Pacific-AI-Corp · ArshaanNazir · Jun 16, 2023 · Jun 7, 2023 · Jun 7, 2023 · Jun 8, 2023
diff --git a/conda/meta.yaml b/conda/meta.yaml
@@ -1,13 +1,13 @@
 {% set name = "nlptest" %}
-{% set version = "1.4.0" %}
+{% set version = "1.5.0" %}
 
 package:
   name: {{ name|lower }}
   version: {{ version }}
 
 source:
   url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/nlptest-{{ version }}.tar.gz
-  sha256: e2e28f9c3d0f35e3e3c07bebadb25b52fdd4f22e8ff4939d343cb68d4984d8ff
+  sha256: 556ac72ef3e8a1c1851bbfe8d22e4312f01e7607d6d3a44775279766744ff9e2
 
 build:
   number: 0

diff --git a/demo/tutorials/end-to-end-notebooks/JohnSnowLabs_RealWorld_Custom_Pipeline_Notebook.ipynb b/demo/tutorials/end-to-end-notebooks/JohnSnowLabs_RealWorld_Custom_Pipeline_Notebook.ipynb
@@ -82,7 +82,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 3,
+      "execution_count": null,
       "metadata": {
         "id": "lTzSJpMlhgq5"
       },
@@ -105,11 +105,11 @@
         "<br/>\n",
         "\n",
         "\n",
-        "| Parameter  | Description |  \n",
-        "| - | - | \n",
+        "| Parameter  | Description |\n",
+        "| - | - |\n",
         "|**task**     |Task for which the model is to be evaluated (text-classification or ner)|\n",
         "|**model**     |PipelineModel or path to a saved model or pretrained pipeline/model from hub.\n",
-        "|**data**       |Path to the data that is to be used for evaluation. Can be .csv or .conll file in the CoNLL format \n",
+        "|**data**       |Path to the data that is to be used for evaluation. Can be .csv or .conll file in the CoNLL format\n",
         "|**config**     |Configuration for the tests to be performed, specified in form of a YAML file.\n",
         "|**hub**       |model hub to load from the path. Required if model param is passed as path.|\n",
         "\n",
@@ -142,7 +142,7 @@
         "\n",
         "2. Test NER model robustness on CoNLL test set\n",
         "\n",
-        "3. Augment CoNLL training set based on test results \n",
+        "3. Augment CoNLL training set based on test results\n",
         "\n",
         "4. Train new NER model on augmented CoNLL training set\n",
         "\n",
@@ -186,7 +186,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 5,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -214,7 +214,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 6,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -268,7 +268,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 7,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -315,7 +315,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 8,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -351,7 +351,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 9,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -399,7 +399,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 10,
+      "execution_count": null,
       "metadata": {
         "id": "RVk9NWn7u-Lm"
       },
@@ -410,7 +410,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 11,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -436,9 +436,9 @@
         "harness.configure({\n",
         "    'tests': {\n",
         "        'defaults': {'min_pass_rate': 0.65},\n",
-        "        \n",
+        "\n",
         "        'robustness': {\n",
-        "            'lowercase': {'min_pass_rate': 0.60}, \n",
+        "            'lowercase': {'min_pass_rate': 0.60},\n",
         "            'uppercase':{'min_pass_rate': 0.60}\n",
         "        }\n",
         "    }\n",
@@ -468,7 +468,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 12,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -508,7 +508,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 13,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
@@ -791,7 +791,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 14,
+      "execution_count": null,
       "metadata": {
         "id": "XYIMxVN8GFGH"
       },
@@ -811,7 +811,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 15,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -851,7 +851,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 16,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
@@ -1162,7 +1162,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 17,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
@@ -1345,7 +1345,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 18,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -1387,7 +1387,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 19,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -1440,7 +1440,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 20,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -1478,7 +1478,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 21,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -1514,7 +1514,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 22,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -1571,7 +1571,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 23,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -1589,7 +1589,7 @@
         }
       ],
       "source": [
-        "harness = Harness.load(\"saved_test_configurations\",model=ner_model_2)"
+        "harness = Harness.load(\"saved_test_configurations\",model=ner_model_2, task=\"ner\")"
       ]
     },
     {
@@ -1603,7 +1603,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 24,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
@@ -1782,14 +1782,40 @@
         "We can see that after augmentation, both tests are performing better than earlier."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Comparison Table ( Normal V/S Augmented Model )"
+      ],
+      "metadata": {
+        "id": "APU3YhhLc1hH"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "| Model            | Category   | Test_Type | Fail_Count | Pass_Count | Pass_Rate | Minimum_Pass_Rate | Pass  |\n",
+        "|------------------|------------|-----------|------------|------------|-----------|-------------------|-------|\n",
+        "| Initial_Model    | Robustness | Lowercase |     198    |     28     |    12%    |        60%        | False |\n",
+        "|                  |            |           |            |            |           |                   |       |\n",
+        "| Initial_Model    | Robustness | Uppercase |     83     |    143     |    63%    |        60%        | True  |\n",
+        "|                  |            |           |            |            |           |                   |       |\n",
+        "| Augmented_Model  | Robustness | Lowercase |     31     |    195     |    86%    |        60%        | True  |\n",
+        "|                  |            |           |            |            |           |                   |       |\n",
+        "| Augmented_Model  | Robustness | Uppercase |     39     |    187     |    83%    |        60%        | True  |\n"
+      ],
+      "metadata": {
+        "id": "ooTBisOrdR7g"
+      }
+    },
     {
       "cell_type": "code",
-      "execution_count": 24,
+      "source": [],
       "metadata": {
-        "id": "HQgkz0tOFuDV"
+        "id": "u007fehpdSRl"
       },
-      "outputs": [],
-      "source": []
+      "execution_count": null,
+      "outputs": []
     }
   ],
   "metadata": {
@@ -1807,4 +1833,4 @@
   },
   "nbformat": 4,
   "nbformat_minor": 0
-}
+}
diff --git a/demo/tutorials/misc/Augementation_Control_Notebook.ipynb b/demo/tutorials/misc/Augementation_Control_Notebook.ipynb
diff --git a/demo/tutorials/misc/Comparing_Models_Notebook.ipynb b/demo/tutorials/misc/Comparing_Models_Notebook.ipynb
diff --git a/demo/tutorials/misc/HuggingFace_Dataset_Notebook.ipynb b/demo/tutorials/misc/HuggingFace_Dataset_Notebook.ipynb
diff --git a/demo/tutorials/misc/RuntimeTest_Notebook.ipynb b/demo/tutorials/misc/RuntimeTest_Notebook.ipynb
diff --git a/demo/tutorials/test-specific-notebooks/Custom_Bias_Demo.ipynb b/demo/tutorials/test-specific-notebooks/Custom_Bias_Demo.ipynb
diff --git a/docs/pages/docs/data.md b/docs/pages/docs/data.md
@@ -10,15 +10,16 @@ modify_date: "2019-05-16"
 
 <div class="main-docs" markdown="1"><div class="h3-box" markdown="1">
 
-Supported data input formats are task-dependent. For `ner` and `text-classification`, the user is meant to provide a **`CoNLL`** or **`CSV`** dataset. For `question-answering` the user is meant to choose from a list of benchmark datasets.
+Supported data input formats are task-dependent. For `ner` and `text-classification`, the user is meant to provide a **`CoNLL`** or **`CSV`** dataset. For `question-answering`, `summarization` and `toxicity`  the user is meant to choose from a list of benchmark datasets we support.
 
 {:.table2}
 | Task  | Supported Data Inputs |  
 | - | - | 
 |**ner**     |CoNLL and CSV|
-|**text-classification**     |CSV
+|**text-classification**     |CSV or a Dictionary (containing the name, subset, split, feature_column and target_column for loading the HF dataset.)
 |**question-answering**     |Select list of benchmark datasets
 |**summarization**     |Select list of benchmark datasets
+|**toxicity**     |Select list of benchmark datasets
 
 </div><div class="h3-box" markdown="1">
 
@@ -69,7 +70,7 @@ harness = Harness(task='ner',
 
 ### Text Classification
 
-There is 1 option for datasets to test Text Classification models: **`CSV`** datasets. Here are some details of what these may look like:
+There are 2 options for datasets to test Text Classification models: **`CSV`** datasets or a **`Dictionary`** containing the name, subset, split, feature_column and target_column for loading the HF datasets. Here are some details of what these may look like:
 
 #### CSV Format for Text Classification
 
@@ -90,7 +91,7 @@ For `CSV` files, we support different variations of the column names. They are s
 
 </div><div class="h3-box" markdown="1">
 
-#### Passing a Text Classification Dataset to the Harness
+#### Passing a CSV Text Classification Dataset to the Harness
 
 In the Harness, we specify the data input in the following way:
 
@@ -107,6 +108,42 @@ harness = Harness(task='text-classification',
 
 </div><div class="h3-box" markdown="1">
 
+#### Dictionary Format for Text Classification
+To handle text classification task for Hugging Face Datasets, the Harness class accepts the data parameter as a dictionary with following attributes:
+
+<i class="fa fa-info-circle"></i>
+<em>It's important to note that the default values for the **`split`**, **`feature_column`**, and **`target_column`** attributes are **`test`**, **`text`**, and **`label`**, respectively.</em>
+
+```python
+{
+   "name": "",
+   "subset": "",
+   "feature_column": "",
+   "target_column": "",
+   "split": ""
+}
+```
+
+#### Passing a Hugging Face Dataset for Text Classification to the Harness
+
+In the Harness, we specify the data input in the following way:
+
+```python
+# Import Harness from the nlptest library
+from nlptest import Harness
+
+harness = Harness(task="text-classification", hub="huggingface",
+                   model="distilbert-base-uncased-finetuned-sst-2-english",
+                  data={"name":'glue',
+                  "subset":"sst2",
+                  "feature_column":"sentence",
+                  "target_column":'label',
+                  "split":"train"
+                  })
+```
+
+</div><div class="h3-box" markdown="1">
+
 ### Question Answering
 
 To test Question Answering models, the user is meant to select a benchmark dataset from the following list:

diff --git a/docs/pages/docs/harness.md b/docs/pages/docs/harness.md
@@ -30,9 +30,9 @@ Here is a list of the different parameters that can be passed to the `Harness` c
 | Parameter   | Description |  
 | - | - | 
 |**task**     |Task for which the model is to be evaluated ('text-classification', 'question-answering', 'ner')|
-|**model**    |Pretrained pipeline or model from the corresponding hub, or path to a saved model from the corresponding hub, or PipelineModel object - see [Model Input](https://nlptest.org/docs/pages/docs/model_input) for more details
+|**model**    |Pretrained pipeline or model from the corresponding hub, or path to a saved model from the corresponding hub, or PipelineModel object or a dictionary containing the names of the models you want to compare, each paired with its respective hub - see [Model Input](https://nlptest.org/docs/pages/docs/model_input) for more details
 |**hub**      |Hub (library) to use in back-end for loading model from public models hub or from path|
-|**data**     |Path to the data to be used for evaluation. Should be `.csv` for text classification, or `.conll` or `.txt` file in CoNLL format for NER - see [Data Input](https://nlptest.org/docs/pages/docs/data_input) for more details
+|**data**     |Path to the data to be used for evaluation. Should be `.csv` or a dictionary containing the name, subset, split, feature_column and target_column for loading the HF dataset for text classification, or `.conll` or `.txt` file in CoNLL format for NER - see [Data Input](https://nlptest.org/docs/pages/docs/data_input) for more details
 |**config**   |Path to the YAML file with configuration of tests to be performed
 
 </div></div>