GSK-184 Giskard client: drop target variable from df

giskard-common/proto/ml-worker.proto

@@ -59,6 +59,9 @@ message DataFrame{
 message RunModelForDataFrameRequest{
   SerializedGiskardModel model = 1;
   DataFrame dataframe = 2;
+  string target = 3;
+  map<string, string> feature_types = 4;
+
 }
 message RunModelRequest{
   SerializedGiskardModel model = 1;

giskard-frontend/src/api.ts

@@ -12,7 +12,8 @@ import {
     ExplainResponseDTO,
     FeatureMetadataDTO,
     FeedbackDTO,
-    FeedbackMinimalDTO, GeneralSettings,
+    FeedbackMinimalDTO,
+    GeneralSettings,
     InspectionCreateDTO,
     InspectionDTO,
     JWTToken,
@@ -21,6 +22,7 @@ import {
     ModelDTO,
     PasswordResetRequest,
     PredictionDTO,
+    PredictionInputDTO,
     ProjectDTO,
     ProjectPostDTO,
     RoleDTO,
@@ -266,8 +268,12 @@ export const api = {
         config.headers['content-type'] = 'multipart/form-data';
         return apiV2.post<unknown, DatasetDTO>(`/project/data/upload`, formData, config);
     },
-    async predict(modelId: number, inputData: object) {
-        return apiV2.post<unknown, PredictionDTO>(`/models/${modelId}/predict`, {features: inputData});
+    async predict(modelId: number, datasetId: number, inputData: { [key: string]: string }) {
+        const data: PredictionInputDTO = {
+            datasetId: datasetId,
+            features: inputData
+        }
+        return apiV2.post<unknown, PredictionDTO>(`/models/${modelId}/predict`, data);
     },
 
     async prepareInspection(payload: InspectionCreateDTO) {

giskard-frontend/src/generated-sources/ai/giskard/web/dto/prediction-input-dto.ts

@@ -2,5 +2,6 @@
  * Generated from ai.giskard.web.dto.PredictionInputDTO
  */
 export interface PredictionInputDTO {
+    datasetId: number;
     features: {[key: string]: string};
 }

giskard-frontend/src/views/main/project/FeedbackDetail.vue

@@ -93,7 +93,7 @@ export default class FeedbackDetail extends Vue {
   @Prop({required: true}) id!: number;
 
   data: FeedbackDTO | null = null;
-  userData: object | null = null;
+  userData: {[key: string]: string} | null = null;
   originalData: object | null = null;
 
   async mounted() {

giskard-frontend/src/views/main/project/Inspector.vue

@@ -95,6 +95,7 @@
         <v-col cols="12" md="6">
           <PredictionResults
               :model="model"
+              :dataset-id="dataset.id"
               :targetFeature="dataset.target"
               :classificationLabels="model.classificationLabels"
               :predictionTask="model.modelType"
@@ -166,7 +167,7 @@ export default class Inspector extends Vue {
   @Prop({required: true}) model!: ModelDTO
   @Prop({required: true}) dataset!: DatasetDTO
   @Prop({required: true}) originalData!: object // used for the variation feedback
-  @Prop({required: true}) inputData!: object
+  @Prop({required: true}) inputData!: {[key: string]: string}
   @Prop({default: false}) isMiniMode!: boolean;
   loadingData = false;
   inputMetaData: FeatureMetadataDTO[] = [];

giskard-frontend/src/views/main/project/PredictionResults.vue

@@ -95,10 +95,11 @@ Vue.component("v-chart", ECharts);
 })
 export default class PredictionResults extends Vue {
   @Prop({required: true}) model!: ModelDTO;
+  @Prop({required: true}) datasetId!: number;
   @Prop({required: true}) predictionTask!: ModelType;
   @Prop() targetFeature!: string;
   @Prop() classificationLabels!: string[];
-  @Prop() inputData!: object;
+  @Prop() inputData!: {[key: string]: string};
   @Prop({default: false}) modified!: boolean;
 
   prediction: string | number | undefined = "";
@@ -120,6 +121,7 @@ export default class PredictionResults extends Vue {
         this.loading = true;
         const predictionResult = (await api.predict(
             this.model.id,
+            this.datasetId,
             this.inputData
         ))
         this.prediction = predictionResult.prediction;

giskard-ml-worker/ml_worker/core/model.py

@@ -4,6 +4,8 @@
 import pandas as pd
 from pydantic import BaseModel
 
+from ml_worker.core.giskard_dataset import GiskardDataset
+
 
 class ModelPredictionResults(BaseModel):
     prediction: Any
@@ -26,8 +28,13 @@ def __init__(self,
         self.feature_names = feature_names
         self.classification_labels = classification_labels
 
-    def run_predict(self, input_df: pd.DataFrame):
-        raw_prediction = self.prediction_function(input_df[self.feature_names])
+    def run_predict(self, dataset: GiskardDataset):
+        df = dataset.df.copy()
+        if dataset.target and dataset.target in df.columns:
+            df.drop(dataset.target, axis=1, inplace=True)
+        if self.feature_names:
+            df = df[self.feature_names]
+        raw_prediction = self.prediction_function(df)
         if self.model_type == "regression":
             result = ModelPredictionResults(
                 prediction=raw_prediction,

giskard-ml-worker/ml_worker/server/ml_worker_service.py

@@ -9,6 +9,7 @@
 from generated.ml_worker_pb2 import RunTestRequest, TestResultMessage, RunModelResponse, RunModelRequest, DataFrame, \
     DataRow, RunModelForDataFrameResponse, RunModelForDataFrameRequest, ExplainRequest, ExplainTextRequest
 from generated.ml_worker_pb2_grpc import MLWorkerServicer
+from ml_worker.core.giskard_dataset import GiskardDataset
 from ml_worker.core.model_explanation import explain, text_explanation_prediction_wrapper, parse_text_explainer_response
 from ml_worker.exceptions.IllegalArgumentError import IllegalArgumentError
 from ml_worker.utils.grpc_mapper import deserialize_model, deserialize_dataset
@@ -62,7 +63,9 @@ def explainText(self, request: ExplainTextRequest, context) -> ExplainTextRespon
         if request.feature_types[text_column] != "text":
             raise ValueError(f"Column {text_column} is not of type text")
         text_document = request.columns[text_column]
-        input_df = pd.DataFrame({k: [v] for k, v in request.columns.items()})[model.feature_names]
+        input_df = pd.DataFrame({k: [v] for k, v in request.columns.items()})
+        if model.feature_names:
+            input_df = input_df[model.feature_names]
         text_explainer = TextExplainer(random_state=42, n_samples=n_samples)
         prediction_function = text_explanation_prediction_wrapper(
             model.prediction_function, input_df, text_column
@@ -73,8 +76,10 @@ def explainText(self, request: ExplainTextRequest, context) -> ExplainTextRespon
 
     def runModelForDataFrame(self, request: RunModelForDataFrameRequest, context):
         model = deserialize_model(request.model)
-        df = pd.DataFrame([r.columns for r in request.dataframe.rows])
-        predictions = model.run_predict(df)
+        ds = GiskardDataset(pd.DataFrame([r.columns for r in request.dataframe.rows]),
+                            target=request.target,
+                            feature_types=request.feature_types)
+        predictions = model.run_predict(ds)
         if model.model_type == "classification":
             return RunModelForDataFrameResponse(all_predictions=self.pandas_df_to_proto_df(predictions.all_predictions),
                                                 prediction=predictions.prediction.astype(str))
@@ -86,7 +91,7 @@ def runModel(self, request: RunModelRequest, context) -> RunModelResponse:
         import numpy as np
         model = deserialize_model(request.model)
         dataset = deserialize_dataset(request.dataset)
-        prediction_results = model.run_predict(dataset.df)
+        prediction_results = model.run_predict(dataset)
 
         if model.model_type == "classification":
             results = prediction_results.all_predictions

giskard-ml-worker/ml_worker/testing/drift_tests.py

@@ -306,8 +306,8 @@ def test_drift_prediction_psi(self, reference_slice: GiskardDataset, actual_slic
                 psi result message
 
         """
-        prediction_reference = model.run_predict(reference_slice.df).prediction
-        prediction_actual = model.run_predict(actual_slice.df).prediction
+        prediction_reference = model.run_predict(reference_slice).prediction
+        prediction_actual = model.run_predict(actual_slice).prediction
         total_psi, output_data = self._calculate_drift_psi(prediction_reference, prediction_actual, max_categories)
 
         passed = True if threshold is None else total_psi <= threshold
@@ -362,8 +362,8 @@ def test_drift_prediction_chi_square(self, reference_slice, actual_slice, model,
                 message describing if prediction is drifting or not
 
         """
-        prediction_reference = model.run_predict(reference_slice.df).prediction
-        prediction_actual = model.run_predict(actual_slice.df).prediction
+        prediction_reference = model.run_predict(reference_slice).prediction
+        prediction_actual = model.run_predict(actual_slice).prediction
         chi_square, p_value, output_data = self._calculate_chi_square(prediction_reference, prediction_actual,
                                                                       max_categories)
 
@@ -420,10 +420,10 @@ def test_drift_prediction_ks(self,
         assert model.model_type != "classification" or classification_label in model.classification_labels, \
             f'"{classification_label}" is not part of model labels: {",".join(model.classification_labels)}'
 
-        prediction_reference = model.run_predict(reference_slice.df).all_predictions[classification_label].values if \
-            model.model_type == "classification" else model.run_predict(reference_slice.df).prediction
-        prediction_actual = model.run_predict(actual_slice.df).all_predictions[classification_label].values if \
-            model.model_type == "classification" else model.run_predict(actual_slice.df).prediction
+        prediction_reference = model.run_predict(reference_slice).all_predictions[classification_label].values if \
+            model.model_type == "classification" else model.run_predict(reference_slice).prediction
+        prediction_actual = model.run_predict(actual_slice).all_predictions[classification_label].values if \
+            model.model_type == "classification" else model.run_predict(actual_slice).prediction
 
         result: Ks_2sampResult = self._calculate_ks(prediction_reference, prediction_actual)
         passed = True if threshold is None else result.pvalue >= threshold
@@ -478,10 +478,10 @@ def test_drift_prediction_earth_movers_distance(self,
 
         """
 
-        prediction_reference = model.run_predict(reference_slice.df).all_predictions[classification_label].values if \
-            model.model_type == "classification" else model.run_predict(reference_slice.df).prediction
-        prediction_actual = model.run_predict(actual_slice.df).all_predictions[classification_label].values if \
-            model.model_type == "classification" else model.run_predict(actual_slice.df).prediction
+        prediction_reference = model.run_predict(reference_slice).all_predictions[classification_label].values if \
+            model.model_type == "classification" else model.run_predict(reference_slice).prediction
+        prediction_actual = model.run_predict(actual_slice).all_predictions[classification_label].values if \
+            model.model_type == "classification" else model.run_predict(actual_slice).prediction
 
         metric = self._calculate_earth_movers_distance(prediction_reference, prediction_actual)
 

giskard-ml-worker/ml_worker/testing/heuristic_tests.py

@@ -41,7 +41,7 @@ def test_right_label(self,
 
         """
 
-        prediction_results = model.run_predict(actual_slice.df).prediction
+        prediction_results = model.run_predict(actual_slice).prediction
         assert classification_label in model.classification_labels, \
             f'"{classification_label}" is not part of model labels: {",".join(model.classification_labels)}'
 
@@ -103,7 +103,7 @@ def test_output_in_range(self,
         """
         results_df = pd.DataFrame()
 
-        prediction_results = model.run_predict(actual_slice.df)
+        prediction_results = model.run_predict(actual_slice)
 
         if model.model_type == "regression":
             results_df["output"] = prediction_results.raw_prediction

giskard-ml-worker/ml_worker/testing/metamorphic_tests.py

@@ -10,28 +10,30 @@
 
 class MetamorphicTests(AbstractTestCollection):
     @staticmethod
-    def _predict_numeric_result(df, model: GiskardModel, output_proba=True, classification_label=None):
+    def _predict_numeric_result(ds: GiskardDataset, model: GiskardModel, output_proba=True, classification_label=None):
         if model.model_type == 'regression' or not output_proba:
-            return model.run_predict(df).raw_prediction
+            return model.run_predict(ds).raw_prediction
         elif model.model_type == 'classification' and classification_label is not None:
-            return model.run_predict(df).all_predictions[classification_label].values
+            return model.run_predict(ds).all_predictions[classification_label].values
         elif model.model_type == 'classification':
-            return model.run_predict(df).probabilities
+            return model.run_predict(ds).probabilities
 
     @staticmethod
     def _prediction_ratio(prediction, perturbed_prediction):
         return abs(perturbed_prediction - prediction) / prediction if prediction != 0 else abs(perturbed_prediction)
 
     @staticmethod
-    def _perturb_and_predict(df, model: GiskardModel, perturbation_dict, output_proba=True,
+    def _perturb_and_predict(ds: GiskardDataset, model: GiskardModel, perturbation_dict, output_proba=True,
                              classification_label=None):
         results_df = pd.DataFrame()
-        results_df["prediction"] = MetamorphicTests._predict_numeric_result(df, model, output_proba,
+        results_df["prediction"] = MetamorphicTests._predict_numeric_result(ds, model, output_proba,
                                                                             classification_label)
-        modified_rows = apply_perturbation_inplace(df, perturbation_dict)
+        modified_rows = apply_perturbation_inplace(ds.df, perturbation_dict)
         if len(modified_rows):
+            ds.df = ds.df.iloc[modified_rows]
             results_df = results_df.iloc[modified_rows]
-            results_df["perturbed_prediction"] = MetamorphicTests._predict_numeric_result(df.iloc[modified_rows], model,
+            results_df["perturbed_prediction"] = MetamorphicTests._predict_numeric_result(ds,
+                                                                                          model,
                                                                                           output_proba,
                                                                                           classification_label)
         else:
@@ -70,7 +72,7 @@ def _test_metamorphic(self,
                           output_sensitivity=None,
                           output_proba=True
                           ) -> SingleTestResult:
-        results_df, modified_rows_count = self._perturb_and_predict(actual_slice.df,
+        results_df, modified_rows_count = self._perturb_and_predict(actual_slice,
                                                                     model,
                                                                     perturbation_dict,
                                                                     classification_label=classification_label,
@@ -196,7 +198,7 @@ def test_metamorphic_increasing(self,
 
         """
         assert model.model_type != "classification" or classification_label in model.classification_labels, \
-                    f'"{classification_label}" is not part of model labels: {",".join(model.classification_labels)}'
+            f'"{classification_label}" is not part of model labels: {",".join(model.classification_labels)}'
 
         return self._test_metamorphic(flag='Increasing',
                                       actual_slice=df,
@@ -254,7 +256,7 @@ def test_metamorphic_decreasing(self,
         """
 
         assert model.model_type != "classification" or classification_label in model.classification_labels, \
-                    f'"{classification_label}" is not part of model labels: {",".join(model.classification_labels)}'
+            f'"{classification_label}" is not part of model labels: {",".join(model.classification_labels)}'
 
         return self._test_metamorphic(flag='Decreasing',
                                       actual_slice=df,

giskard-ml-worker/ml_worker/testing/performance_tests.py

@@ -36,10 +36,10 @@ def test_auc(self, actual_slice: GiskardDataset, model: GiskardModel, threshold=
         """
         if len(model.classification_labels) == 2:
             metric = roc_auc_score(actual_slice.df[actual_slice.target],
-                                   model.run_predict(actual_slice.df).raw_prediction)
+                                   model.run_predict(actual_slice).raw_prediction)
         else:
             metric = roc_auc_score(actual_slice.df[actual_slice.target],
-                                   model.run_predict(actual_slice.df).all_predictions, multi_class='ovr')
+                                   model.run_predict(actual_slice).all_predictions, multi_class='ovr')
 
         return self.save_results(
             SingleTestResult(
@@ -51,7 +51,7 @@ def test_auc(self, actual_slice: GiskardDataset, model: GiskardModel, threshold=
     def _test_classification_score(self, score_fn, gsk_dataset: GiskardDataset, model: GiskardModel, threshold=1):
         is_binary_classification = len(model.classification_labels) == 2
         dataframe = gsk_dataset.df
-        prediction = model.run_predict(dataframe).raw_prediction
+        prediction = model.run_predict(gsk_dataset).raw_prediction
         labels_mapping = {model.classification_labels[i]: i for i in range(len(model.classification_labels))}
         if is_binary_classification:
             metric = score_fn(dataframe[gsk_dataset.target].map(labels_mapping), prediction)
@@ -67,7 +67,7 @@ def _test_classification_score(self, score_fn, gsk_dataset: GiskardDataset, mode
 
     def _test_accuracy_score(self, score_fn, gsk_dataset: GiskardDataset, model: GiskardModel, threshold=1):
         dataframe = gsk_dataset.df
-        prediction = model.run_predict(dataframe).raw_prediction
+        prediction = model.run_predict(gsk_dataset).raw_prediction
         labels_mapping = {model.classification_labels[i]: i for i in range(len(model.classification_labels))}
         metric = score_fn(dataframe[gsk_dataset.target].map(labels_mapping), prediction)
 
@@ -81,7 +81,7 @@ def _test_accuracy_score(self, score_fn, gsk_dataset: GiskardDataset, model: Gis
     def _test_regression_score(self, score_fn, giskard_ds, model: GiskardModel, threshold=1, negative=False,
                                r2=False):
         metric = (-1 if negative else 1) * score_fn(
-            model.run_predict(giskard_ds.df).raw_prediction,
+            model.run_predict(giskard_ds).raw_prediction,
             giskard_ds.df[giskard_ds.target]
         )
         return self.save_results(

giskard-ml-worker/ml_worker/testing/utils.py

@@ -29,7 +29,7 @@ def ge_result_to_test_result(result, passed=True) -> SingleTestResult:
     )
 
 
-def apply_perturbation_inplace(df, perturbation_dict):
+def apply_perturbation_inplace(df: pd.DataFrame, perturbation_dict):
     modified_rows = []
     i = 0
     for idx, r in df.iterrows():

giskard-ml-worker/test/test_metamorphic_direction.py

@@ -1,14 +1,15 @@
 import pytest
 
+from ml_worker.core.giskard_dataset import GiskardDataset
 from ml_worker.testing.functions import GiskardTestFunctions
 
 
-def _test_metamorphic_increasing_regression(df, model, threshold):
+def _test_metamorphic_increasing_regression(ds: GiskardDataset, model, threshold):
     tests = GiskardTestFunctions()
     perturbation = {
         "bmi": lambda x: x.bmi + x.bmi * 0.1}
     results = tests.metamorphic.test_metamorphic_increasing(
-        df=df,
+        df=ds,
         model=model,
         perturbation_dict=perturbation,
         threshold=threshold
@@ -20,12 +21,12 @@ def _test_metamorphic_increasing_regression(df, model, threshold):
     return results.passed
 
 
-def _test_metamorphic_decreasing_regression(df, model, threshold):
+def _test_metamorphic_decreasing_regression(ds: GiskardDataset, model, threshold):
     tests = GiskardTestFunctions()
     perturbation = {
         "age": lambda x: x.age - x.age * 0.1}
     results = tests.metamorphic.test_metamorphic_decreasing(
-        df=df,
+        df=ds,
         model=model,
         perturbation_dict=perturbation,
         threshold=threshold