First draft of multi-objective optimization

mfeurer · KEggensperger · mfeurer · commit 288569253d09 · 2022-05-03T17:31:59.000+02:00
Co-authored-by: Katharina Eggensperger &lt;eggenspk@informatik.uni-freiburg.de&gt;
diff --git a/autosklearn/automl.py b/autosklearn/automl.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Callable, Iterable, Mapping, Optional, Tuple
+from typing import Any, Callable, Iterable, List, Mapping, Optional, Tuple, Union
 
 import copy
 import io
@@ -210,7 +210,7 @@ def __init__(
         get_smac_object_callback: Optional[Callable] = None,
         smac_scenario_args: Optional[Mapping] = None,
         logging_config: Optional[Mapping] = None,
-        metric: Optional[Scorer] = None,
+        metric: Optional[Union[Scorer, List[Scorer], Tuple[Scorer]]] = None,
         scoring_functions: Optional[list[Scorer]] = None,
         get_trials_callback: Optional[IncorporateRunResultCallback] = None,
         dataset_compression: bool | Mapping[str, Any] = True,
@@ -265,7 +265,7 @@ def __init__(
             initial_configurations_via_metalearning
         )
 
-        self._scoring_functions = scoring_functions or {}
+        self._scoring_functions = scoring_functions or []
         self._resampling_strategy_arguments = resampling_strategy_arguments or {}
 
         # Single core, local runs should use fork to prevent the __main__ requirements
@@ -692,10 +692,14 @@ def fit(
         # defined in the estimator fit call
         if self._metric is None:
             raise ValueError("No metric given.")
-        if not isinstance(self._metric, Scorer):
-            raise ValueError(
-                "Metric must be instance of " "autosklearn.metrics.Scorer."
-            )
+        if isinstance(self._metric, (List, Tuple)):
+            for entry in self._metric:
+                if not isinstance(entry, Scorer):
+                    raise ValueError(
+                        "Metric must be instance of autosklearn.metrics.Scorer."
+                    )
+        elif not isinstance(self._metric, Scorer):
+            raise ValueError("Metric must be instance of autosklearn.metrics.Scorer.")
 
         # If no dask client was provided, we create one, so that we can
         # start a ensemble process in parallel to smbo optimize
@@ -790,7 +794,11 @@ def fit(
                     backend=copy.deepcopy(self._backend),
                     dataset_name=dataset_name,
                     task=self._task,
-                    metric=self._metric,
+                    metric=(
+                        self._metric[0]
+                        if isinstance(self._metric, (List, Tuple))
+                        else self._metric
+                    ),
                     ensemble_size=self._ensemble_size,
                     ensemble_nbest=self._ensemble_nbest,
                     max_models_on_disc=self._max_models_on_disc,
@@ -1492,7 +1500,11 @@ def fit_ensemble(
             backend=copy.deepcopy(self._backend),
             dataset_name=dataset_name if dataset_name else self._dataset_name,
             task=task if task else self._task,
-            metric=self._metric,
+            metric=(
+                self._metric[0]
+                if isinstance(self._metric, (List, Tuple))
+                else self._metric
+            ),
             ensemble_size=ensemble_size if ensemble_size else self._ensemble_size,
             ensemble_nbest=ensemble_nbest if ensemble_nbest else self._ensemble_nbest,
             max_models_on_disc=self._max_models_on_disc,
@@ -1912,7 +1924,6 @@ def show_models(self) -> dict[int, Any]:
         .. code-block:: python
 
             import sklearn.datasets
-            import sklearn.metrics
             import autosklearn.regression
 
             X, y = sklearn.datasets.load_diabetes(return_X_y=True)
diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py
@@ -46,7 +46,7 @@ def __init__(
         smac_scenario_args=None,
         logging_config=None,
         metadata_directory=None,
-        metric=None,
+        metric: Optional[Union[Scorer, List[Scorer], Tuple[Scorer]]] = None,
         scoring_functions: Optional[List[Scorer]] = None,
         load_models: bool = True,
         get_trials_callback=None,
diff --git a/autosklearn/evaluation/__init__.py b/autosklearn/evaluation/__init__.py
@@ -85,10 +85,13 @@ def fit_predict_try_except_decorator(
         queue.close()
 
 
-def get_cost_of_crash(metric: Scorer) -> float:
+def get_cost_of_crash(
+    metric: Union[Scorer, List[Scorer], Tuple[Scorer]]
+) -> Union[float, List[float]]:
 
-    # The metric must always be defined to extract optimum/worst
-    if not isinstance(metric, Scorer):
+    if isinstance(metric, (List, Tuple)):
+        return [cast(float, get_cost_of_crash(metric_)) for metric_ in metric]
+    elif not isinstance(metric, Scorer):
         raise ValueError("The metric must be stricly be an instance of Scorer")
 
     # Autosklearn optimizes the err. This function translates
@@ -126,7 +129,7 @@ def __init__(
         resampling_strategy: Union[
             str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
         ],
-        metric: Scorer,
+        metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
         cost_for_crash: float,
         abort_on_first_run_crash: bool,
         port: int,
@@ -144,7 +147,7 @@ def __init__(
         disable_file_output: bool = False,
         init_params: Optional[Dict[str, Any]] = None,
         budget_type: Optional[str] = None,
-        ta: Optional[Callable] = None,
+        ta: Optional[Callable] = None,  # Required by SMAC's parent class
         **resampling_strategy_args: Any,
     ):
         if resampling_strategy == "holdout":
@@ -186,6 +189,7 @@ def __init__(
             par_factor=par_factor,
             cost_for_crash=self.worst_possible_result,
             abort_on_first_run_crash=abort_on_first_run_crash,
+            multi_objectives=multi_objectives,
         )
 
         self.backend = backend
@@ -550,4 +554,32 @@ def run(
 
         autosklearn.evaluation.util.empty_queue(queue)
         self.logger.info("Finished evaluating configuration %d" % config_id)
+
+        # Do some sanity checking (for multi objective)
+        if len(self.multi_objectives) > 1:
+            error = (
+                f"Returned costs {cost} does not match the number of objectives"
+                f" {len(self.multi_objectives)}."
+            )
+
+            # If dict convert to array
+            # Make sure the ordering is correct
+            if isinstance(cost, dict):
+                ordered_cost = []
+                for name in self.multi_objectives:
+                    if name not in cost:
+                        raise RuntimeError(
+                            f"Objective {name} was not found in the returned costs."
+                        )
+
+                    ordered_cost.append(cost[name])
+                cost = ordered_cost
+
+            if isinstance(cost, list):
+                if len(cost) != len(self.multi_objectives):
+                    raise RuntimeError(error)
+
+            if isinstance(cost, float):
+                raise RuntimeError(error)
+
         return status, cost, runtime, additional_run_info
diff --git a/autosklearn/evaluation/abstract_evaluator.py b/autosklearn/evaluation/abstract_evaluator.py
@@ -184,7 +184,7 @@ def __init__(
         self,
         backend: Backend,
         queue: multiprocessing.Queue,
-        metric: Scorer,
+        metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
         additional_components: Dict[str, ThirdPartyComponents],
         port: Optional[int],
         configuration: Optional[Union[int, Configuration]] = None,
@@ -326,7 +326,6 @@ def _loss(
         self,
         y_true: np.ndarray,
         y_hat: np.ndarray,
-        scoring_functions: Optional[List[Scorer]] = None,
     ) -> Union[float, Dict[str, float]]:
         """Auto-sklearn follows a minimization goal.
         The calculate_loss internally translate a score function to
@@ -338,21 +337,30 @@ def _loss(
         ----------
             y_true
         """
-        scoring_functions = (
-            self.scoring_functions if scoring_functions is None else scoring_functions
-        )
         if not isinstance(self.configuration, Configuration):
-            if scoring_functions:
-                return {self.metric.name: self.metric._worst_possible_result}
+            if self.scoring_functions:
+                if isinstance(self.metric, Scorer):
+                    return {self.metric.name: self.metric._worst_possible_result}
+                else:
+                    return {
+                        metric.name: metric._worst_possible_result
+                        for metric in self.metric
+                    }
             else:
-                return self.metric._worst_possible_result
+                if isinstance(self.metric, Scorer):
+                    return self.metric._worst_possible_result
+                else:
+                    return {
+                        metric.name: metric._worst_possible_result
+                        for metric in self.metric
+                    }
 
         return calculate_loss(
             y_true,
             y_hat,
             self.task_type,
             self.metric,
-            scoring_functions=scoring_functions,
+            scoring_functions=self.scoring_functions,
         )
 
     def finish_up(
@@ -402,7 +410,10 @@ def finish_up(
 
         if isinstance(loss, dict):
             loss_ = loss
-            loss = loss_[self.metric.name]
+            if isinstance(self.metric, Scorer):
+                loss = loss_[self.metric.name]
+            else:
+                loss = {metric: loss_[metric] for metric in loss_}
         else:
             loss_ = {}
 
diff --git a/autosklearn/evaluation/train_evaluator.py b/autosklearn/evaluation/train_evaluator.py
@@ -182,7 +182,7 @@ def __init__(
         self,
         backend: Backend,
         queue: multiprocessing.Queue,
-        metric: Scorer,
+        metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
         additional_components: Dict[str, ThirdPartyComponents],
         port: Optional[int],
         configuration: Optional[Union[int, Configuration]] = None,
@@ -645,19 +645,31 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
             # train_losses is a list of either scalars or dicts. If it contains dicts,
             # then train_loss is computed using the target metric (self.metric).
             if all(isinstance(elem, dict) for elem in train_losses):
-                train_loss = np.average(
-                    [
-                        train_losses[i][str(self.metric)]
-                        for i in range(self.num_cv_folds)
-                    ],
-                    weights=train_fold_weights,
-                )
+                if isinstance(self.metric, Scorer):
+                    train_loss = np.average(
+                        [
+                            train_losses[i][str(self.metric)]
+                            for i in range(self.num_cv_folds)
+                        ],
+                        weights=train_fold_weights,
+                    )
+                else:
+                    train_loss = [
+                        np.average(
+                            [
+                                train_losses[i][str(metric)]
+                                for i in range(self.num_cv_folds)
+                            ],
+                            weights=train_fold_weights,
+                        )
+                        for metric in self.metric
+                    ]
             else:
                 train_loss = np.average(train_losses, weights=train_fold_weights)
 
             # if all_scoring_function is true, return a dict of opt_loss. Otherwise,
             # return a scalar.
-            if self.scoring_functions:
+            if self.scoring_functions or not isinstance(self.metric, Scorer):
                 opt_loss = {}
                 for metric in opt_losses[0].keys():
                     opt_loss[metric] = np.average(
@@ -1316,7 +1328,7 @@ def eval_holdout(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Scorer,
+    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1363,7 +1375,7 @@ def eval_iterative_holdout(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Scorer,
+    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1410,7 +1422,7 @@ def eval_partial_cv(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Scorer,
+    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1463,7 +1475,7 @@ def eval_partial_cv_iterative(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Scorer,
+    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1511,7 +1523,7 @@ def eval_cv(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Scorer,
+    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1559,7 +1571,7 @@ def eval_iterative_cv(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Scorer,
+    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
diff --git a/autosklearn/metrics/__init__.py b/autosklearn/metrics/__init__.py
diff --git a/autosklearn/smbo.py b/autosklearn/smbo.py
