First draft of multi-objective optimization

autosklearn/automl.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Callable, Iterable, List, Mapping, Optional, Tuple, Union
+from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, Tuple
 
 import copy
 import io
@@ -210,7 +210,7 @@ def __init__(
         get_smac_object_callback: Optional[Callable] = None,
         smac_scenario_args: Optional[Mapping] = None,
         logging_config: Optional[Mapping] = None,
-        metric: Optional[Union[Scorer, List[Scorer], Tuple[Scorer]]] = None,
+        metric: Optional[Scorer | Sequence[Scorer]] = None,
         scoring_functions: Optional[list[Scorer]] = None,
         get_trials_callback: Optional[IncorporateRunResultCallback] = None,
         dataset_compression: bool | Mapping[str, Any] = True,
@@ -692,7 +692,7 @@ def fit(
         # defined in the estimator fit call
         if self._metric is None:
             raise ValueError("No metric given.")
-        if isinstance(self._metric, (List, Tuple)):
+        if isinstance(self._metric, Sequence):
             for entry in self._metric:
                 if not isinstance(entry, Scorer):
                     raise ValueError(
@@ -796,7 +796,7 @@ def fit(
                     task=self._task,
                     metric=(
                         self._metric[0]
-                        if isinstance(self._metric, (List, Tuple))
+                        if isinstance(self._metric, Sequence)
                         else self._metric
                     ),
                     ensemble_size=self._ensemble_size,
@@ -1501,9 +1501,7 @@ def fit_ensemble(
             dataset_name=dataset_name if dataset_name else self._dataset_name,
             task=task if task else self._task,
             metric=(
-                self._metric[0]
-                if isinstance(self._metric, (List, Tuple))
-                else self._metric
+                self._metric[0] if isinstance(self._metric, Sequence) else self._metric
             ),
             ensemble_size=ensemble_size if ensemble_size else self._ensemble_size,
             ensemble_nbest=ensemble_nbest if ensemble_nbest else self._ensemble_nbest,

autosklearn/estimators.py

@@ -1,5 +1,7 @@
 # -*- encoding: utf-8 -*-
-from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
 
 import dask.distributed
 import joblib
@@ -46,7 +48,7 @@ def __init__(
         smac_scenario_args=None,
         logging_config=None,
         metadata_directory=None,
-        metric: Optional[Union[Scorer, List[Scorer], Tuple[Scorer]]] = None,
+        metric: Optional[Scorer | Sequence[Scorer]] = None,
         scoring_functions: Optional[List[Scorer]] = None,
         load_models: bool = True,
         get_trials_callback=None,

autosklearn/evaluation/__init__.py

@@ -1,5 +1,18 @@
 # -*- encoding: utf-8 -*-
-from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union, cast
+from __future__ import annotations
+
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
 
 import functools
 import json
@@ -86,10 +99,10 @@ def fit_predict_try_except_decorator(
 
 
 def get_cost_of_crash(
-    metric: Union[Scorer, List[Scorer], Tuple[Scorer]]
+    metric: Union[Scorer | Sequence[Scorer]],
 ) -> Union[float, List[float]]:
 
-    if isinstance(metric, (List, Tuple)):
+    if isinstance(metric, Sequence):
         return [cast(float, get_cost_of_crash(metric_)) for metric_ in metric]
     elif not isinstance(metric, Scorer):
         raise ValueError("The metric must be stricly be an instance of Scorer")
@@ -129,7 +142,7 @@ def __init__(
         resampling_strategy: Union[
             str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
         ],
-        metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+        metric: Union[Scorer | Sequence[Scorer]],
         cost_for_crash: float,
         abort_on_first_run_crash: bool,
         port: int,

autosklearn/evaluation/abstract_evaluator.py

@@ -1,4 +1,6 @@
-from typing import Any, Dict, List, Optional, TextIO, Tuple, Type, Union, cast
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Sequence, TextIO, Tuple, Type, Union, cast
 
 import logging
 import multiprocessing
@@ -184,7 +186,7 @@ def __init__(
         self,
         backend: Backend,
         queue: multiprocessing.Queue,
-        metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+        metric: Union[Scorer | Sequence[Scorer]],
         additional_components: Dict[str, ThirdPartyComponents],
         port: Optional[int],
         configuration: Optional[Union[int, Configuration]] = None,
@@ -338,6 +340,7 @@ def _loss(
             y_true
         """
         if not isinstance(self.configuration, Configuration):
+            # Dummy prediction
             if self.scoring_functions:
                 if isinstance(self.metric, Scorer):
                     return {self.metric.name: self.metric._worst_possible_result}

autosklearn/evaluation/test_evaluator.py

@@ -1,5 +1,7 @@
 # -*- encoding: utf-8 -*-
-from typing import Any, Dict, List, Optional, Tuple, Union
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
 import multiprocessing
 
@@ -23,7 +25,7 @@ def __init__(
         self,
         backend: Backend,
         queue: multiprocessing.Queue,
-        metric: Scorer,
+        metric: Union[Scorer | Sequence[Scorer]],
         additional_components: Dict[str, ThirdPartyComponents],
         port: Optional[int],
         configuration: Optional[Union[int, Configuration]] = None,
@@ -111,7 +113,7 @@ def eval_t(
     queue: multiprocessing.Queue,
     config: Union[int, Configuration],
     backend: Backend,
-    metric: Scorer,
+    metric: Union[Scorer | Sequence[Scorer]],
     seed: int,
     num_run: int,
     instance: Dict[str, Any],

autosklearn/evaluation/train_evaluator.py

@@ -1,4 +1,6 @@
-from typing import Any, Dict, List, Optional, Tuple, Union, cast
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, cast
 
 import copy
 import json
@@ -182,7 +184,7 @@ def __init__(
         self,
         backend: Backend,
         queue: multiprocessing.Queue,
-        metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+        metric: Union[Scorer | Sequence[Scorer]],
         additional_components: Dict[str, ThirdPartyComponents],
         port: Optional[int],
         configuration: Optional[Union[int, Configuration]] = None,
@@ -1328,7 +1330,7 @@ def eval_holdout(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+    metric: Union[Scorer | Sequence[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1375,7 +1377,7 @@ def eval_iterative_holdout(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+    metric: Union[Scorer | Sequence[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1422,7 +1424,7 @@ def eval_partial_cv(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+    metric: Union[Scorer | Sequence[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1475,7 +1477,7 @@ def eval_partial_cv_iterative(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+    metric: Union[Scorer | Sequence[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1523,7 +1525,7 @@ def eval_cv(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+    metric: Union[Scorer | Sequence[Scorer]],
     seed: int,
     num_run: int,
     instance: str,
@@ -1571,7 +1573,7 @@ def eval_iterative_cv(
         str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit
     ],
     resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]],
-    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+    metric: Union[Scorer | Sequence[Scorer]],
     seed: int,
     num_run: int,
     instance: str,

autosklearn/metrics/__init__.py

@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 from abc import ABCMeta, abstractmethod
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union, cast
 
 from functools import partial
 from itertools import product
@@ -388,7 +390,7 @@ def calculate_score(
     solution: np.ndarray,
     prediction: np.ndarray,
     task_type: int,
-    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+    metric: Scorer | Sequence[Scorer],
     scoring_functions: Optional[List[Scorer]] = None,
 ) -> Union[float, Dict[str, float]]:
     """
@@ -420,7 +422,7 @@ def calculate_score(
     to_score = []
     if scoring_functions:
         to_score.extend(scoring_functions)
-    if isinstance(metric, (list, tuple)):
+    if isinstance(metric, Sequence):
         to_score.extend(metric)
     else:
         to_score.append(metric)
@@ -480,7 +482,7 @@ def calculate_loss(
     solution: np.ndarray,
     prediction: np.ndarray,
     task_type: int,
-    metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+    metric: Scorer | Sequence[Scorer],
     scoring_functions: Optional[List[Scorer]] = None,
 ) -> Union[float, Dict[str, float]]:
     """
@@ -516,7 +518,7 @@ def calculate_loss(
         scoring_functions=scoring_functions,
     )
 
-    if scoring_functions or isinstance(metric, (list, tuple)):
+    if scoring_functions or isinstance(metric, Sequence):
         score = cast(Dict, score)
         scoring_functions = cast(List, scoring_functions)
         metric_list = list(cast(List, metric))  # Please mypy

autosklearn/smbo.py

@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 import typing
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Sequence
 
 import copy
 import json
@@ -260,7 +262,7 @@ def __init__(
         total_walltime_limit,
         func_eval_time_limit,
         memory_limit,
-        metric: Union[Scorer, List[Scorer], Tuple[Scorer]],
+        metric: Scorer | Sequence[Scorer],
         stopwatch: StopWatch,
         n_jobs,
         dask_client: dask.distributed.Client,
@@ -362,9 +364,7 @@ def collect_metalearning_suggestions(self, meta_base):
                 meta_base=meta_base,
                 basename=self.dataset_name,
                 metric=(
-                    self.metric[0]
-                    if isinstance(self.metric, (List, Tuple))
-                    else self.metric
+                    self.metric[0] if isinstance(self.metric, Sequence) else self.metric
                 ),
                 configuration_space=self.config_space,
                 task=self.task,