diff --git a/autosklearn/automl.py b/autosklearn/automl.py
index 8726ab8c6d..cf971456e3 100644
--- a/autosklearn/automl.py
+++ b/autosklearn/automl.py
@@ -79,7 +79,7 @@
     AbstractMultiObjectiveEnsemble,
 )
 from autosklearn.ensembles.ensemble_selection import EnsembleSelection
-from autosklearn.ensembles.singlebest_ensemble import SingleBest
+from autosklearn.ensembles.singlebest_ensemble import SingleBestFromRunhistory
 from autosklearn.evaluation import ExecuteTaFuncWithQueue, get_cost_of_crash
 from autosklearn.evaluation.abstract_evaluator import _fit_and_suppress_warnings
 from autosklearn.evaluation.train_evaluator import TrainEvaluator, _fit_with_budget
@@ -1666,7 +1666,7 @@ def _load_best_individual_model(self):
             return None
 
         # SingleBest contains the best model found by AutoML
-        ensemble = SingleBest(
+        ensemble = SingleBestFromRunhistory(
             metrics=self._metrics,
             task_type=self._task,
             seed=self._seed,
@@ -1693,13 +1693,12 @@ def _load_pareto_set(self) -> Sequence[VotingClassifier | VotingRegressor]:
             raise ValueError("Pareto set only available if ensemble can be loaded.")
 
         if isinstance(self.ensemble_, AbstractMultiObjectiveEnsemble):
-            pareto_set = self.ensemble_.get_pareto_set()
+            pareto_set = self.ensemble_.pareto_set
         else:
             self._logger.warning(
                 "Pareto set not available for single objective ensemble "
                 "method. The Pareto set will only include the single ensemble "
-                "constructed by %s",
-                type(self.ensemble_),
+                f"constructed by {type(self.ensemble_)},"
             )
             pareto_set = [self.ensemble_]
 
@@ -2148,7 +2147,7 @@ def show_models(self) -> dict[int, Any]:
 
         ensemble_dict = {}
 
-        if self._ensemble_class is not None:
+        if self._ensemble_class is None:
             warnings.warn(
                 "No models in the ensemble. Kindly provide an ensemble class."
             )
diff --git a/autosklearn/ensembles/__init__.py b/autosklearn/ensembles/__init__.py
index 4e1af926cf..40bb380bdb 100644
--- a/autosklearn/ensembles/__init__.py
+++ b/autosklearn/ensembles/__init__.py
@@ -1,10 +1,18 @@
 from .abstract_ensemble import AbstractEnsemble, AbstractMultiObjectiveEnsemble
 from .ensemble_selection import EnsembleSelection
-from .singlebest_ensemble import SingleBest
+from .multiobjective_dummy_ensemble import MultiObjectiveDummyEnsemble
+from .singlebest_ensemble import (
+    SingleBest,
+    SingleBestFromRunhistory,
+    SingleModelEnsemble,
+)
 
 __all__ = [
     "AbstractEnsemble",
     "AbstractMultiObjectiveEnsemble",
     "EnsembleSelection",
+    "SingleBestFromRunhistory",
     "SingleBest",
+    "SingleModelEnsemble",
+    "MultiObjectiveDummyEnsemble",
 ]
diff --git a/autosklearn/ensembles/abstract_ensemble.py b/autosklearn/ensembles/abstract_ensemble.py
index ea14a45e90..c4388b71fd 100644
--- a/autosklearn/ensembles/abstract_ensemble.py
+++ b/autosklearn/ensembles/abstract_ensemble.py
@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from typing import Dict, List, Sequence, Tuple, Union
+from typing import Any, Dict, List, Sequence, Tuple, Union
 
 import numpy as np
 
@@ -18,19 +18,29 @@ def __init__(
         self,
         task_type: int,
         metrics: Sequence[Scorer] | Scorer,
-        random_state: int | np.random.RandomState | None,
         backend: Backend,
+        random_state: int | np.random.RandomState | None = None,
     ):
         pass
 
+    def __getstate__(self) -> Dict[str, Any]:
+        # Cannot serialize a metric if
+        # it is user defined.
+        # That is, if doing pickle dump
+        # the metric won't be the same as the
+        # one in __main__. we don't use the metric
+        # in the EnsembleSelection so this should
+        # be fine
+        return {key: value for key, value in self.__dict__.items() if key != "metrics"}
+
     @abstractmethod
     def fit(
         self,
         base_models_predictions: np.ndarray | List[np.ndarray],
-        X_data: SUPPORTED_FEAT_TYPES,
         true_targets: np.ndarray,
         model_identifiers: List[Tuple[int, int, float]],
         runs: Sequence[Run],
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
     ) -> "AbstractEnsemble":
         """Fit an ensemble given predictions of base models and targets.
 
@@ -79,7 +89,7 @@ def predict(
 
         Returns
         -------
-        array : [n_data_points]
+        np.ndarray
         """
         pass
 
@@ -97,7 +107,7 @@ def get_models_with_weights(
 
         Returns
         -------
-        array : [(weight_1, model_1), ..., (weight_n, model_n)]
+        List[Tuple[float, BasePipeline]]
         """
 
     @abstractmethod
@@ -115,7 +125,7 @@ def get_identifiers_with_weights(
 
         Returns
         -------
-        array : [(identifier_1, weight_1), ..., (identifier_n, weight_n)]
+        List[Tuple[Tuple[int, int, float], float]
         """
 
     @abstractmethod
@@ -133,12 +143,25 @@ def get_selected_model_identifiers(self) -> List[Tuple[int, int, float]]:
     def get_validation_performance(self) -> float:
         """Return validation performance of ensemble.
 
-        Return
-        ------
+        Returns
+        -------
         float
         """
 
 
 class AbstractMultiObjectiveEnsemble(AbstractEnsemble):
-    def get_pareto_set(self) -> Sequence[AbstractEnsemble]:
-        pass
+    @property
+    @abstractmethod
+    def pareto_set(self) -> Sequence[AbstractEnsemble]:
+        """Get a sequence on ensembles that are on the pareto front
+
+        Raises
+        ------
+        SklearnNotFittedError
+            If ``fit`` has not been called and the pareto set does not exist yet
+
+        Returns
+        -------
+        Sequence[AbstractEnsemble]
+        """
+        ...
diff --git a/autosklearn/ensembles/ensemble_selection.py b/autosklearn/ensembles/ensemble_selection.py
index 310f2eb6a4..e5e3c835b5 100644
--- a/autosklearn/ensembles/ensemble_selection.py
+++ b/autosklearn/ensembles/ensemble_selection.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
+from typing import Dict, List, Sequence, Tuple, Union
 
 import random
 import warnings
@@ -23,11 +23,11 @@ def __init__(
         self,
         task_type: int,
         metrics: Sequence[Scorer] | Scorer,
-        random_state: Optional[Union[int, np.random.RandomState]],
         backend: Backend,
         ensemble_size: int = 50,
         bagging: bool = False,
         mode: str = "fast",
+        random_state: int | np.random.RandomState | None = None,
     ) -> None:
         """An ensemble of selected algorithms
 
@@ -43,14 +43,6 @@ def __init__(
             The metric used to evaluate the models. If multiple metrics are passed,
             ensemble selection only optimizes for the first
 
-        random_state: Optional[int | RandomState] = None
-            The random_state used for ensemble selection.
-
-            * None - Uses numpy's default RandomState object
-            * int - Successive calls to fit will produce the same results
-            * RandomState - Truly random, each call to fit will produce
-              different results, even with the same object.
-
         backend : Backend
             Gives access to the backend of Auto-sklearn. Not used by Ensemble Selection.
 
@@ -62,6 +54,14 @@ def __init__(
             * 'slow' - The original method used in Rich Caruana's ensemble selection.
             * 'fast' - A faster version of Rich Caruanas' ensemble selection.
 
+        random_state: int | RandomState | None = None
+            The random_state used for ensemble selection.
+
+            * None - Uses numpy's default RandomState object
+            * int - Successive calls to fit will produce the same results
+            * RandomState - Truly random, each call to fit will produce
+              different results, even with the same object.
+
         References
         ----------
         | Ensemble selection from libraries of models
@@ -92,23 +92,13 @@ def __init__(
         # https://scikit-learn.org/stable/common_pitfalls.html#controlling-randomness
         self.random_state = random_state
 
-    def __getstate__(self) -> Dict[str, Any]:
-        # Cannot serialize a metric if
-        # it is user defined.
-        # That is, if doing pickle dump
-        # the metric won't be the same as the
-        # one in __main__. we don't use the metric
-        # in the EnsembleSelection so this should
-        # be fine
-        return {key: value for key, value in self.__dict__.items() if key != "metrics"}
-
     def fit(
         self,
         base_models_predictions: List[np.ndarray],
-        X_data: SUPPORTED_FEAT_TYPES,
         true_targets: np.ndarray,
         model_identifiers: List[Tuple[int, int, float]],
         runs: Sequence[Run],
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
     ) -> EnsembleSelection:
         self.ensemble_size = int(self.ensemble_size)
         if self.ensemble_size < 1:
@@ -141,20 +131,22 @@ def fit(
     def _fit(
         self,
         predictions: List[np.ndarray],
-        X_data: SUPPORTED_FEAT_TYPES,
         labels: np.ndarray,
+        *,
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
     ) -> EnsembleSelection:
         if self.mode == "fast":
-            self._fast(predictions, X_data, labels)
+            self._fast(predictions=predictions, X_data=X_data, labels=labels)
         else:
-            self._slow(predictions, X_data, labels)
+            self._slow(predictions=predictions, X_data=X_data, labels=labels)
         return self
 
     def _fast(
         self,
         predictions: List[np.ndarray],
-        X_data: SUPPORTED_FEAT_TYPES,
         labels: np.ndarray,
+        *,
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
     ) -> None:
         """Fast version of Rich Caruana's ensemble selection method."""
         self.num_input_models_ = len(predictions)
@@ -231,8 +223,9 @@ def _fast(
     def _slow(
         self,
         predictions: List[np.ndarray],
-        X_data: SUPPORTED_FEAT_TYPES,
         labels: np.ndarray,
+        *,
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
     ) -> None:
         """Rich Caruana's ensemble selection method."""
         self.num_input_models_ = len(predictions)
@@ -311,7 +304,7 @@ def _bagging(
             # Bagging a set of models
             indices = sorted(random.sample(range(0, n_models), bag_size))
             bag = predictions[indices, :, :]
-            order, _ = self._fit(bag, labels)
+            order, _ = self._fit(predictions=bag, labels=labels)
             order_of_each_bag.append(order)
 
         return np.array(
diff --git a/autosklearn/ensembles/multiobjective_dummy_ensemble.py b/autosklearn/ensembles/multiobjective_dummy_ensemble.py
new file mode 100644
index 0000000000..8751507ecd
--- /dev/null
+++ b/autosklearn/ensembles/multiobjective_dummy_ensemble.py
@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+from typing import Sequence
+
+import warnings
+
+import numpy as np
+from sklearn.exceptions import NotFittedError
+
+from autosklearn.automl_common.common.utils.backend import Backend
+from autosklearn.constants import TASK_TYPES
+from autosklearn.data.validation import SUPPORTED_FEAT_TYPES
+from autosklearn.ensemble_building.run import Run
+from autosklearn.ensembles.abstract_ensemble import (
+    AbstractEnsemble,
+    AbstractMultiObjectiveEnsemble,
+)
+from autosklearn.ensembles.singlebest_ensemble import SingleModelEnsemble
+from autosklearn.metrics import Scorer, calculate_losses
+from autosklearn.pipeline.base import BasePipeline
+from autosklearn.util.multiobjective import pareto_front
+
+
+class MultiObjectiveDummyEnsemble(AbstractMultiObjectiveEnsemble):
+    def __init__(
+        self,
+        task_type: int,
+        metrics: Sequence[Scorer] | Scorer,
+        backend: Backend,
+        random_state: int | np.random.RandomState | None = None,
+    ) -> None:
+        """A dummy implementation of a multi-objective ensemble.
+
+        Builds ensembles that are individual models on the Pareto front each.
+
+        Parameters
+        ----------
+        task_type: int
+            An identifier indicating which task is being performed.
+
+        metrics: Sequence[Scorer] | Scorer
+            The metrics used to evaluate the models.
+
+        backend : Backend
+            Gives access to the backend of Auto-sklearn. Not used.
+
+        random_state: int | RandomState | None = None
+            Not used.
+        """
+        self.task_type = task_type
+        if isinstance(metrics, Sequence):
+            if len(metrics) == 1:
+                warnings.warn(
+                    "Passed only a single metric to a multi-objective ensemble. "
+                    "Please use a single-objective ensemble in such cases."
+                )
+            self.metrics = metrics
+        else:
+            self.metric = [metrics]
+        self.random_state = random_state
+        self.backend = backend
+
+    @property
+    def pareto_set(self) -> Sequence[AbstractEnsemble]:
+        if not hasattr(self, "pareto_set_"):
+            raise NotFittedError("`pareto_set` not created, please call `fit()` first")
+        return self.pareto_set_
+
+    def fit(
+        self,
+        base_models_predictions: list[np.ndarray],
+        true_targets: np.ndarray,
+        model_identifiers: list[tuple[int, int, float]],
+        runs: Sequence[Run],
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
+    ) -> MultiObjectiveDummyEnsemble:
+        """Select dummy ensembles given predictions of base models and targets.
+
+        Parameters
+        ----------
+        base_models_predictions: np.ndarray
+            shape = (n_base_models, n_data_points, n_targets)
+            n_targets is the number of classes in case of classification,
+            n_targets is 0 or 1 in case of regression
+
+            Can be a list of 2d numpy arrays as well to prevent copying all
+            predictions into a single, large numpy array.
+
+        true_targets : array of shape [n_targets]
+
+        model_identifiers : identifier for each base model.
+            Can be used for practical text output of the ensemble.
+
+        runs: Sequence[Run]
+            Additional information for each run executed by SMAC that was
+            considered by the ensemble builder. Not used.
+
+        X_data : list-like | sparse matrix | None = None
+            X data to give to the metric if required
+
+        Returns
+        -------
+        self
+        """
+        if self.task_type not in TASK_TYPES:
+            raise ValueError("Unknown task type %s." % self.task_type)
+
+        all_costs = np.empty((len(base_models_predictions), len(self.metrics)))
+        for i, base_model_prediction in enumerate(base_models_predictions):
+            losses = calculate_losses(
+                solution=true_targets,
+                prediction=base_model_prediction,
+                task_type=self.task_type,
+                metrics=self.metrics,
+                X_data=X_data,
+            )
+            all_costs[i] = [losses[metric.name] for metric in self.metrics]
+
+        all_costs = np.array(all_costs)
+        sort_by_first_metric = np.argsort(all_costs[:, 0])
+        efficient_points = pareto_front(all_costs, is_loss=True)
+        pareto_set = []
+
+        for argsort_idx in sort_by_first_metric:
+            if not efficient_points[argsort_idx]:
+                continue
+            ensemble = SingleModelEnsemble(
+                task_type=self.task_type,
+                metrics=self.metrics,
+                random_state=self.random_state,
+                backend=self.backend,
+                model_index=argsort_idx,
+            )
+            ensemble.fit(
+                base_models_predictions=base_models_predictions,
+                true_targets=true_targets,
+                model_identifiers=model_identifiers,
+                runs=runs,
+                X_data=X_data,
+            )
+            pareto_set.append(ensemble)
+
+        self.pareto_set_ = pareto_set
+        return self
+
+    def predict(
+        self, base_models_predictions: np.ndarray | list[np.ndarray]
+    ) -> np.ndarray:
+        """Predict using the ensemble which is best for the 1st metric.
+
+        Parameters
+        ----------
+        base_models_predictions : np.ndarray
+            shape = (n_base_models, n_data_points, n_targets)
+            Same as in the fit method.
+
+        Returns
+        -------
+        np.ndarray
+        """
+        return self.pareto_set[0].predict(base_models_predictions)
+
+    def __str__(self) -> str:
+        return "MultiObjectiveDummyEnsemble: %d models" % len(self.pareto_set)
+
+    def get_models_with_weights(
+        self, models: dict[tuple[int, int, float], BasePipeline]
+    ) -> list[tuple[float, BasePipeline]]:
+        """Return a list of (weight, model) pairs for the ensemble that is
+        best for the 1st metric.
+
+        Parameters
+        ----------
+        models : dict {identifier : model object}
+            The identifiers are the same as the one presented to the fit()
+            method. Models can be used for nice printing.
+
+        Returns
+        -------
+        list[tuple[float, BasePipeline]]
+        """
+        return self.pareto_set[0].get_models_with_weights(models)
+
+    def get_identifiers_with_weights(
+        self,
+    ) -> list[tuple[tuple[int, int, float], float]]:
+        """Return a (identifier, weight)-pairs for all models that were passed to the
+        ensemble builder based on the ensemble that is best for the 1st metric.
+
+        Parameters
+        ----------
+        models : dict {identifier : model object}
+            The identifiers are the same as the one presented to the fit()
+            method. Models can be used for nice printing.
+
+        Returns
+        -------
+        list[tuple[tuple[int, int, float], float]
+        """
+        return self.pareto_set[0].get_identifiers_with_weights()
+
+    def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]:
+        """Return identifiers of models in the ensemble that is best for the 1st metric.
+
+        This includes models which have a weight of zero!
+
+        Returns
+        -------
+        list
+        """
+        return self.pareto_set[0].get_selected_model_identifiers()
+
+    def get_validation_performance(self) -> float:
+        """Return validation performance of the ensemble that is best for the 1st metric.
+
+        Returns
+        -------
+        float
+        """
+        return self.pareto_set[0].get_validation_performance()
diff --git a/autosklearn/ensembles/singlebest_ensemble.py b/autosklearn/ensembles/singlebest_ensemble.py
index 497ee4d495..b072bcab11 100644
--- a/autosklearn/ensembles/singlebest_ensemble.py
+++ b/autosklearn/ensembles/singlebest_ensemble.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Dict, List, Sequence, Tuple, Union
+from typing import Sequence
 
 import os
 
@@ -11,29 +11,36 @@
 from autosklearn.data.validation import SUPPORTED_FEAT_TYPES
 from autosklearn.ensemble_building.run import Run
 from autosklearn.ensembles.abstract_ensemble import AbstractEnsemble
-from autosklearn.metrics import Scorer
+from autosklearn.metrics import Scorer, calculate_losses
 from autosklearn.pipeline.base import BasePipeline
 
 
-class SingleBest(AbstractEnsemble):
-    """
-    In the case of a crash, this class searches
-    for the best individual model.
+class AbstractSingleModelEnsemble(AbstractEnsemble):
+    """Ensemble consisting of a single model.
 
-    Such model is returned as an ensemble of a single
-    object, to comply with the expected interface of an
-    AbstractEnsemble.
+    Parameters
+    ----------
+    task_type: int
+        An identifier indicating which task is being performed.
+
+    metrics: Sequence[Scorer] | Scorer
+        The metrics used to evaluate the models.
+
+    backend : Backend
+        Gives access to the backend of Auto-sklearn. Not used.
+
+    random_state: int | RandomState | None = None
+        Not used.
     """
 
     def __init__(
         self,
         task_type: int,
         metrics: Sequence[Scorer] | Scorer,
-        random_state: int | np.random.RandomState | None,
         backend: Backend,
-        run_history: RunHistory,
-        seed: int,
+        random_state: int | np.random.RandomState | None = None,
     ):
+        self.weights_ = [1.0]
         self.task_type = task_type
         if isinstance(metrics, Sequence):
             self.metrics = metrics
@@ -41,26 +48,344 @@ def __init__(
             self.metrics = [metrics]
         else:
             raise TypeError(type(metrics))
-        self.seed = seed
+        self.random_state = random_state
         self.backend = backend
 
-        # Add some default values -- at least 1 model in ensemble is assumed
-        self.indices_ = [0]
-        self.weights_ = [1.0]
-        self.run_history = run_history
-        self.identifiers_ = self.get_identifiers_from_run_history()
+    def fit(
+        self,
+        base_models_predictions: np.ndarray | list[np.ndarray],
+        true_targets: np.ndarray,
+        model_identifiers: list[tuple[int, int, float]],
+        runs: Sequence[Run],
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
+    ) -> AbstractSingleModelEnsemble:
+        """Fit the ensemble
+
+        Parameters
+        ----------
+        base_models_predictions: np.ndarray
+            shape = (n_base_models, n_data_points, n_targets)
+            n_targets is the number of classes in case of classification,
+            n_targets is 0 or 1 in case of regression
+
+            Can be a list of 2d numpy arrays as well to prevent copying all
+            predictions into a single, large numpy array.
+
+        true_targets : array of shape [n_targets]
+
+        model_identifiers : identifier for each base model.
+            Can be used for practical text output of the ensemble.
+
+        runs: Sequence[Run]
+            Additional information for each run executed by SMAC that was
+            considered by the ensemble builder.
+
+        X_data : list-like | sparse matrix | None = None
+
+        Returns
+        -------
+        self
+        """
+        return self
+
+    def predict(self, predictions: np.ndarray | list[np.ndarray]) -> np.ndarray:
+        """Select the predictions of the selected model.
+
+        Parameters
+        ----------
+        base_models_predictions : np.ndarray
+            shape = (n_base_models, n_data_points, n_targets)
+            Same as in the fit method.
+
+        Returns
+        -------
+        np.ndarray
+        """
+        return predictions[0]
+
+    def __str__(self) -> str:
+        return "%s:\n\tMembers: %s" "\n\tWeights: %s\n\tIdentifiers: [%s]" % (
+            self.__class__.__name__,
+            self.indices_,  # type: ignore [attr-defined]
+            self.weights_,
+            self.identifiers_[0],  # type: ignore [attr-defined]
+        )
+
+    def get_models_with_weights(
+        self, models: dict[tuple[int, int, float], BasePipeline]
+    ) -> list[tuple[float, BasePipeline]]:
+        """Return a list of (weight, model) pairs for the model selected by this ensemble.
+
+        Parameters
+        ----------
+        models : dict {identifier : model object}
+            The identifiers are the same as the one presented to the fit()
+            method. Models can be used for nice printing.
+
+        Returns
+        -------
+        list[tuple[float, BasePipeline]]
+        """
+        return [(self.weights_[0], models[self.identifiers_[0]])]  # type: ignore [attr-defined]  # noqa: E501
+
+    def get_identifiers_with_weights(
+        self,
+    ) -> list[tuple[tuple[int, int, float], float]]:
+        """Return a (identifier, weight)-pairs for the model selected by this ensemble.
+
+        Parameters
+        ----------
+        models : dict {identifier : model object}
+            The identifiers are the same as the one presented to the fit()
+            method. Models can be used for nice printing.
+
+        Returns
+        -------
+        list[tuple[tuple[int, int, float], float]
+        """
+        return list(zip(self.identifiers_, self.weights_))  # type: ignore [attr-defined]  # noqa: E501
+
+    def get_selected_model_identifiers(self) -> list[tuple[int, int, float]]:
+        """Return identifier of models in the ensemble.
+
+        This includes models which have a weight of zero!
+
+        Returns
+        -------
+        list
+        """
+        return self.identifiers_  # type: ignore [attr-defined]
+
+    def get_validation_performance(self) -> float:
+        """Return validation performance of ensemble.
+
+        In case of multi-objective problem, only the first metric will be returned.
+
+        Return
+        ------
+        float
+        """
+        return self.best_model_score_  # type: ignore [attr-defined]
+
+
+class SingleModelEnsemble(AbstractSingleModelEnsemble):
+    """Ensemble consisting of a single model.
+
+    This class is used by the :class:`MultiObjectiveDummyEnsemble` to represent
+    ensembles consisting of a single model, and this class should not be used
+    on its own.
+
+    Do not use by yourself!
+
+    Parameters
+    ----------
+    task_type: int
+        An identifier indicating which task is being performed.
+
+    metrics: Sequence[Scorer] | Scorer
+        The metrics used to evaluate the models.
+
+    backend : Backend
+        Gives access to the backend of Auto-sklearn. Not used.
+
+    model_index : int
+        Index of the model that constitutes the ensemble. This index will
+        be used to select the correct predictions that will be passed during
+        ``fit`` and ``predict``.
+
+    random_state: int | RandomState | None = None
+        Not used.
+    """
+
+    def __init__(
+        self,
+        task_type: int,
+        metrics: Sequence[Scorer] | Scorer,
+        backend: Backend,
+        model_index: int,
+        random_state: int | np.random.RandomState | None = None,
+    ):
+        super().__init__(
+            task_type=task_type,
+            metrics=metrics,
+            random_state=random_state,
+            backend=backend,
+        )
+        self.indices_ = [model_index]
+
+    def fit(
+        self,
+        base_models_predictions: np.ndarray | list[np.ndarray],
+        true_targets: np.ndarray,
+        model_identifiers: list[tuple[int, int, float]],
+        runs: Sequence[Run],
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
+    ) -> SingleModelEnsemble:
+        """Dummy implementation of the ``fit`` method.
+
+        Actualy work of passing the model index is done in the constructor. This
+        method only stores the identifier of the selected model and computes it's
+        validation loss.
+
+        Parameters
+        ----------
+        base_models_predictions: np.ndarray
+            shape = (n_base_models, n_data_points, n_targets)
+            n_targets is the number of classes in case of classification,
+            n_targets is 0 or 1 in case of regression
+
+            Can be a list of 2d numpy arrays as well to prevent copying all
+            predictions into a single, large numpy array.
+
+        true_targets : array of shape [n_targets]
+
+        model_identifiers : identifier for each base model.
+            Can be used for practical text output of the ensemble.
+
+        runs: Sequence[Run]
+            Additional information for each run executed by SMAC that was
+            considered by the ensemble builder. Not used.
+
+        X_data : list-like | spmatrix | None = None
+           X data to feed to a metric if it requires it
+
+        Returns
+        -------
+        self
+        """
+        self.identifiers_ = [model_identifiers[self.indices_[0]]]
+        loss = calculate_losses(
+            solution=true_targets,
+            prediction=base_models_predictions[self.indices_[0]],
+            task_type=self.task_type,
+            metrics=self.metrics,
+            X_data=X_data,
+        )
+        self.best_model_score_ = loss[self.metrics[0].name]
+        return self
+
+
+class SingleBest(AbstractSingleModelEnsemble):
+    """Ensemble consisting of the single best model.
+
+    Parameters
+    ----------
+    task_type: int
+        An identifier indicating which task is being performed.
+
+    metrics: Sequence[Scorer] | Scorer
+        The metrics used to evaluate the models.
+
+    random_state: int | RandomState | None = None
+        Not used.
+
+    backend : Backend
+        Gives access to the backend of Auto-sklearn. Not used.
+    """
+
+    def __init__(
+        self,
+        task_type: int,
+        metrics: Sequence[Scorer] | Scorer,
+        backend: Backend,
+        random_state: int | np.random.RandomState | None = None,
+    ):
+        super().__init__(
+            task_type=task_type,
+            metrics=metrics,
+            random_state=random_state,
+            backend=backend,
+        )
 
     def fit(
         self,
-        base_models_predictions: np.ndarray | List[np.ndarray],
-        X_data: SUPPORTED_FEAT_TYPES,
+        base_models_predictions: np.ndarray | list[np.ndarray],
         true_targets: np.ndarray,
-        model_identifiers: List[Tuple[int, int, float]],
+        model_identifiers: list[tuple[int, int, float]],
         runs: Sequence[Run],
-    ) -> "AbstractEnsemble":
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
+    ) -> SingleBest:
+        """Select the single best model.
+
+        Parameters
+        ----------
+        base_models_predictions: np.ndarray
+            shape = (n_base_models, n_data_points, n_targets)
+            n_targets is the number of classes in case of classification,
+            n_targets is 0 or 1 in case of regression
+
+            Can be a list of 2d numpy arrays as well to prevent copying all
+            predictions into a single, large numpy array.
+
+        true_targets : array of shape [n_targets]
+
+        model_identifiers : identifier for each base model.
+            Can be used for practical text output of the ensemble.
+
+        runs: Sequence[Run]
+            Additional information for each run executed by SMAC that was
+            considered by the ensemble builder. Not used.
+
+        X_data : array-like | sparse matrix | None = None
+
+        Returns
+        -------
+         self
+        """
+        losses = [
+            calculate_losses(
+                solution=true_targets,
+                prediction=base_model_prediction,
+                task_type=self.task_type,
+                metrics=self.metrics,
+                X_data=X_data,
+            )[self.metrics[0].name]
+            for base_model_prediction in base_models_predictions
+        ]
+        argmin = np.argmin(losses)
+        self.indices_ = [argmin]
+        self.identifiers_ = [model_identifiers[argmin]]
+        self.best_model_score_ = losses[argmin]
         return self
 
-    def get_identifiers_from_run_history(self) -> List[Tuple[int, int, float]]:
+
+class SingleBestFromRunhistory(AbstractSingleModelEnsemble):
+    """
+    In the case of a crash, this class searches
+    for the best individual model.
+
+    Such model is returned as an ensemble of a single
+    object, to comply with the expected interface of an
+    AbstractEnsemble.
+
+    Do not use by yourself!
+    """
+
+    def __init__(
+        self,
+        task_type: int,
+        metrics: Sequence[Scorer] | Scorer,
+        backend: Backend,
+        run_history: RunHistory,
+        seed: int,
+        random_state: int | np.random.RandomState | None = None,
+    ):
+        super().__init__(
+            task_type=task_type,
+            metrics=metrics,
+            random_state=random_state,
+            backend=backend,
+        )
+
+        # The seed here is seperate from RandomState and is used to indiicate a
+        # directory for the backend to search in
+        self.seed = seed
+        self.indices_ = [0]
+        self.weights_ = [1.0]
+        self.run_history = run_history
+        self.identifiers_ = self.get_identifiers_from_run_history()
+
+    def get_identifiers_from_run_history(self) -> list[tuple[int, int, float]]:
         """Parses the run history, to identify the best performing model
 
         Populates the identifiers attribute, which is used by the backend to access
@@ -115,55 +440,3 @@ def get_identifiers_from_run_history(self) -> List[Tuple[int, int, float]]:
         self.best_model_score_ = best_model_score
 
         return best_model_identifier
-
-    def predict(self, predictions: Union[np.ndarray, List[np.ndarray]]) -> np.ndarray:
-        return predictions[0]
-
-    def __str__(self) -> str:
-        return (
-            "Single Model Selection:\n\tMembers: %s"
-            "\n\tWeights: %s\n\tIdentifiers: %s"
-            % (
-                self.indices_,
-                self.weights_,
-                " ".join(
-                    [
-                        str(identifier)
-                        for idx, identifier in enumerate(self.identifiers_)
-                        if self.weights_[idx] > 0
-                    ]
-                ),
-            )
-        )
-
-    def get_models_with_weights(
-        self, models: Dict[Tuple[int, int, float], BasePipeline]
-    ) -> List[Tuple[float, BasePipeline]]:
-        output = []
-        for i, weight in enumerate(self.weights_):
-            if weight > 0.0:
-                identifier = self.identifiers_[i]
-                model = models[identifier]
-                output.append((weight, model))
-
-        output.sort(reverse=True, key=lambda t: t[0])
-
-        return output
-
-    def get_identifiers_with_weights(
-        self,
-    ) -> List[Tuple[Tuple[int, int, float], float]]:
-        return list(zip(self.identifiers_, self.weights_))
-
-    def get_selected_model_identifiers(self) -> List[Tuple[int, int, float]]:
-        output = []
-
-        for i, weight in enumerate(self.weights_):
-            identifier = self.identifiers_[i]
-            if weight > 0.0:
-                output.append(identifier)
-
-        return output
-
-    def get_validation_performance(self) -> float:
-        return self.best_model_score_
diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py
index 5afd8c597c..607fb3dc2f 100644
--- a/autosklearn/estimators.py
+++ b/autosklearn/estimators.py
@@ -36,6 +36,9 @@
 )
 from autosklearn.ensembles.abstract_ensemble import AbstractEnsemble
 from autosklearn.ensembles.ensemble_selection import EnsembleSelection
+from autosklearn.ensembles.multiobjective_dummy_ensemble import (
+    MultiObjectiveDummyEnsemble,
+)
 from autosklearn.metrics import Scorer
 from autosklearn.pipeline.base import BasePipeline
 from autosklearn.util.smac_wrap import SMACCallback
@@ -48,7 +51,7 @@ def __init__(
         per_run_time_limit=None,
         initial_configurations_via_metalearning=25,
         ensemble_size: int | None = None,
-        ensemble_class: Type[AbstractEnsemble] | None = EnsembleSelection,
+        ensemble_class: Type[AbstractEnsemble] | Literal["default"] | None = "default",
         ensemble_kwargs: Dict[str, Any] | None = None,
         ensemble_nbest=50,
         max_models_on_disc=50,
@@ -104,12 +107,16 @@ def __init__(
             this argument via ``ensemble_kwargs={"ensemble_size": int}``
             if you want to change the ensemble size for ensemble selection.
 
-        ensemble_class : Type[AbstractEnsemble], optional (default=EnsembleSelection)
+        ensemble_class : Type[AbstractEnsemble] | "default", optional (default="default")
             Class implementing the post-hoc ensemble algorithm. Set to
-            ``None`` to disable ensemble building or use ``SingleBest``
+            ``None`` to disable ensemble building or use :class:`SingleBest`
             to obtain only use the single best model instead of an
             ensemble.
 
+            If set to "default" it will use :class:`EnsembleSelection` for
+            single-objective problems and :class:`MultiObjectiveDummyEnsemble`
+            for multi-objective problems.
+
         ensemble_kwargs : Dict, optional
             Keyword arguments that are passed to the ensemble class upon
             initialization.
@@ -396,6 +403,11 @@ def __init__(
         self.initial_configurations_via_metalearning = (
             initial_configurations_via_metalearning
         )
+
+        # Need to resolve the ensemble class here so we can act on it below.
+        if ensemble_class == "default":
+            ensemble_class = self._resolve_ensemble_class(metric)
+
         self.ensemble_class = ensemble_class
 
         # User specified `ensemble_size` explicitly, warn them about deprecation
@@ -610,8 +622,8 @@ def fit_ensemble(
         ensemble_size: int | None = None,
         ensemble_kwargs: Optional[Dict[str, Any]] = None,
         ensemble_nbest: Optional[int] = None,
-        ensemble_class: Optional[AbstractEnsemble] = EnsembleSelection,
-        metrics: Scorer | Sequence[Scorer] | None = None,
+        ensemble_class: Type[AbstractEnsemble] | Literal["default"] | None = "default",
+        metric: Scorer | Sequence[Scorer] | None = None,
     ):
         """Fit an ensemble to models trained during an optimization process.
 
@@ -655,19 +667,39 @@ def fit_ensemble(
             is independent of the ``ensemble_class`` argument and this
             pruning step is done prior to constructing an ensemble.
 
-        ensemble_class : Type[AbstractEnsemble], optional (default=EnsembleSelection)
+        ensemble_class : Type[AbstractEnsemble] | "default", optional (default="default")
             Class implementing the post-hoc ensemble algorithm. Set to
-            ``None`` to disable ensemble building or use ``SingleBest``
+            ``None`` to disable ensemble building or use class:`SingleBest`
             to obtain only use the single best model instead of an
             ensemble.
 
-        metrics: Scorer | Sequence[Scorer] | None = None
+            If set to "default" it will use :class:`EnsembleSelection` for
+            single-objective problems and :class:`MultiObjectiveDummyEnsemble`
+            for multi-objective problems.
+
+        metric: Scorer | Sequence[Scorer] | None = None
             A metric or list of metrics to score the ensemble with
 
         Returns
         -------
         self
-        """
+        """  # noqa: E501
+
+        if ensemble_class == "default":
+            # Things are actually a little more nuanced here:
+            # * If they passed `metric=None` at init, we would infer this in automl
+            #   during `fit` and store it in the automl instance.
+            # * If they passed a `metric` in init and left it `None` here, this would
+            #   also be in the automl instance
+            # => We can use self.automl_ as ground truth for metric if no metrics passed
+            #   and we have one created
+            if metric is None and self.automl_ is not None and self.automl_._metrics:
+                metric = self.automl_._metrics
+
+            ensemble_class = self._resolve_ensemble_class(metric)
+
+        self.ensemble_class = ensemble_class
+
         # User specified `ensemble_size` explicitly, warn them about deprecation
         if ensemble_size is not None:
             # Keep consistent behaviour
@@ -720,10 +752,20 @@ def fit_ensemble(
             ensemble_nbest=ensemble_nbest,
             ensemble_class=ensemble_class,
             ensemble_kwargs=ensemble_kwargs,
-            metrics=metrics,
+            metrics=metric,
         )
         return self
 
+    def _resolve_ensemble_class(
+        self,
+        metric: Scorer | Sequence[Scorer] | None,
+    ) -> type[AbstractEnsemble]:
+        return (
+            EnsembleSelection
+            if metric is None or isinstance(metric, Scorer) or len(metric) == 1
+            else MultiObjectiveDummyEnsemble
+        )
+
     def refit(self, X, y):
         """Refit all models found with fit to new data.
 
diff --git a/autosklearn/metrics/__init__.py b/autosklearn/metrics/__init__.py
index 0485db892c..e6b2e87802 100644
--- a/autosklearn/metrics/__init__.py
+++ b/autosklearn/metrics/__init__.py
@@ -48,6 +48,7 @@ def __call__(
         self,
         y_true: np.ndarray,
         y_pred: np.ndarray,
+        *,
         X_data: Optional[SUPPORTED_XDATA_TYPES] = None,
         sample_weight: Optional[List[float]] = None,
     ) -> float:
@@ -62,6 +63,7 @@ def __call__(
         self,
         y_true: np.ndarray,
         y_pred: np.ndarray,
+        *,
         X_data: Optional[SUPPORTED_XDATA_TYPES] = None,
         sample_weight: Optional[List[float]] = None,
     ) -> float:
@@ -129,6 +131,7 @@ def __call__(
         self,
         y_true: np.ndarray,
         y_pred: np.ndarray,
+        *,
         X_data: Optional[SUPPORTED_XDATA_TYPES] = None,
         sample_weight: Optional[List[float]] = None,
     ) -> float:
@@ -189,6 +192,7 @@ def __call__(
         self,
         y_true: np.ndarray,
         y_pred: np.ndarray,
+        *,
         X_data: Optional[SUPPORTED_XDATA_TYPES] = None,
         sample_weight: Optional[List[float]] = None,
     ) -> float:
@@ -508,11 +512,11 @@ def calculate_scores(
 
             try:
                 score_dict[metric_.name] = _compute_single_scorer(
-                    metric_,
-                    prediction,
-                    solution,
-                    task_type,
-                    X_data,
+                    metric=metric_,
+                    prediction=prediction,
+                    solution=solution,
+                    task_type=task_type,
+                    X_data=X_data,
                 )
             except ValueError as e:
                 print(e, e.args[0])
@@ -532,11 +536,11 @@ def calculate_scores(
 
             try:
                 score_dict[metric_.name] = _compute_single_scorer(
-                    metric_,
-                    prediction,
-                    solution,
-                    task_type,
-                    X_data,
+                    metric=metric_,
+                    prediction=prediction,
+                    solution=solution,
+                    task_type=task_type,
+                    X_data=X_data,
                 )
             except ValueError as e:
                 if e.args[0] == "multiclass format is not supported":
diff --git a/autosklearn/util/multiobjective.py b/autosklearn/util/multiobjective.py
new file mode 100644
index 0000000000..f12b111153
--- /dev/null
+++ b/autosklearn/util/multiobjective.py
@@ -0,0 +1,40 @@
+import operator
+
+import numpy as np
+
+
+def pareto_front(values: np.ndarray, *, is_loss: bool = True) -> np.ndarray:
+    """Calculate the pareto front
+
+    source from: https://stackoverflow.com/a/40239615
+
+    Note
+    ----
+    Works on the assumption that every value is either something to minimize or
+    something to maximize, based on ``is_loss``.
+
+    Parameters
+    ----------
+    values: np.ndarray [n_models, n_metrics]
+        The value for each of the metrics
+
+    is_loss: bool = True
+        Whether the metrics are a loss or a score
+
+    Returns
+    -------
+    np.ndarray
+        A boolean mask where true indicates if the model on the pareto front
+    """
+    op = operator.lt if is_loss else operator.gt
+
+    is_efficient = np.ones(values.shape[0], dtype=bool)
+    for i, c in enumerate(values):
+        if is_efficient[i]:
+            # Keep any point with a lower cost
+            is_efficient[is_efficient] = np.any(op(values[is_efficient], c), axis=1)
+
+            # And keep self
+            is_efficient[i] = True
+
+    return is_efficient
diff --git a/doc/api.rst b/doc/api.rst
index 789d2d24e3..581c74d98d 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -117,14 +117,30 @@ Extension Interfaces
 Ensembles
 =========
 
-.. autoclass:: autosklearn.ensembles.AbstractEnsemble
+~~~~~~~~~~~~~~~~
+Single objective
+~~~~~~~~~~~~~~~~
+
+.. autoclass:: autosklearn.ensembles.EnsembleSelection
+    :members:
+
+Single model classes
+~~~~~~~~~~~~~~~~~~~~
+
+These classes wrap a single model to provide a unified interface in Auto-sklearn.
+
+.. autoclass:: autosklearn.ensembles.SingleBest
     :members:
 
-.. autoclass:: autosklearn.ensembles.AbstractMultiObjectiveEnsemble
+.. autoclass:: autosklearn.ensembles.SingleModelEnsemble
     :members:
 
-.. autoclass:: autosklearn.ensembles.EnsembleSelection
+.. autoclass:: autosklearn.ensembles.SingleBestFromRunhistory
     :members:
 
-.. autoclass:: autosklearn.ensembles.SingleBest
+~~~~~~~~~~~~~~~
+Multi-objective
+~~~~~~~~~~~~~~~
+
+.. autoclass:: autosklearn.ensembles.MultiObjectiveDummyEnsemble
     :members:
diff --git a/examples/60_search/example_parallel_manual_spawning_cli.py b/examples/60_search/example_parallel_manual_spawning_cli.py
index 8a1a9d5a7c..7df0907527 100644
--- a/examples/60_search/example_parallel_manual_spawning_cli.py
+++ b/examples/60_search/example_parallel_manual_spawning_cli.py
@@ -193,7 +193,7 @@ def cli_start_worker(scheduler_file_name):
         delete_tmp_folder_after_terminate=False,
         time_left_for_this_task=30,
         per_run_time_limit=10,
-        memory_limit=1024,
+        memory_limit=2048,
         tmp_folder=tmp_folder,
         seed=777,
         # n_jobs is ignored internally as we pass a dask client.
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testEntireEnsembleBuilder/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testFallBackNBest/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testGetValidTestPreds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testLimit/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc2/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1499.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[1500.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[4-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[9999.0-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testMaxModelsOnDisc[None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[0.9-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-2-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1-None-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[1.0-None-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testNBest[2-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.0-1.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.1-2-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.3-4-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.5-1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.6-10-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[0.8-0.5-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThresholdMaxBest[1-1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.0-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.1-4]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.3-3]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.5-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.6-2]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[0.8-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testPerformanceRangeThreshold[1.0-1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_testRead/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_nbest_remembered/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_ensemble_builder_process_realrun/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric0]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_get_identifiers_from_run_history[metric1]/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_main/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_1_0.0/0.1.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/0.2.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np
deleted file mode 100644
index fee3160c86..0000000000
Binary files a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_2_0.0/predictions_test_0_2_0.0.np and /dev/null differ
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.0.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model b/test/test_ensemble_builder/test.test_ensemble_builder.test_ensemble_test_read_pickle_read_preds/.auto-sklearn/runs/0_3_100.0/0.3.100.0.model
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/test/test_ensemble_builder/test_multiobjective_dummy_ensemble.py b/test/test_ensemble_builder/test_multiobjective_dummy_ensemble.py
new file mode 100644
index 0000000000..eb639236fd
--- /dev/null
+++ b/test/test_ensemble_builder/test_multiobjective_dummy_ensemble.py
@@ -0,0 +1,73 @@
+import numpy as np
+import sklearn.metrics
+
+from autosklearn.constants import REGRESSION
+from autosklearn.ensembles.multiobjective_dummy_ensemble import (
+    MultiObjectiveDummyEnsemble,
+)
+from autosklearn.ensembles.singlebest_ensemble import SingleModelEnsemble
+from autosklearn.metrics import MAXINT, make_scorer, root_mean_squared_error
+
+
+def test_MultiObjectiveDummyEnsemble(backend):
+    negative_root_mean_squared_error = make_scorer(
+        "negative_root_mean_squared_error",
+        sklearn.metrics.mean_squared_error,
+        optimum=0,
+        worst_possible_result=-MAXINT,
+        greater_is_better=True,
+        squared=False,
+    )
+
+    ensemble = MultiObjectiveDummyEnsemble(
+        task_type=REGRESSION,
+        metrics=[root_mean_squared_error, negative_root_mean_squared_error],
+        random_state=0,
+        backend=backend,
+    )
+
+    # fit the whole thing
+    X_data = np.random.random(size=(100, 2))
+    y_true = np.full((100), 5.5)
+    predictions = []
+    for i in range(20, 1, -1):
+        pred = np.ones((100), dtype=np.float32) * i
+        predictions.append(pred)
+    identifiers = [(i, i, 0.0) for i in range(20)]
+
+    ensemble.fit(
+        base_models_predictions=predictions,
+        X_data=X_data,
+        true_targets=y_true,
+        model_identifiers=identifiers,
+        runs=[],
+    )
+
+    # Because the target value is 5.5 some of the predictions have the same error
+    # -> not every prediction is on the Pareto front
+    assert len(ensemble.pareto_set_) == 15
+    for sub_ensemble in ensemble.pareto_set_:
+        assert isinstance(sub_ensemble, SingleModelEnsemble)
+
+    # test the prediction
+    ensemble_prediction = np.random.random((1, 100, 2))
+    prediction = ensemble.predict(ensemble_prediction)
+    assert prediction.shape == (100, 2)
+
+    assert str(ensemble) == """MultiObjectiveDummyEnsemble: 15 models"""
+
+    # From here on everything is supposed to return the output of the best
+    # ensemble according to the 1st metric
+    models_with_weights = ensemble.get_models_with_weights(
+        {identifier: i for i, identifier in enumerate(identifiers)}
+    )
+    assert models_with_weights == [(1.0, 14)]
+
+    identifiers_with_weights = ensemble.get_identifiers_with_weights()
+    assert identifiers_with_weights == [((14, 14, 0.0), 1.0)]
+
+    selected_model_identifiers = ensemble.get_selected_model_identifiers()
+    assert selected_model_identifiers == [(14, 14, 0.0)]
+
+    best_model_score = ensemble.get_validation_performance()
+    assert best_model_score == 0.5
diff --git a/test/test_ensemble_builder/test_singlebest_ensemble.py b/test/test_ensemble_builder/test_singlebest_ensemble.py
new file mode 100644
index 0000000000..915f4bc6cf
--- /dev/null
+++ b/test/test_ensemble_builder/test_singlebest_ensemble.py
@@ -0,0 +1,132 @@
+import numpy as np
+
+from autosklearn.constants import REGRESSION
+from autosklearn.ensembles.singlebest_ensemble import SingleBest, SingleModelEnsemble
+from autosklearn.metrics import root_mean_squared_error
+
+
+def test_SingleBest(backend):
+
+    ensemble = SingleBest(
+        task_type=REGRESSION,
+        metrics=[root_mean_squared_error],
+        random_state=0,
+        backend=backend,
+    )
+
+    # fit the whole thing
+    X_data = np.random.random(size=(100, 2))
+    y_true = np.full((100), 5.5)
+    predictions = []
+    for i in range(20, 1, -1):
+        pred = np.ones((100), dtype=np.float32) * i
+        predictions.append(pred)
+    identifiers = [(i, i, 0.0) for i in range(20)]
+
+    # Check that the weight isn't touched
+    assert ensemble.weights_ == [1.0]
+
+    ensemble.fit(
+        base_models_predictions=predictions,
+        X_data=X_data,
+        true_targets=y_true,
+        model_identifiers=identifiers,
+        runs=[],
+    )
+
+    assert ensemble.weights_ == [1.0]
+    assert ensemble.indices_ == [14]
+    assert ensemble.identifiers_ == [(14, 14, 0.0)]
+    assert ensemble.best_model_score_ == 0.5
+
+    # test the prediction
+    ensemble_prediction = np.random.random((1, 100, 2))
+    prediction = ensemble.predict(ensemble_prediction)
+    assert prediction.shape == (100, 2)
+
+    # test string representation; it selects the 14th prediction (6)
+    # and ties with the 15th (5), but then afterwards the RMSE goes up again
+    assert (
+        str(ensemble)
+        == """SingleBest:
+\tMembers: [14]
+\tWeights: [1.0]
+\tIdentifiers: [(14, 14, 0.0)]"""
+    )
+
+    models_with_weights = ensemble.get_models_with_weights(
+        {identifier: i for i, identifier in enumerate(identifiers)}
+    )
+    assert models_with_weights == [(1.0, 14)]
+
+    identifiers_with_weights = ensemble.get_identifiers_with_weights()
+    assert identifiers_with_weights == [((14, 14, 0.0), 1.0)]
+
+    selected_model_identifiers = ensemble.get_selected_model_identifiers()
+    assert selected_model_identifiers == [(14, 14, 0.0)]
+
+    best_model_score = ensemble.get_validation_performance()
+    assert best_model_score == 0.5
+
+
+def test_SingleModelEnsemble(backend):
+
+    ensemble = SingleModelEnsemble(
+        task_type=REGRESSION,
+        metrics=[root_mean_squared_error],
+        random_state=0,
+        backend=backend,
+        model_index=5,
+    )
+
+    # fit the whole thing
+    X_data = np.random.random(size=(100, 2))
+    y_true = np.full((100), 5.5)
+    predictions = []
+    for i in range(20, 1, -1):
+        pred = np.ones((100), dtype=np.float32) * i
+        predictions.append(pred)
+    identifiers = [(i, i, 0.0) for i in range(20)]
+
+    # Check that the weight isn't touched
+    assert ensemble.weights_ == [1.0]
+
+    ensemble.fit(
+        base_models_predictions=predictions,
+        X_data=X_data,
+        true_targets=y_true,
+        model_identifiers=identifiers,
+        runs=[],
+    )
+
+    assert ensemble.weights_ == [1.0]
+    assert ensemble.indices_ == [5]
+    assert ensemble.identifiers_ == [(5, 5, 0.0)]
+    assert ensemble.best_model_score_ == 9.5
+
+    # test the prediction
+    ensemble_prediction = np.random.random((1, 100, 2))
+    prediction = ensemble.predict(ensemble_prediction)
+    assert prediction.shape == (100, 2)
+
+    assert (
+        str(ensemble)
+        == """SingleModelEnsemble:
+\tMembers: [5]
+\tWeights: [1.0]
+\tIdentifiers: [(5, 5, 0.0)]"""
+    )
+
+    models_with_weights = ensemble.get_models_with_weights(
+        {identifier: i for i, identifier in enumerate(identifiers)}
+    )
+    assert models_with_weights == [(1.0, 5)]
+
+    identifiers_with_weights = ensemble.get_identifiers_with_weights()
+    assert identifiers_with_weights == [((5, 5, 0.0), 1.0)]
+
+    selected_model_identifiers = ensemble.get_selected_model_identifiers()
+    assert selected_model_identifiers == [(5, 5, 0.0)]
+
+    best_model_score = ensemble.get_validation_performance()
+    assert best_model_score == 9.5
diff --git a/test/test_estimators/test_estimators.py b/test/test_estimators/test_estimators.py
index d0d3f28bdb..7eb3a441f0 100644
--- a/test/test_estimators/test_estimators.py
+++ b/test/test_estimators/test_estimators.py
@@ -29,6 +29,7 @@
 from autosklearn.automl import AutoMLClassifier
 from autosklearn.data.validation import InputValidator
 from autosklearn.ensemble_building.run import Run
+from autosklearn.ensembles import EnsembleSelection, MultiObjectiveDummyEnsemble
 from autosklearn.estimators import (
     AutoSklearnClassifier,
     AutoSklearnEstimator,
@@ -1729,3 +1730,16 @@ def test_param_dataset_compression(dataset_compression: Union[bool, Dict[str, An
     model = AutoSklearnClassifier(dataset_compression=dataset_compression)
 
     assert model.dataset_compression == dataset_compression
+
+
+def test_ensemble_default_resolves():
+    model = AutoSklearnClassifier(
+        ensemble_class="default",
+        metric=accuracy,
+    )
+    assert model.ensemble_class == EnsembleSelection
+    model = AutoSklearnClassifier(
+        ensemble_class="default",
+        metric=[accuracy, f1_macro],
+    )
+    assert model.ensemble_class == MultiObjectiveDummyEnsemble