Remove parameter metadata

src/ert/config/__init__.py

@@ -31,7 +31,7 @@
 from .known_response_types import KnownResponseTypes
 from .lint_file import lint_file
 from .model_config import ModelConfig
-from .parameter_config import ParameterCardinality, ParameterConfig, ParameterMetadata
+from .parameter_config import ParameterCardinality, ParameterConfig
 from .parsing import (
     ConfigValidationError,
     ConfigWarning,
@@ -119,7 +119,6 @@
     "OutlierSettings",
     "ParameterCardinality",
     "ParameterConfig",
-    "ParameterMetadata",
     "PostExperimentFixtures",
     "PostSimulationFixtures",
     "PostUpdateFixtures",

src/ert/config/ext_param_config.py

@@ -17,7 +17,7 @@
 
 from ert.substitutions import substitute_runpath_name
 
-from .parameter_config import ParameterConfig, ParameterMetadata
+from .parameter_config import ParameterConfig
 
 if TYPE_CHECKING:
     import numpy.typing as npt
@@ -154,10 +154,6 @@ class ExtParamConfig(ParameterConfig):
     def parameter_keys(self) -> list[str]:
         return self.input_keys
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return []
-
     type: Literal["everest_parameters"] = "everest_parameters"
     input_keys: list[str] = field(default_factory=list)
     forward_init: bool = False

src/ert/config/field.py

@@ -29,7 +29,7 @@
 from ert.utils import log_duration
 
 from ._str_to_bool import str_to_bool
-from .parameter_config import ParameterConfig, ParameterMetadata
+from .parameter_config import ParameterConfig
 from .parsing import ConfigValidationError, ConfigWarning
 
 if TYPE_CHECKING:
@@ -70,6 +70,7 @@ def create_flattened_cube_graph(px: int, py: int, pz: int) -> nx.Graph[int]:
 
 class Field(ParameterConfig):
     type: Literal["field"] = "field"
+    dimensionality: Literal[3] = 3
     ertbox_params: ErtboxParameters
     file_format: FieldFileFormat
     output_transformation: str | None
@@ -88,17 +89,6 @@ def serialize_output_file(self, path: Path) -> str:
     def parameter_keys(self) -> list[str]:
         return []
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=self.name,
-                transformation=self.output_transformation,
-                dimensionality=3,
-                userdata={"data_origin": "FIELD", "ertbox_params": self.ertbox_params},
-            )
-        ]
-
     @classmethod
     def from_config_list(
         cls,

src/ert/config/gen_kw_config.py

@@ -15,7 +15,7 @@
 
 from ._str_to_bool import str_to_bool
 from .distribution import DISTRIBUTION_CLASSES, DistributionSettings, get_distribution
-from .parameter_config import ParameterCardinality, ParameterConfig, ParameterMetadata
+from .parameter_config import ParameterCardinality, ParameterConfig
 from .parsing import ConfigValidationError, ConfigWarning
 
 if TYPE_CHECKING:
@@ -53,6 +53,7 @@ class DataSource(StrEnum):
 
 class GenKwConfig(ParameterConfig):
     type: Literal["gen_kw"] = "gen_kw"
+    dimensionality: Literal[1] = 1
     distribution: DistributionSettings
     forward_init: bool = False
     update: bool = True
@@ -73,17 +74,6 @@ def parameter_keys(self) -> list[str]:
     def cardinality(self) -> ParameterCardinality:
         return ParameterCardinality.multiple_configs_per_ensemble_dataset
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=f"{self.group}:{self.name}",
-                transformation=self.distribution.name.upper(),
-                dimensionality=1,
-                userdata={"data_origin": "GEN_KW"},
-            )
-        ]
-
     @classmethod
     def templates_from_config(
         cls, gen_kw: list[str | dict[str, str]]

src/ert/config/parameter_config.py

@@ -5,7 +5,7 @@
 from enum import StrEnum, auto
 from hashlib import sha256
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING
 
 import networkx as nx
 import numpy as np
@@ -39,13 +39,6 @@ class ParameterCardinality(StrEnum):
     one_config_per_realization_dataset = auto()
 
 
-class ParameterMetadata(BaseModel):
-    key: str
-    transformation: str | None
-    dimensionality: Literal[1, 2, 3] = 1
-    userdata: dict[str, Any]
-
-
 class ParameterConfig(BaseModel):
     type: str
     name: str
@@ -59,14 +52,6 @@ def parameter_keys(self) -> list[str]:
         Returns a list of parameter keys within this parameter group
         """
 
-    @property
-    @abstractmethod
-    def metadata(self) -> list[ParameterMetadata]:
-        """
-        Returns metadata describing this parameter
-
-        """
-
     @abstractmethod
     def __len__(self) -> int:
         """Number of parameters"""

src/ert/config/surface_config.py

@@ -19,7 +19,7 @@
 
 from ._str_to_bool import str_to_bool
 from .field import create_flattened_cube_graph
-from .parameter_config import InvalidParameterFile, ParameterConfig, ParameterMetadata
+from .parameter_config import InvalidParameterFile, ParameterConfig
 from .parsing import ConfigValidationError, ErrorInfo
 
 if TYPE_CHECKING:
@@ -51,6 +51,7 @@ def __init__(
 
 class SurfaceConfig(ParameterConfig):
     type: Literal["surface"] = "surface"
+    dimensionality: Literal[2] = 2
     ncol: int
     nrow: int
     xori: float
@@ -75,21 +76,6 @@ def serialize_base_surface_path(self, base_surface_path: Path) -> str:
     def parameter_keys(self) -> list[str]:
         return []
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=self.name,
-                dimensionality=2,
-                transformation=None,
-                userdata={
-                    "data_origin": "SURFACE",
-                    "nx": self.ncol,
-                    "ny": self.nrow,
-                },
-            )
-        ]
-
     @classmethod
     def from_config_list(cls, config_list: list[str | dict[str, str]]) -> Self:
         name = cast(str, config_list[0])

src/ert/dark_storage/endpoints/experiments.py

@@ -29,7 +29,7 @@ def get_experiments(
             priors=create_priors(experiment),
             userdata={},
             parameters={
-                group: [m.model_dump() for m in config.metadata]
+                group: [config.model_dump()]
                 for group, config in experiment.parameter_configuration.items()
                 if not isinstance(config, SurfaceConfig)
             },
@@ -65,7 +65,7 @@ def get_experiment_by_id(
         priors=create_priors(experiment),
         userdata={},
         parameters={
-            group: [m.model_dump() for m in config.metadata]
+            group: [config.model_dump()]
             for group, config in experiment.parameter_configuration.items()
         },
         responses={

src/ert/dark_storage/endpoints/parameters.py

@@ -113,27 +113,11 @@ def get_parameter_std_dev(
     return Response(content=buffer.getvalue(), media_type="application/octet-stream")
 
 
-def _extract_parameter_group_and_key(key: str) -> tuple[str, str] | tuple[None, None]:
-    key = key.removeprefix("LOG10_")
-    if ":" not in key:
-        # Assume all incoming keys are in format group:key for now
-        return None, None
-
-    param_group, param_key = key.split(":", maxsplit=1)
-    return param_group, param_key
-
-
 def data_for_parameter(ensemble: Ensemble, key: str) -> pd.DataFrame:
-    group, _ = _extract_parameter_group_and_key(key)
-    if group is None:
-        logger.warning(
-            f"Parameter with key '{key}' does not have a group, "
-            "fetching data for all parameters"
-        )
     try:
-        df = ensemble.load_scalars(group)
+        df = ensemble.load_scalar_keys([key], transformed=True)
         if df.is_empty():
-            logger.warning(f"No data found for parameter '{group}:{key}'")
+            logger.warning(f"No data found for parameter '{key}'")
             return pd.DataFrame()
     except KeyError as e:
         logger.error(e)

src/ert/gui/tools/plot/plot_api.py

@@ -18,8 +18,9 @@
 from pandas.errors import ParserError
 from resfo_utilities import history_key
 
-from ert.config import ParameterMetadata, ResponseMetadata
+from ert.config import ParameterConfig, ResponseMetadata
 from ert.services import ErtServer
+from ert.storage.local_experiment import _parameters_adapter as parameter_config_adapter
 from ert.storage.realization_storage_state import RealizationStorageState
 
 logger = logging.getLogger(__name__)
@@ -44,7 +45,7 @@ class PlotApiKeyDefinition(NamedTuple):
     dimensionality: int
     metadata: dict[Any, Any]
     filter_on: dict[Any, Any] | None = None
-    parameter_metadata: ParameterMetadata | None = None
+    parameter: ParameterConfig | None = None
     response_metadata: ResponseMetadata | None = None
 
 
@@ -145,14 +146,18 @@ def parameters_api_key_defs(self) -> list[PlotApiKeyDefinition]:
             for experiment in response.json():
                 for param_metadatas in experiment["parameters"].values():
                     for metadata in param_metadatas:
-                        param_key = metadata["key"]
+                        param_cfg = parameter_config_adapter.validate_python(metadata)
+                        if group := metadata.get("group"):
+                            param_key = f"{group}:{metadata['name']}"
+                        else:
+                            param_key = metadata["name"]
                         all_keys[param_key] = PlotApiKeyDefinition(
                             key=param_key,
                             index_type=None,
                             observations=False,
                             dimensionality=metadata["dimensionality"],
-                            metadata=metadata["userdata"],
-                            parameter_metadata=ParameterMetadata(**metadata),
+                            metadata={"data_origin": metadata["type"]},
+                            parameter=param_cfg,
                         )
                         all_params[param_key] = all_keys[param_key]
 

src/ert/gui/tools/plot/plot_window.py

@@ -22,6 +22,7 @@
     QWidget,
 )
 
+from ert.config.field import Field
 from ert.dark_storage.common import get_storage_api_version
 from ert.gui.ertwidgets import CopyButton, showWaitCursorWhileWaiting
 from ert.services._base_service import ServerBootFail
@@ -270,12 +271,12 @@ def updatePlot(self, layer: int | None = None) -> None:
                             filter_on=key_def.filter_on,
                         )
                     elif (
-                        key_def.parameter_metadata is not None
-                        and "GEN_KW" in key_def.metadata["data_origin"]
+                        key_def.parameter is not None
+                        and key_def.parameter.type == "gen_kw"
                     ):
                         ensemble_to_data_map[ensemble] = self._api.data_for_parameter(
                             ensemble_id=ensemble.id,
-                            parameter_key=key_def.parameter_metadata.key,
+                            parameter_key=key_def.parameter.name,
                         )
                 except BaseException as e:
                     handle_exception(e)
@@ -290,10 +291,10 @@ def updatePlot(self, layer: int | None = None) -> None:
                     handle_exception(e)
 
             std_dev_images: dict[str, npt.NDArray[np.float32]] = {}
-            if "FIELD" in key_def.metadata["data_origin"]:
-                plot_widget.showLayerWidget.emit(True)
 
-                layers = key_def.metadata["ertbox_params"]["nz"]
+            if isinstance(key_def.parameter, Field):
+                plot_widget.showLayerWidget.emit(True)
+                layers = key_def.parameter.ertbox_params.nz
                 plot_widget.updateLayerWidget.emit(layers)
 
                 if layer is None:

src/ert/storage/local_ensemble.py

@@ -570,6 +570,9 @@ def load_scalar_keys(
     ) -> pl.DataFrame:
         if keys is None:
             keys = self.experiment.parameter_keys
+        elif set(keys) - set(self.experiment.parameter_keys):
+            missing = set(keys) - set(self.experiment.parameter_keys)
+            raise KeyError(f"Parameters not registered to the experiment: {missing}")
 
         df_lazy = self._load_parameters_lazy(SCALAR_FILENAME)
         df_lazy = df_lazy.select(["realization", *keys])

src/ert/storage/local_storage.py

@@ -504,6 +504,7 @@ def _migrate(self, version: int) -> None:
             to16,
             to17,
             to18,
+            to19,
         )
 
         try:
@@ -551,6 +552,7 @@ def _migrate(self, version: int) -> None:
                     15: to16,
                     16: to17,
                     17: to18,
+                    18: to19,
                 }
                 for from_version in range(version, _LOCAL_STORAGE_VERSION):
                     migrations[from_version].migrate(self.path)

src/ert/storage/migration/to19.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Add dimensionality attribute to parameters"
+
+
+def migrate_param(parameters_json: dict[str, Any]) -> dict[str, Any]:
+    new_configs = {}
+    for param_config in parameters_json.values():
+        if param_config["type"] == "surface":
+            param_config["dimensionality"] = 2
+        elif param_config["type"] == "field":
+            param_config["dimensionality"] = 3
+        else:
+            param_config["dimensionality"] = 1
+
+        new_configs[param_config["name"]] = param_config
+    return new_configs
+
+
+def migrate_parameters_for_experiment(experiment: Path) -> None:
+    with open(experiment / "parameter.json", encoding="utf-8") as fin:
+        parameters_json = json.load(fin)
+
+    new_parameter_configs = migrate_param(parameters_json)
+    Path(experiment / "parameter.json").write_text(
+        json.dumps(new_parameter_configs, indent=2), encoding="utf-8"
+    )
+
+
+def migrate(path: Path) -> None:
+    for experiment in path.glob("experiments/*"):
+        migrate_parameters_for_experiment(experiment)

tests/ert/performance_tests/test_dark_storage_performance.py

@@ -335,7 +335,7 @@ def run():
         for key_info in key_infos_params:
             for ensemble in all_ensembles:
                 api.data_for_parameter(
-                    ensemble_id=ensemble.id, parameter_key=key_info.key
+                    ensemble_id=ensemble.id, parameter_key=key_info.parameter.name
                 )
 
         for key_info in key_infos_responses: