remove observations from ErtConfig

Author: eqbech

src/ert/config/ert_config.py

@@ -12,14 +12,13 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Self, cast, overload
 
-import polars as pl
 from numpy.random import SeedSequence
-from pydantic import BaseModel, Field, PrivateAttr, model_validator
+from pydantic import BaseModel, Field, model_validator
 from pydantic import ValidationError as PydanticValidationError
 
+from ert.config._create_observation_dataframes import create_observation_dataframes
 from ert.substitutions import Substitutions
 
-from ._create_observation_dataframes import create_observation_dataframes
 from ._design_matrix_validator import DesignMatrixValidator
 from ._observations import (
     GeneralObservation,
@@ -714,32 +713,6 @@ class ErtConfig(BaseModel):
     config_path: str = Field(init=False, default="")
     observation_declarations: list[Observation] = Field(default_factory=list)
     zonemap: dict[int, list[str]] = Field(default_factory=dict)
-    _observations: dict[str, pl.DataFrame] | None = PrivateAttr(None)
-
-    @property
-    def observations(self) -> dict[str, pl.DataFrame]:
-        if self._observations is None:
-            has_rft_observations = any(
-                isinstance(o, RFTObservation) for o in self.observation_declarations
-            )
-            if (
-                has_rft_observations
-                and "rft" not in self.ensemble_config.response_configs
-            ):
-                self.ensemble_config.response_configs["rft"] = RFTConfig(
-                    input_files=[self.runpath_config.eclbase_format_string],
-                    data_to_read={},
-                    locations=[],
-                    zonemap=self.zonemap,
-                )
-            self._observations = create_observation_dataframes(
-                self.observation_declarations,
-                cast(
-                    RFTConfig | None,
-                    self.ensemble_config.response_configs.get("rft", None),
-                ),
-            )
-        return self._observations
 
     @model_validator(mode="after")
     def set_fields(self) -> Self:
@@ -834,28 +807,6 @@ def validate_observations_against_responses(self) -> Self:
 
         return self
 
-    def __eq__(self, other: object) -> bool:
-        if not isinstance(other, ErtConfig):
-            return False
-
-        for attr in vars(self):
-            if attr == "observations":
-                if self.observations.keys() != other.observations.keys():
-                    return False
-
-                if not all(
-                    self.observations[k].equals(other.observations[k])
-                    for k in self.observations
-                ):
-                    return False
-
-                continue
-
-            if getattr(self, attr) != getattr(other, attr):
-                return False
-
-        return True
-
     @staticmethod
     def with_plugins(runtime_plugins: ErtRuntimePlugins) -> type[ErtConfig]:
         class ErtConfigWithPlugins(ErtConfig):
@@ -1157,12 +1108,10 @@ def from_dict(cls, config_dict: ConfigDict) -> Self:
 
             # PS:
             # This mutates the rft config and is necessary for the moment
-            cls_config._observations = create_observation_dataframes(
+            # Consider changing this pattern
+            _ = create_observation_dataframes(
                 obs_configs,
-                cast(
-                    RFTConfig | None,
-                    ensemble_config.response_configs.get("rft", None),
-                ),
+                cast(RFTConfig | None, ensemble_config.response_configs.get("rft")),
             )
         except PydanticValidationError as err:
             raise ConfigValidationError.from_pydantic(err) from err

tests/ert/unit_tests/config/test_observations.py

@@ -3,6 +3,7 @@
 from datetime import datetime, timedelta
 from pathlib import Path
 from textwrap import dedent
+from typing import cast
 
 import hypothesis.strategies as st
 import polars as pl
@@ -25,6 +26,7 @@
     ObservationConfigError,
     ObservationType,
 )
+from ert.config.rft_config import RFTConfig
 from ert.namespace import Namespace
 
 pytestmark = pytest.mark.filterwarnings("ignore:Config contains a SUMMARY key")
@@ -103,7 +105,7 @@ def make_refcase_observations(
     run_convert_observations(Namespace(config="config.ert"))
 
     migrated_config = ErtConfig.from_file("config.ert")
-    return migrated_config.observations
+    return create_observation_dataframes(migrated_config.observation_declarations, None)
 
 
 @pytest.mark.usefixtures("use_tmpdir")
@@ -228,7 +230,9 @@ def test_that_summary_observations_can_use_restart_for_index_if_refcase_is_given
         run_convert_observations(Namespace(config=str(config_file)))
 
         migrated_config = ErtConfig.from_file("config.ert")
-        observations = migrated_config.observations["summary"]
+        observations = create_observation_dataframes(
+            migrated_config.observation_declarations, None
+        )["summary"]
 
         assert len(observations["time"]) == 1
         assert list(observations["observations"]) == pytest.approx([value])
@@ -279,7 +283,9 @@ def test_that_summary_observations_can_use_restart_for_index_if_time_map_is_give
     run_convert_observations(Namespace(config=str(config_file)))
 
     migrated_config = ErtConfig.from_file("config.ert")
-    observations = migrated_config.observations["summary"]
+    observations = create_observation_dataframes(
+        migrated_config.observation_declarations, None
+    )["summary"]
 
     # RESTART is a 1-based index; Python lists are 0-based.
     assert list(observations["time"]) == [datetime.fromisoformat(time_map[restart])]
@@ -308,8 +314,10 @@ def test_that_rft_config_is_created_from_observations():
             ),
         }
     )
-
-    observations = ert_config.observations["rft"]
+    rft_config = cast(RFTConfig, ert_config.ensemble_config.response_configs["rft"])
+    observations = create_observation_dataframes(
+        ert_config.observation_declarations, rft_config
+    )["rft"]
     assert_frame_equal(
         observations,
         pl.DataFrame(
@@ -325,14 +333,13 @@ def test_that_rft_config_is_created_from_observations():
             }
         ),
     )
-    rft_config = ert_config.ensemble_config.response_configs["rft"]
     assert rft_config.data_to_read == {"well": {"2013-03-31": ["PRESSURE"]}}
     assert rft_config.locations == [(30.0, 71.0, 2000.0)]
 
 
 def test_that_rft_observations_with_unknown_zones_errors():
     with pytest.raises(ConfigValidationError, match="no such zone"):
-        _ = ErtConfig.from_dict(
+        ErtConfig.from_dict(
             {
                 "ECLBASE": "ECLIPSE_CASE",
                 "OBS_CONFIG": (
@@ -388,7 +395,9 @@ def test_that_the_date_keyword_sets_the_summary_index_without_time_map_or_refcas
     run_convert_observations(Namespace(config="config.ert"))
 
     migrated = ErtConfig.from_file("config.ert")
-    observations = migrated.observations["summary"]
+    observations = create_observation_dataframes(
+        migrated.observation_declarations, None
+    )["summary"]
 
     assert list(observations["time"]) == [datetime.fromisoformat(date)]
 
@@ -457,12 +466,10 @@ def test_that_the_date_keyword_sets_the_general_index_by_looking_up_time_map():
     Path("config.ert").write_text(config_content, encoding="utf-8")
 
     run_convert_observations(Namespace(config="config.ert"))
-    assert (
-        ErtConfig.from_file("config.ert")
-        .observations["gen_data"]
-        .to_dicts()[0]["report_step"]
-        == restart
+    observations = create_observation_dataframes(
+        ErtConfig.from_file("config.ert").observation_declarations, None
     )
+    assert observations["gen_data"].to_dicts()[0]["report_step"] == restart
 
 
 @given(summary=summaries(), data=st.data())
@@ -512,12 +519,10 @@ def test_that_the_date_keyword_sets_the_report_step_by_looking_up_refcase(
         )
         Path("config.ert").write_text(config_content, encoding="utf-8")
         run_convert_observations(Namespace(config="config.ert"))
-        assert (
-            ErtConfig.from_file("config.ert")
-            .observations["gen_data"]
-            .to_dicts()[0]["report_step"]
-            == restart
+        observations = create_observation_dataframes(
+            ErtConfig.from_file("config.ert").observation_declarations, None
         )
+        assert observations["gen_data"].to_dicts()[0]["report_step"] == restart
 
 
 @pytest.mark.parametrize("std", [-1.0, 0, 0.0])
@@ -1228,7 +1233,9 @@ def test_that_history_observations_values_are_fetched_from_refcase(
         Path("config.ert").write_text(config_content, encoding="utf-8")
 
         run_convert_observations(Namespace(config="config.ert"))
-        observations = ErtConfig.from_file("config.ert").observations["summary"]
+        observations = create_observation_dataframes(
+            ErtConfig.from_file("config.ert").observation_declarations, None
+        )["summary"]
 
         steps = len(unsmry.steps)
         assert list(observations["response_key"]) == ["FOPR"] * steps
@@ -1407,7 +1414,9 @@ def test_that_history_observation_errors_are_calculated_correctly(tmpdir):
         Path("config.ert").write_text(config_content, encoding="utf-8")
 
         run_convert_observations(Namespace(config="config.ert"))
-        observations = ErtConfig.from_file("config.ert").observations["summary"]
+        observations = create_observation_dataframes(
+            ErtConfig.from_file("config.ert").observation_declarations, None
+        )["summary"]
 
         assert list(observations["response_key"]) == ["FGPR", "FOPR", "FWPR"]
         assert list(observations["observations"]) == pytest.approx([15, 20, 25])
@@ -1448,7 +1457,9 @@ def test_that_segment_defaults_are_applied(tmpdir):
         Path("config.ert").write_text(config_content, encoding="utf-8")
 
         run_convert_observations(Namespace(config="config.ert"))
-        observations = ErtConfig.from_file("config.ert").observations["summary"]
+        observations = create_observation_dataframes(
+            ErtConfig.from_file("config.ert").observation_declarations, None
+        )["summary"]
 
         # default error_min is 0.1
         # default error method is RELMIN

tests/ert/unit_tests/config/test_summary_config.py

@@ -16,7 +16,10 @@
     InvalidResponseFile,
     SummaryConfig,
 )
-from ert.config._create_observation_dataframes import DEFAULT_LOCALIZATION_RADIUS
+from ert.config._create_observation_dataframes import (
+    DEFAULT_LOCALIZATION_RADIUS,
+    create_observation_dataframes,
+)
 
 
 @settings(max_examples=10)
@@ -102,7 +105,9 @@ def create_summary_observation(loc_config_lines):
     Path("prior.txt").write_text("MY_KEYWORD NORMAL 0 1", encoding="utf-8")
 
     ert_config = ErtConfig.from_file("config.ert")
-    return ert_config.observations["summary"]
+    return create_observation_dataframes(ert_config.observation_declarations, None)[
+        "summary"
+    ]
 
 
 @pytest.mark.filterwarnings("ignore:Config contains a SUMMARY key but no forward model")
@@ -198,7 +203,9 @@ def test_that_adding_one_localized_observation_to_snake_oil_case_can_be_internal
         obs_lines.insert(observation_index + 2 + i, line)
     new_obs_content = "\n".join(obs_lines)
     Path("observations/observations.txt").write_text(new_obs_content, encoding="utf-8")
-    summary = ErtConfig.from_file("snake_oil.ert").observations["summary"]
+    summary = create_observation_dataframes(
+        ErtConfig.from_file("snake_oil.ert").observation_declarations, None
+    )["summary"]
     assert summary["east"].dtype == pl.Float32
     assert summary["north"].dtype == pl.Float32
     assert summary["radius"].dtype == pl.Float32

tests/ert/unit_tests/scenarios/test_summary_response.py

@@ -11,6 +11,7 @@
 
 from ert.analysis import ErtAnalysisError, smoother_update
 from ert.config import ErtConfig, ESSettings, ObservationSettings
+from ert.config._create_observation_dataframes import create_observation_dataframes
 from ert.data import MeasuredData
 from ert.sample_prior import sample_prior
 from ert.storage.local_ensemble import load_parameters_and_responses_from_runpath
@@ -21,7 +22,9 @@ def prior_ensemble(storage, ert_config):
     return storage.create_experiment(
         parameters=ert_config.ensemble_config.parameter_configuration,
         responses=ert_config.ensemble_config.response_configuration,
-        observations=ert_config.observations,
+        observations=create_observation_dataframes(
+            ert_config.observation_declarations, None
+        ),
     ).create_ensemble(ensemble_size=3, name="prior")