Skip to content

Add possibility to read rft observations from csv file #12628

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HakonSohoel merged 5 commits into from
Jan 14, 2026
Merged

Add possibility to read rft observations from csv file #12628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
23b6e4d
Add option to read rft observations from csv file
HakonSohoel Jan 12, 2026
8c63d10
Improve error message if non existing csv file
HakonSohoel Jan 14, 2026
2a29195
Add tests for more edge cases
HakonSohoel Jan 14, 2026
475c02e
Specify pd.read_csv() parameters explicitly
HakonSohoel Jan 14, 2026
acddbb9
Add test for missing user specified property
HakonSohoel Jan 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 115 additions & 36 deletions src/ert/config/_observations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
from itertools import starmap
from typing import Any, Self

import pandas as pd

from .parsing import (
ErrorInfo,
ObservationConfigError,
Expand Down Expand Up @@ -48,7 +50,9 @@ def key(self) -> str:
return self.name

@classmethod
def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
def from_obs_dict(
cls, directory: str, observation_dict: ObservationDict
) -> list[Self]:
error_mode = ErrorModes.RELMIN
error = 0.1
error_min = 0.1
Expand All @@ -68,13 +72,15 @@ def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Sel
case _:
raise _unknown_key_error(str(key), observation_dict["name"])

return cls(
name=observation_dict["name"],
error_mode=error_mode,
error=error,
error_min=error_min,
segments=segments,
)
return [
cls(
name=observation_dict["name"],
error_mode=error_mode,
error=error,
error_min=error_min,
segments=segments,
)
]


@dataclass
Expand All @@ -98,7 +104,9 @@ class _SummaryValues:
@dataclass
class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
@classmethod
def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
def from_obs_dict(
cls, directory: str, observation_dict: ObservationDict
) -> list[Self]:
error_mode = ErrorModes.ABS
summary_key = None

Expand Down Expand Up @@ -140,18 +148,20 @@ def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Sel
if "ERROR" not in float_values:
raise _missing_value_error(observation_dict["name"], "ERROR")

return cls(
name=observation_dict["name"],
error_mode=error_mode,
error=float_values["ERROR"],
error_min=float_values["ERROR_MIN"],
key=summary_key,
value=float_values["VALUE"],
location_x=localization_values.get("x"),
location_y=localization_values.get("y"),
location_range=localization_values.get("range"),
**date_dict.__dict__,
)
return [
cls(
name=observation_dict["name"],
error_mode=error_mode,
error=float_values["ERROR"],
error_min=float_values["ERROR_MIN"],
key=summary_key,
value=float_values["VALUE"],
location_x=localization_values.get("x"),
location_y=localization_values.get("y"),
location_range=localization_values.get("range"),
**date_dict.__dict__,
)
]


@dataclass
Expand All @@ -168,7 +178,9 @@ class _GeneralObservation:
@dataclass
class GeneralObservation(ObservationDate, _GeneralObservation):
@classmethod
def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
def from_obs_dict(
cls, directory: str, observation_dict: ObservationDict
) -> list[Self]:
try:
data = observation_dict["DATA"]
except KeyError as err:
Expand Down Expand Up @@ -211,7 +223,7 @@ def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Sel
f" VALUE = {output.value}, ERROR must also be given.",
observation_dict["name"],
)
return output
return [output]


@dataclass
Expand All @@ -227,7 +239,63 @@ class RFTObservation:
tvd: float

@classmethod
def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
def from_csv(
cls,
directory: str,
observation_dict: ObservationDict,
filename: str,
observed_property: str = "PRESSURE",
) -> list[Self]:
if not os.path.isabs(filename):
filename = os.path.join(directory, filename)
if not os.path.exists(filename):
raise ObservationConfigError.with_context(
f"The CSV file ({filename}) does not exist or is not accessible.",
filename,
)
csv_file = pd.read_csv(
filename,
encoding="utf-8",
on_bad_lines="error",
)

required_columns = {
"WELL_NAME",
"DATE",
observed_property,
"ERROR",
"NORTH",
"EAST",
"TVD",
}
missing_required_columns = required_columns - set(csv_file.keys())
if missing_required_columns:
raise ObservationConfigError.with_context(
f"The rft observations file {filename} is missing required column(s) "
f"{', '.join(sorted(missing_required_columns))}.",
filename,
)

return [
cls(
f"{observation_dict['name']}[{row.Index}]",
str(row.WELL_NAME),
str(row.DATE),
observed_property,
validate_float(str(getattr(row, observed_property)), observed_property),
validate_float(str(row.ERROR), "ERROR"),
validate_float(str(row.NORTH), "NORTH"),
validate_float(str(row.EAST), "EAST"),
validate_float(str(row.TVD), "TVD"),
)
for row in csv_file.itertuples(index=True)
]

@classmethod
def from_obs_dict(
cls, directory: str, observation_dict: ObservationDict
) -> list[Self]:
csv_filename = None
well = None
observed_property = None
observed_value = None
Expand Down Expand Up @@ -256,8 +324,17 @@ def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Sel
east = validate_float(value, key)
case "TVD":
tvd = validate_float(value, key)
case "CSV":
csv_filename = value
case _:
raise _unknown_key_error(str(key), observation_dict["name"])
if csv_filename is not None:
return cls.from_csv(
directory,
observation_dict,
csv_filename,
observed_property or "PRESSURE",
)
if well is None:
raise _missing_value_error(observation_dict["name"], "WELL")
if observed_value is None:
Expand All @@ -274,17 +351,19 @@ def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Sel
raise _missing_value_error(observation_dict["name"], "EAST")
if tvd is None:
raise _missing_value_error(observation_dict["name"], "TVD")
return cls(
observation_dict["name"],
well,
date,
observed_property,
observed_value,
error,
north,
east,
tvd,
)
return [
cls(
observation_dict["name"],
well,
date,
observed_property,
observed_value,
error,
north,
east,
tvd,
)
]


Observation = (
Expand Down Expand Up @@ -313,7 +392,7 @@ def make_observations(
error_list: list[ErrorInfo] = []
for obs_dict in observation_dicts:
try:
result.append(
result.extend(
_TYPE_TO_CLASS[obs_dict["type"]].from_obs_dict(directory, obs_dict)
)
except KeyError as err:
Expand Down
34 changes: 25 additions & 9 deletions tests/ert/unit_tests/config/parsing/test_observations_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,24 +88,34 @@ def test_parse_observations():
def test_parse_rft_observations():
assert parse_observations(
"""
RFT_OBSERVATION NAME
RFT_OBSERVATION NAME1
{
VALUE = 700;
ERROR = 0.1;
DATE = 2010-03-31;
PROPERTY = PRESSURE;
};

RFT_OBSERVATION NAME2
{
CSV = rft_observations.csv;
};
""",
"",
) == [
{
"type": ObservationType.RFT,
"name": "NAME",
"name": "NAME1",
"VALUE": "700",
"ERROR": "0.1",
"DATE": "2010-03-31",
"PROPERTY": "PRESSURE",
},
{
"type": ObservationType.RFT,
"name": "NAME2",
"CSV": "rft_observations.csv",
},
]


Expand All @@ -115,14 +125,16 @@ def test_that_missing_assignment_in_observation_body_shows_informative_error_mes
)
with pytest.raises(ObservationConfigError, match=expected_match):
parse_observations(
content=dedent("""\
content=dedent(
"""\
GENERAL_OBSERVATION POLY_OBS_0 {
DATA = POLY_RES;
INDEX_LIST = 0,2,4,6,8;
OBS_FILE = poly_obs_data.txt;
A
};
"""),
"""
),
filename="",
)

Expand All @@ -134,13 +146,15 @@ def test_that_misspelled_observation_type_shows_informative_error_message():
)
with pytest.raises(ObservationConfigError, match=expected_match):
parse_observations(
content=dedent("""\
content=dedent(
"""\
MISSPELLED_OBSERVATION POLY_OBS_0 {
DATA = POLY_RES;
INDEX_LIST = 0,2,4,6,8;
OBS_FILE = poly_obs_data.txt;
};
"""),
"""
),
filename="",
)

Expand All @@ -152,13 +166,15 @@ def test_that_invalid_start_of_observation_body_symbol_show_informative_error_me
)
with pytest.raises(ObservationConfigError, match=expected_match):
parse_observations(
content=dedent("""\
content=dedent(
"""\
GENERAL_OBSERVATION POLY_OBS_0 (
DATA = POLY_RES;
INDEX_LIST = 0,2,4,6,8;
OBS_FILE = poly_obs_data.txt;
};
"""),
"""
),
filename="",
)

Expand All @@ -170,7 +186,7 @@ def test_that_repeated_comments_are_ignored():
FOPR;
""",
"",
) == [({"type": ObservationType.SUMMARY, "name": "FOPR"})]
) == [{"type": ObservationType.SUMMARY, "name": "FOPR"}]


@pytest.mark.parametrize("observation_type", ["HISTORY", "GENERAL", "SUMMARY"])
Expand Down
Loading
Loading