feat: Making the configs' properties relative paths.

Author: SamuelLarkin

everyvoice/config/preprocessing_config.py

@@ -4,9 +4,9 @@
 from typing import List, Optional, Union
 
 from loguru import logger
-from pydantic import Field, FilePath, field_validator, model_validator
+from pydantic import Field, FilePath, ValidationInfo, field_validator, model_validator
 
-from everyvoice.config.shared_types import ConfigModel
+from everyvoice.config.shared_types import ConfigModel, PartialLoadConfig, init_context
 from everyvoice.config.utils import (
     PossiblyRelativePath,
     PossiblySerializedCallable,
@@ -50,7 +50,7 @@ class PitchCalculationMethod(Enum):
     cwt = "cwt"
 
 
-class Dataset(ConfigModel):
+class Dataset(PartialLoadConfig):
     label: str = "YourDataSet"
     data_dir: PossiblyRelativePath = Path("/please/create/a/path/to/your/dataset/data")
     textgrid_dir: Union[PossiblyRelativePath, None] = None
@@ -60,8 +60,17 @@ class Dataset(ConfigModel):
     filelist_loader: PossiblySerializedCallable = generic_dict_loader
     sox_effects: list = [["channels", "1"]]
 
+    @field_validator(
+        "data_dir",
+        "textgrid_dir",
+        "filelist",
+    )
+    @classmethod
+    def relative_to_absolute(cls, value: Path, info: ValidationInfo) -> Path:
+        return PartialLoadConfig.path_relative_to_absolute(value, info)
+
 
-class PreprocessingConfig(ConfigModel):
+class PreprocessingConfig(PartialLoadConfig):
     dataset: str = "YourDataSet"
     pitch_type: Union[
         PitchCalculationMethod, str
@@ -76,9 +85,16 @@ class PreprocessingConfig(ConfigModel):
     path_to_audio_config_file: Optional[FilePath] = None
     source_data: List[Dataset] = Field(default_factory=lambda: [Dataset()])
 
-    @model_validator(mode="before")
-    def load_partials(self):
-        return load_partials(self, ["audio"])
+    @model_validator(mode="before")  # type: ignore
+    def load_partials(self, info: ValidationInfo):
+        config_path = (
+            info.context.get("config_path", None) if info.context is not None else None
+        )
+        return load_partials(
+            self,  # type: ignore
+            ("audio",),
+            config_path=config_path,
+        )
 
     @field_validator("save_dir", mode="after")
     def create_dir(cls, value: Path):
@@ -93,4 +109,6 @@ def create_dir(cls, value: Path):
     def load_config_from_path(path: Path) -> "PreprocessingConfig":
         """Load a config from a path"""
         config = load_config_from_json_or_yaml_path(path)
-        return PreprocessingConfig(**config)
+        with init_context({"config_path": path}):
+            config = PreprocessingConfig(**config)
+        return config

everyvoice/config/shared_types.py

@@ -1,14 +1,35 @@
 from collections.abc import Mapping, Sequence
+from contextlib import contextmanager
+from contextvars import ContextVar
 from functools import cached_property
 from pathlib import Path
-from typing import Tuple, Union
+from typing import Any, Dict, Iterator, Tuple, Union
 
 from loguru import logger
-from pydantic import BaseModel, ConfigDict, DirectoryPath, Field, validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    DirectoryPath,
+    Field,
+    ValidationInfo,
+    field_validator,
+    validator,
+)
 
 from everyvoice.config.utils import PossiblyRelativePath, PossiblySerializedCallable
 from everyvoice.utils import generic_dict_loader, get_current_time, rel_path_to_abs_path
 
+_init_context_var = ContextVar("_init_context_var", default=None)
+
+
+@contextmanager
+def init_context(value: Dict[str, Any]) -> Iterator[None]:
+    token = _init_context_var.set(value)  # type: ignore
+    try:
+        yield
+    finally:
+        _init_context_var.reset(token)
+
 
 class ConfigModel(BaseModel):
     model_config = ConfigDict(
@@ -48,7 +69,26 @@ def combine_configs(orig_dict: Union[dict, Sequence], new_dict: dict):
         return orig_dict
 
 
-class LoggerConfig(ConfigModel):
+class PartialLoadConfig(ConfigModel):
+    """Models that have partial models which requires a context to properly load."""
+
+    # [Using validation context with BaseModel initialization](https://docs.pydantic.dev/2.3/usage/validators/#using-validation-context-with-basemodel-initialization)
+    def __init__(__pydantic_self__, **data: Any) -> None:
+        __pydantic_self__.__pydantic_validator__.validate_python(
+            data,
+            self_instance=__pydantic_self__,
+            context=_init_context_var.get(),
+        )
+
+    @classmethod
+    def path_relative_to_absolute(cls, value: Path, info: ValidationInfo) -> Path:
+        if info.context and value is not None and not value.is_absolute():
+            config_path = info.context.get("config_path", Path("."))
+            value = (config_path / value).resolve()
+        return value
+
+
+class LoggerConfig(PartialLoadConfig):
     """The logger configures all the information needed for where to store your experiment's logs and checkpoints.
     The structure of your logs will then be:
     <name> / <version> / <sub_dir>
@@ -67,6 +107,11 @@ class LoggerConfig(ConfigModel):
     version: str = "base"
     """The version of your experiment"""
 
+    @field_validator("save_dir")
+    @classmethod
+    def relative_to_absolute(cls, value: Path, info: ValidationInfo) -> Path:
+        return PartialLoadConfig.path_relative_to_absolute(value, info)
+
     @cached_property
     def sub_dir(self) -> str:
         return self.sub_dir_callable()
@@ -83,7 +128,7 @@ def convert_path(cls, v, values):
         return path
 
 
-class BaseTrainingConfig(ConfigModel):
+class BaseTrainingConfig(PartialLoadConfig):
     batch_size: int = 16
     save_top_k_ckpts: int = 5
     ckpt_steps: Union[int, None] = None
@@ -102,6 +147,11 @@ class BaseTrainingConfig(ConfigModel):
     val_data_workers: int = 0
     train_data_workers: int = 4
 
+    @field_validator("training_filelist", "validation_filelist")
+    @classmethod
+    def relative_to_absolute(cls, value: Path, info: ValidationInfo) -> Path:
+        return PartialLoadConfig.path_relative_to_absolute(value, info)
+
 
 class BaseOptimizer(ConfigModel):
     learning_rate: float = 1e-4

everyvoice/config/utils.py

@@ -1,6 +1,6 @@
 from importlib import import_module
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Union
+from typing import Any, Callable, Dict, Optional, Sequence, Union
 
 from loguru import logger
 from pydantic import PlainSerializer, WithJsonSchema
@@ -10,10 +10,15 @@
 from everyvoice.utils import load_config_from_json_or_yaml_path, rel_path_to_abs_path
 
 
-def load_partials(pre_validated_model_dict: Dict[Any, Any], partial_keys: List[str]):
-    """Loads all partials based on a list of partial keys. For this to work, your model
-    must have a {key}_config_file: Optional[FilePath] = None field defined, and you must
-    have a model_validator(mode="before") that runs this function.
+def load_partials(
+    pre_validated_model_dict: Dict[Any, Any],
+    partial_keys: Sequence[str],
+    config_path: Optional[Path] = None,
+):
+    """Loads all partials based on a list of partial keys. For this to work,
+    your model must have a {key}_config_file: Optional[FilePath] = None field
+    defined, and you must have a model_validator(mode="before") that runs this
+    function.
     """
     # If there's nothing there, just return the dict
     if not pre_validated_model_dict:
@@ -25,9 +30,10 @@ def load_partials(pre_validated_model_dict: Dict[Any, Any], partial_keys: List[s
             key_for_path_to_partial in pre_validated_model_dict
             and pre_validated_model_dict[key_for_path_to_partial]
         ):
-            subconfig_path = rel_path_to_abs_path(
-                pre_validated_model_dict[key_for_path_to_partial]
-            )
+            subconfig_path = Path(pre_validated_model_dict[key_for_path_to_partial])
+            if not subconfig_path.is_absolute() and config_path is not None:
+                subconfig_path = (config_path.parent / subconfig_path).resolve()
+            pre_validated_model_dict[key_for_path_to_partial] = subconfig_path
             # anything defined in the key will override the path
             # so audio would override any values in path_to_audio_config_file
             if key in pre_validated_model_dict:

everyvoice/model/aligner/DeepForcedAligner

@@ -1,9 +1,13 @@
 from pathlib import Path
 from typing import Optional, Union
 
-from pydantic import Field, FilePath, model_validator
+from pydantic import Field, FilePath, ValidationInfo, model_validator
 
-from everyvoice.config.shared_types import BaseTrainingConfig, ConfigModel
+from everyvoice.config.shared_types import (
+    BaseTrainingConfig,
+    PartialLoadConfig,
+    init_context,
+)
 from everyvoice.config.utils import PossiblyRelativePath, load_partials
 from everyvoice.model.aligner.config import AlignerConfig
 from everyvoice.model.feature_prediction.config import FeaturePredictionConfig
@@ -16,7 +20,7 @@ class E2ETrainingConfig(BaseTrainingConfig):
     vocoder_checkpoint: Union[None, PossiblyRelativePath] = None
 
 
-class EveryVoiceConfig(ConfigModel):
+class EveryVoiceConfig(PartialLoadConfig):
     aligner: AlignerConfig = Field(default_factory=AlignerConfig)
     path_to_aligner_config_file: Optional[FilePath] = None
 
@@ -31,10 +35,15 @@ class EveryVoiceConfig(ConfigModel):
     training: E2ETrainingConfig = Field(default_factory=E2ETrainingConfig)
     path_to_training_config_file: Optional[FilePath] = None
 
-    @model_validator(mode="before")
-    def load_partials(self):
+    @model_validator(mode="before")  # type: ignore
+    def load_partials(self, info: ValidationInfo):
+        config_path = (
+            info.context.get("config_path", None) if info.context is not None else None
+        )
         return load_partials(
-            self, ["aligner", "feature_prediction", "vocoder", "training"]
+            self,  # type: ignore
+            ("aligner", "feature_prediction", "vocoder", "training"),
+            config_path=config_path,
         )
 
     @staticmethod
@@ -43,4 +52,6 @@ def load_config_from_path(
     ) -> "EveryVoiceConfig":
         """Load a config from a path"""
         config = load_config_from_json_or_yaml_path(path)
-        return EveryVoiceConfig(**config)
+        with init_context({"config_path": path}):
+            config = EveryVoiceConfig(**config)
+        return config

everyvoice/model/e2e/config/__init__.py

@@ -10,7 +10,7 @@
 from loguru import logger
 
 from everyvoice.tests.test_cli import CLITest
-from everyvoice.tests.test_configs import ConfigTest
+from everyvoice.tests.test_configs import ConfigTest, LoadConfigTest
 from everyvoice.tests.test_dataloader import DataLoaderTest
 from everyvoice.tests.test_model import ModelTest
 from everyvoice.tests.test_preprocessing import (
@@ -25,7 +25,9 @@
 
 LOADER = TestLoader()
 
-CONFIG_TESTS = [LOADER.loadTestsFromTestCase(test) for test in [ConfigTest]]
+CONFIG_TESTS = [
+    LOADER.loadTestsFromTestCase(test) for test in [ConfigTest, LoadConfigTest]
+]
 
 DATALOADER_TESTS = [LOADER.loadTestsFromTestCase(test) for test in [DataLoaderTest]]
 

everyvoice/model/feature_prediction/FastSpeech2_lightning

@@ -0,0 +1,25 @@
+model: {conv_dim: 512, lstm_dim: 512}
+path_to_preprocessing_config_file: everyvoice-shared-data.yaml
+path_to_text_config_file: everyvoice-shared-text.yaml
+training:
+  batch_size: 16
+  binned_sampler: true
+  ckpt_epochs: 1
+  extraction_method: dijkstra
+  filelist_loader: everyvoice.utils.generic_dict_loader
+  logger: {name: AlignerExperiment, save_dir: ../logs_and_checkpoints, sub_dir_callable: everyvoice.utils.get_current_time,
+    version: base}
+  max_epochs: 1000
+  max_steps: 100000
+  optimizer:
+    betas: [0.9, 0.98]
+    eps: 1.0e-08
+    learning_rate: 0.0001
+    name: adamw
+    weight_decay: 0.01
+  plot_steps: 1000
+  save_top_k_ckpts: 5
+  train_data_workers: 4
+  training_filelist: ../preprocessed/training_filelist.psv
+  val_data_workers: 0
+  validation_filelist: ../preprocessed/validation_filelist.psv

everyvoice/model/vocoder/HiFiGAN_iSTFT_lightning

@@ -0,0 +1,20 @@
+audio: {alignment_bit_depth: 16, alignment_sampling_rate: 22050, f_max: 8000, f_min: 0,
+  fft_hop_frames: 256, fft_window_frames: 1024, input_sampling_rate: 22050, max_audio_length: 11.0,
+  max_wav_value: 32767.0, min_audio_length: 0.4, n_fft: 1024, n_mels: 80, norm_db: -3.0,
+  output_sampling_rate: 22050, sil_duration: 0.1, sil_threshold: 1.0, spec_type: mel-librosa,
+  target_bit_depth: 16, vocoder_segment_size: 8192}
+dataset: relative
+dataset_split_seed: 1234
+energy_phone_averaging: true
+pitch_phone_averaging: true
+pitch_type: pyworld
+save_dir: ../preprocessed
+source_data:
+- data_dir: ../../lj/wavs
+  filelist: ../r-filelist.psv
+  filelist_loader: everyvoice.utils.generic_dict_loader
+  label: dataset_0
+  sox_effects:
+  - [channel, '1']
+train_split: 0.9
+value_separator: --