feat(job-orchestration): Allow concurrent compression job processing by processing batches of compression tasks per job. (#1637)

Co-authored-by: kirkrodrigues <[email protected]>

Author: junhaoliao

components/clp-py-utils/clp_py_utils/clp_config.py

@@ -81,6 +81,7 @@
 StrEnumSerializer = PlainSerializer(serialize_str_enum)
 # Generic types
 NonEmptyStr = Annotated[str, Field(min_length=1)]
+NonNegativeInt = Annotated[int, Field(ge=0)]
 PositiveFloat = Annotated[float, Field(gt=0)]
 PositiveInt = Annotated[int, Field(gt=0)]
 # Specific types
@@ -338,7 +339,10 @@ def transform_for_container(self):
 
 
 class CompressionScheduler(BaseModel):
+    UNLIMITED_CONCURRENT_TASKS_PER_JOB: ClassVar[NonNegativeInt] = 0
+
     jobs_poll_delay: PositiveFloat = 0.1  # seconds
+    max_concurrent_tasks_per_job: NonNegativeInt = UNLIMITED_CONCURRENT_TASKS_PER_JOB
     logging_level: LoggingLevel = "INFO"