Add/model dataset group

api/api.py

@@ -81,6 +81,7 @@ def _enable_datadog_if_setup():
     api_keys,
     quota,
     ssh_keys,
+    asset_versions,
     trackio,
 )
 from transformerlab.routers.auth import get_user_and_team  # noqa: E402
@@ -339,6 +340,7 @@ async def validation_exception_handler(request, exc):
 app.include_router(api_keys.router)
 app.include_router(quota.router)
 app.include_router(ssh_keys.router, dependencies=[Depends(get_user_and_team)])
+app.include_router(asset_versions.router, dependencies=[Depends(get_user_and_team)])
 app.include_router(trackio.router, dependencies=[Depends(get_user_and_team)])
 
 

api/transformerlab/routers/asset_versions.py

@@ -0,0 +1,237 @@
+"""
+asset_versions.py
+
+API router for managing versioned groups of models and datasets.
+Groups are stored as JSON files under the ``asset_groups/`` directory.
+"""
+
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
+
+from transformerlab.services import asset_version_service
+
+
+router = APIRouter(prefix="/asset_versions", tags=["asset_versions"])
+
+
+# ─── Request / Response schemas ───────────────────────────────────────────────
+
+
+class CreateVersionRequest(BaseModel):
+    asset_type: str  # 'model' or 'dataset'
+    group_name: str
+    asset_id: str
+    version_label: str = "v1"
+    job_id: Optional[str] = None
+    description: Optional[str] = None
+    title: Optional[str] = None
+    long_description: Optional[str] = None
+    cover_image: Optional[str] = None
+    evals: Optional[dict] = None
+    extra_metadata: Optional[dict] = None
+    tag: Optional[str] = "latest"
+
+
+class SetTagRequest(BaseModel):
+    tag: str
+
+
+class UpdateVersionRequest(BaseModel):
+    description: Optional[str] = None
+    title: Optional[str] = None
+    long_description: Optional[str] = None
+    cover_image: Optional[str] = None
+    evals: Optional[dict] = None
+    extra_metadata: Optional[dict] = None
+    tag: Optional[str] = None
+
+
+# ─── Group endpoints ─────────────────────────────────────────────────────────
+
+
+@router.get("/groups", summary="List all version groups for a given asset type.")
+async def list_groups(asset_type: str = Query(..., description="'model' or 'dataset'")):
+    try:
+        return await asset_version_service.list_groups(asset_type)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@router.delete(
+    "/groups/{asset_type}/{group_name}",
+    summary="Delete all versions in a group.",
+)
+async def delete_group(asset_type: str, group_name: str):
+    try:
+        count = await asset_version_service.delete_group(asset_type, group_name)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    return {"status": "success", "deleted_count": count}
+
+
+# ─── Version CRUD ─────────────────────────────────────────────────────────────
+
+
+@router.post("/versions", summary="Create a new version in a group.")
+async def create_version(body: CreateVersionRequest):
+    try:
+        result = await asset_version_service.create_version(
+            asset_type=body.asset_type,
+            group_name=body.group_name,
+            asset_id=body.asset_id,
+            version_label=body.version_label,
+            job_id=body.job_id,
+            description=body.description,
+            title=body.title,
+            long_description=body.long_description,
+            cover_image=body.cover_image,
+            evals=body.evals,
+            extra_metadata=body.extra_metadata,
+            tag=body.tag,
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    return result
+
+
+@router.get(
+    "/versions/{asset_type}/{group_name}",
+    summary="List all versions in a group.",
+)
+async def list_versions(asset_type: str, group_name: str):
+    try:
+        return await asset_version_service.list_versions(asset_type, group_name)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@router.get(
+    "/versions/{asset_type}/{group_name}/{version_label}",
+    summary="Get a specific version by its label.",
+)
+async def get_version(asset_type: str, group_name: str, version_label: str):
+    try:
+        result = await asset_version_service.get_version(asset_type, group_name, version_label)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    if result is None:
+        raise HTTPException(status_code=404, detail="Version not found")
+    return result
+
+
+@router.delete(
+    "/versions/{asset_type}/{group_name}/{version_label}",
+    summary="Delete a specific version.",
+)
+async def delete_version(asset_type: str, group_name: str, version_label: str):
+    try:
+        deleted = await asset_version_service.delete_version(asset_type, group_name, version_label)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    if not deleted:
+        raise HTTPException(status_code=404, detail="Version not found")
+    return {"status": "success"}
+
+
+# ─── Version update ──────────────────────────────────────────────────────────
+
+
+@router.patch(
+    "/versions/{asset_type}/{group_name}/{version_label}",
+    summary="Update metadata or tag on a specific version.",
+)
+async def update_version(asset_type: str, group_name: str, version_label: str, body: UpdateVersionRequest):
+    # Build kwargs only for fields the caller actually sent (present in the JSON body).
+    # This lets the service layer distinguish "not provided" from "set to null".
+    raw = body.model_dump(exclude_unset=True)
+
+    # Map body fields to service kwargs using the sentinel pattern
+    kwargs = {}
+    for field in (
+        "description",
+        "title",
+        "long_description",
+        "cover_image",
+        "evals",
+        "extra_metadata",
+        "tag",
+    ):
+        if field in raw:
+            kwargs[field] = raw[field]
+
+    try:
+        result = await asset_version_service.update_version(asset_type, group_name, version_label, **kwargs)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    if result is None:
+        raise HTTPException(status_code=404, detail="Version not found")
+    return result
+
+
+# ─── Tag management ──────────────────────────────────────────────────────────
+
+
+@router.put(
+    "/versions/{asset_type}/{group_name}/{version_label}/tag",
+    summary="Set a tag on a specific version. Moves the tag from any other version in the group.",
+)
+async def set_tag(asset_type: str, group_name: str, version_label: str, body: SetTagRequest):
+    try:
+        result = await asset_version_service.set_tag(asset_type, group_name, version_label, body.tag)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    if result is None:
+        raise HTTPException(status_code=404, detail="Version not found")
+    return result
+
+
+@router.delete(
+    "/versions/{asset_type}/{group_name}/{version_label}/tag",
+    summary="Clear the tag from a specific version.",
+)
+async def clear_tag(asset_type: str, group_name: str, version_label: str):
+    try:
+        result = await asset_version_service.clear_tag(asset_type, group_name, version_label)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    if result is None:
+        raise HTTPException(status_code=404, detail="Version not found")
+    return result
+
+
+# ─── Resolution ──────────────────────────────────────────────────────────────
+
+
+@router.get(
+    "/resolve/{asset_type}/{group_name}",
+    summary="Resolve a group to a specific version. Defaults to 'latest' tag.",
+)
+async def resolve(
+    asset_type: str,
+    group_name: str,
+    tag: Optional[str] = Query(None, description="Tag to resolve (any string, e.g. 'latest', 'production')"),
+    version_label: Optional[str] = Query(None, description="Exact version label to resolve"),
+):
+    try:
+        result = await asset_version_service.resolve(asset_type, group_name, tag=tag, version_label=version_label)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    if result is None:
+        raise HTTPException(status_code=404, detail="No matching version found")
+    return result
+
+
+# ─── Bulk lookups (used by list views) ────────────────────────────────────────
+
+
+@router.get(
+    "/map/{asset_type}",
+    summary="Get a map of asset_id -> group memberships for annotating list views.",
+)
+async def get_asset_group_map(asset_type: str):
+    try:
+        return await asset_version_service.get_all_asset_group_map(asset_type)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))

api/transformerlab/routers/data.py

@@ -841,6 +841,20 @@ async def dataset_list(generated: bool = True):
     except Exception:
         merged_list = []
 
+    # Augment each dataset with version group info if any
+    try:
+        from transformerlab.services import asset_version_service
+
+        group_map = await asset_version_service.get_all_asset_group_map("dataset")
+        for entry in merged_list:
+            dataset_id = entry.get("dataset_id", "")
+            if dataset_id in group_map:
+                entry["version_groups"] = group_map[dataset_id]
+            else:
+                entry["version_groups"] = []
+    except Exception as e:
+        print(f"Warning: could not fetch dataset version groups: {e}")
+
     if generated:
         return merged_list