-
Notifications
You must be signed in to change notification settings - Fork 660
Expand file tree
/
Copy pathviews.py
More file actions
212 lines (182 loc) · 8.71 KB
/
Copy pathviews.py
File metadata and controls
212 lines (182 loc) · 8.71 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
import json
import logging
from typing import Any
from account_v2.custom_exceptions import DuplicateData
from api_v2.exceptions import NoActiveAPIKeyError
from api_v2.key_helper import KeyHelper
from api_v2.postman_collection.dto import PostmanCollection
from django.db import IntegrityError
from django.db.models import F, QuerySet
from django.http import HttpResponse
from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
from plugins import get_plugin
from rest_framework import serializers, status, viewsets
from rest_framework.decorators import action
from rest_framework.filters import OrderingFilter
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from scheduler.helper import SchedulerHelper
from utils.pagination import CustomPagination
from pipeline_v2.constants import (
PipelineConstants,
PipelineErrors,
PipelineExecutionKey,
)
from pipeline_v2.constants import PipelineKey as PK
from pipeline_v2.manager import PipelineManager
from pipeline_v2.models import Pipeline
from pipeline_v2.pipeline_processor import PipelineProcessor
from pipeline_v2.serializers.crud import PipelineSerializer
from pipeline_v2.serializers.execute import (
PipelineExecuteSerializer as ExecuteSerializer,
)
from pipeline_v2.serializers.sharing import SharedUserListSerializer
notification_plugin = get_plugin("notification")
if notification_plugin:
from plugins.notification.constants import ResourceType
logger = logging.getLogger(__name__)
class PipelineViewSet(viewsets.ModelViewSet):
versioning_class = URLPathVersioning
queryset = Pipeline.objects.all()
pagination_class = CustomPagination
filter_backends = [OrderingFilter]
ordering_fields = ["created_at", "last_run_time", "pipeline_name", "run_count"]
# Note: Default ordering with nulls_last is applied in get_queryset()
# DRF's ordering attribute doesn't support nulls_last natively
def get_permissions(self) -> list[Any]:
if self.action in ["destroy", "partial_update", "update"]:
return [IsOwner()]
return [IsOwnerOrSharedUserOrSharedToOrg()]
serializer_class = PipelineSerializer
def get_queryset(self) -> QuerySet:
# Use for_user manager method to include shared pipelines
queryset = Pipeline.objects.for_user(self.request.user)
# Apply type filter if specified
pipeline_type = self.request.query_params.get(PipelineConstants.TYPE)
if pipeline_type is not None:
queryset = queryset.filter(pipeline_type=pipeline_type)
# Filter by workflow ID if provided
workflow_filter = self.request.query_params.get("workflow", None)
if workflow_filter:
queryset = queryset.filter(workflow_id=workflow_filter)
# Search by pipeline name
search = self.request.query_params.get("search", None)
if search:
queryset = queryset.filter(pipeline_name__icontains=search)
# Apply default ordering: last_run_time desc (nulls last), then created_at desc
# This ensures pipelines with recent runs appear first, never-run pipelines at end
queryset = queryset.order_by(
F("last_run_time").desc(nulls_last=True),
F("created_at").desc(),
)
return queryset
def get_serializer_class(self) -> serializers.Serializer:
if self.action == "execute":
return ExecuteSerializer
else:
return PipelineSerializer
# TODO: Refactor to perform an action with explicit arguments
# For eg, passing pipeline ID and with_log=False -> executes pipeline
# For FE however we call the same API twice
# (first call generates execution ID)
def execute(self, request: Request) -> Response:
serializer: ExecuteSerializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
execution_id = serializer.validated_data.get("execution_id", None)
pipeline_id = serializer.validated_data[PK.PIPELINE_ID]
execution = PipelineManager.execute_pipeline(
request=request,
pipeline_id=pipeline_id,
execution_id=execution_id,
)
pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)
serializer = PipelineSerializer(pipeline)
response_data = {
PipelineExecutionKey.PIPELINE: serializer.data,
PipelineExecutionKey.EXECUTION: execution.data,
}
return Response(data=response_data, status=status.HTTP_200_OK)
def create(self, request: Request) -> Response:
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
try:
pipeline_instance = serializer.save()
# Create API key using the created instance
KeyHelper.create_api_key(pipeline_instance, request)
except IntegrityError:
raise DuplicateData(
f"{PipelineErrors.PIPELINE_EXISTS}, {PipelineErrors.DUPLICATE_API}"
)
return Response(data=serializer.data, status=status.HTTP_201_CREATED)
def perform_destroy(self, instance: Pipeline) -> None:
pipeline_to_remove = str(instance.pk)
super().perform_destroy(instance)
return SchedulerHelper.remove_job(pipeline_to_remove)
@action(detail=True, methods=["get"], url_path="users", permission_classes=[IsOwner])
def list_of_shared_users(self, request: Request, pk: str | None = None) -> Response:
"""Returns the list of users the pipeline is shared with."""
pipeline = self.get_object()
serializer = SharedUserListSerializer(pipeline)
return Response(serializer.data, status=status.HTTP_200_OK)
def partial_update(self, request: Request, *args: Any, **kwargs: Any) -> Response:
"""Override to handle sharing notifications."""
instance = self.get_object()
current_shared_users = set(instance.shared_users.all())
response = super().partial_update(request, *args, **kwargs)
if (
response.status_code == 200
and "shared_users" in request.data
and notification_plugin
):
try:
instance.refresh_from_db()
new_shared_users = set(instance.shared_users.all())
newly_shared_users = new_shared_users - current_shared_users
if ResourceType.ETL.value == instance.pipeline_type:
resource_type = ResourceType.ETL.value
elif ResourceType.TASK.value == instance.pipeline_type:
resource_type = ResourceType.TASK.value
if newly_shared_users:
# Get notification service from plugin and send notification
service_class = notification_plugin["service_class"]
notification_service = service_class()
notification_service.send_sharing_notification(
resource_type=resource_type,
resource_name=instance.pipeline_name,
resource_id=str(instance.id),
shared_by=request.user,
shared_to=list(newly_shared_users),
resource_instance=instance,
)
logger.info(
f"Sent sharing notifications for {instance.pipeline_type} "
f"to {len(newly_shared_users)} users"
)
except Exception as e:
# Log error but don't fail the update operation
logger.exception(
f"Failed to send sharing notification, continuing update though: {str(e)}"
)
return response
@action(detail=True, methods=["get"])
def download_postman_collection(
self, request: Request, pk: str | None = None
) -> Response:
"""Downloads a Postman Collection of the API deployment instance."""
instance: Pipeline = self.get_object()
api_key_inst = instance.apikey_set.filter(is_active=True).first()
if not api_key_inst:
logger.error(f"No active API key set for pipeline {instance}")
raise NoActiveAPIKeyError(deployment_name=instance.pipeline_name)
# Create a PostmanCollection for a Pipeline
postman_collection = PostmanCollection.create(
instance=instance, api_key=api_key_inst.api_key
)
response = HttpResponse(
json.dumps(postman_collection.to_dict()), content_type="application/json"
)
response["Content-Disposition"] = (
f'attachment; filename="{instance.pipeline_name}.json"'
)
return response