From e6bf7b7ebda4a22f0b8d92bbd04f3452f941e618 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 00:35:23 +0200 Subject: [PATCH 01/21] implement draft logic to support json schema validation for partner field --- src/django/api/admin.py | 5 +- .../0186_add_json_schema_to_partner_field.py | 19 +++ src/django/api/models/partner_field.py | 8 + .../location_contribution.py | 3 + .../processors/partner_type_processor.py | 149 ++++++++++++++++++ .../processors/permission_processor.py | 80 +--------- src/django/requirements.txt | 1 + 7 files changed, 192 insertions(+), 73 deletions(-) create mode 100644 src/django/api/migrations/0186_add_json_schema_to_partner_field.py create mode 100644 src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py diff --git a/src/django/api/admin.py b/src/django/api/admin.py index 6cbe21a2d..56d64a16f 100644 --- a/src/django/api/admin.py +++ b/src/django/api/admin.py @@ -252,7 +252,10 @@ class PartnerFieldAdminForm(forms.ModelForm): class Meta: model = PartnerField - fields = ['name', 'type', 'unit', 'label', 'source_by'] + fields = ['name', 'type', 'unit', 'label', 'source_by', 'json_schema'] + widgets = { + 'json_schema': forms.Textarea(attrs={'rows': 10, 'cols': 80}), + } class PartnerFieldAdmin(admin.ModelAdmin): diff --git a/src/django/api/migrations/0186_add_json_schema_to_partner_field.py b/src/django/api/migrations/0186_add_json_schema_to_partner_field.py new file mode 100644 index 000000000..139b201db --- /dev/null +++ b/src/django/api/migrations/0186_add_json_schema_to_partner_field.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.17 on 2025-01-18 12:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0185_add_source_by_to_partner_field'), + ] + + operations = [ + migrations.AddField( + model_name='partnerfield', + name='json_schema', + field=models.JSONField(blank=True, help_text='JSON Schema for validating object type partner fields. Used when type is "object".'), + ), + ] + diff --git a/src/django/api/models/partner_field.py b/src/django/api/models/partner_field.py index 72e32630c..d2f27eb18 100644 --- a/src/django/api/models/partner_field.py +++ b/src/django/api/models/partner_field.py @@ -60,6 +60,14 @@ class Meta: ) ) + json_schema = models.JSONField( + blank=True, + help_text=( + 'JSON Schema for validating object type partner fields. ' + 'Used when type is "object".' + ) + ) + created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/location_contribution.py b/src/django/api/moderation_event_actions/creation/location_contribution/location_contribution.py index 3033bce43..1302f828e 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/location_contribution.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/location_contribution.py @@ -9,6 +9,8 @@ import ProductionLocationDataProcessor from api.moderation_event_actions.creation.location_contribution \ .processors.geocoding_processor import GeocodingProcessor +from api.moderation_event_actions.creation.location_contribution \ + .processors.partner_type_processor import PartnerTypeProcessor from api.moderation_event_actions.creation.location_contribution \ .processors.permission_processor import PermissionProcessor from api.moderation_event_actions.creation.dtos.create_moderation_event_dto \ @@ -37,6 +39,7 @@ def serialize( def __setup_location_data_processors() -> ContributionProcessor: location_data_processors = ( PermissionProcessor(), + PartnerTypeProcessor(), SourceProcessor(), ProductionLocationDataProcessor(), GeocodingProcessor() diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py new file mode 100644 index 000000000..f173157ec --- /dev/null +++ b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py @@ -0,0 +1,149 @@ +from typing import Dict, List, Mapping, Tuple + +import jsonschema +from jsonschema.exceptions import ValidationError as JsonSchemaValidationError +from rest_framework import status + +from api.moderation_event_actions.creation.location_contribution \ + .processors.contribution_processor import ContributionProcessor +from api.moderation_event_actions.creation.dtos.create_moderation_event_dto \ + import CreateModerationEventDTO +from api.models.partner_field import PartnerField +from api.constants import APIV1CommonErrorMessages + + +class PartnerTypeProcessor(ContributionProcessor): + + TYPE_VALIDATORS = { + 'int': lambda value: isinstance(value, int) + and not isinstance(value, bool), + 'float': lambda value: isinstance(value, float) + and not isinstance(value, bool), + 'string': lambda value: isinstance(value, str), + 'object': lambda value: isinstance( + value, (dict, list) + ), + } + + def process( + self, + event_dto: CreateModerationEventDTO + ) -> CreateModerationEventDTO: + + raw = event_dto.raw_data or {} + if not raw: + return super().process(event_dto) + + incoming_keys = set(raw.keys()) + + partner_fields_qs = PartnerField.objects \ + .filter(name__in=incoming_keys) \ + .values("name", "type", "json_schema") + partner_fields_data: Dict[str, Dict] = { + field["name"]: { + "type": field["type"], + "json_schema": field["json_schema"] + } + for field in partner_fields_qs + } + + if not partner_fields_data: + return super().process(event_dto) + + # Validate all fields: use type validation or JSON Schema validation + # based on field type and json_schema presence + validation_errors = self.__validate_partner_fields( + raw, + partner_fields_data + ) + + if validation_errors: + event_dto.errors = self.__transform_validation_errors( + validation_errors + ) + event_dto.status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + return event_dto + + return super().process(event_dto) + + @staticmethod + def __validate_partner_fields( + raw: Mapping[str, object], + partner_fields_data: Dict[str, Dict] + ) -> List[Tuple[str, str]]: + """ + Validates partner fields using either type validation or JSON Schema + validation based on field type and json_schema presence. + + Logic: + - If type is NOT object → use type validation + - If type is object AND json_schema is empty → use type validation + - If type is object AND json_schema is NOT empty → use JSON Schema validation + + Returns a list of tuples (field_name, error_message) for fields + that fail validation. + """ + validation_errors: List[Tuple[str, str]] = [] + + for field_name, field_info in partner_fields_data.items(): + field_type = field_info.get("type") + json_schema = field_info.get("json_schema") + value = raw.get(field_name) + + if value is None: + continue + + # Check if we should use JSON Schema validation + # Only if type is object AND json_schema is not empty + use_json_schema = ( + field_type == PartnerField.OBJECT and json_schema + ) + + if use_json_schema: + # Use JSON Schema validation + try: + jsonschema.validate(instance=value, schema=json_schema) + except JsonSchemaValidationError as e: + error_message = e.message + if e.path: + error_path = ".".join(str(p) for p in e.path) + error_message = f"{error_path}: {error_message}" + validation_errors.append((field_name, error_message)) + except Exception as e: + # Handle schema errors (invalid JSON schema itself) + validation_errors.append( + (field_name, f"Schema validation error: {str(e)}") + ) + else: + # Use type validation + if field_type in PartnerTypeProcessor.TYPE_VALIDATORS: + validator = PartnerTypeProcessor.TYPE_VALIDATORS[field_type] + if not validator(value): + validation_errors.append( + ( + field_name, + f'Field {field_name} must be {field_type}, ' + f'not {type(value).__name__}.' + ) + ) + + return validation_errors + + @staticmethod + def __transform_validation_errors( + validation_errors: List[Tuple[str, str]] + ) -> Dict: + """ + Transforms validation errors into the standard error format. + """ + return { + 'detail': APIV1CommonErrorMessages.COMMON_REQ_BODY_ERROR, + 'errors': [ + { + 'field': field_name, + 'detail': error_message, + } + for field_name, error_message in validation_errors + ], + } + diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/processors/permission_processor.py b/src/django/api/moderation_event_actions/creation/location_contribution/processors/permission_processor.py index 761588d62..f72c69c86 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/processors/permission_processor.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/processors/permission_processor.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Mapping, Tuple, Iterable +from typing import Dict, Iterable from rest_framework import status @@ -12,17 +12,6 @@ class PermissionProcessor(ContributionProcessor): - TYPE_VALIDATORS = { - 'int': lambda value: isinstance(value, int) - and not isinstance(value, bool), - 'float': lambda value: isinstance(value, float) - and not isinstance(value, bool), - 'string': lambda value: isinstance(value, str), - 'object': lambda value: isinstance( - value, (dict, list) - ), - } - def process( self, event_dto: CreateModerationEventDTO @@ -34,19 +23,20 @@ def process( incoming_keys = set(raw.keys()) - partner_fields_qs = PartnerField.objects \ - .filter(name__in=incoming_keys) \ - .values_list("name", "type") - partner_fields: Dict[str, str] = dict(partner_fields_qs) + partner_field_names = list( + PartnerField.objects + .filter(name__in=incoming_keys) + .values_list("name", flat=True) + ) - if not partner_fields: + if not partner_field_names: return super().process(event_dto) # Permission validation. contributor_allowed: set[str] = set( event_dto.contributor.partner_fields.values_list("name", flat=True) ) - requested_partner_field_names: set[str] = set(partner_fields.keys()) + requested_partner_field_names: set[str] = set(partner_field_names) unauthorized: set[str] = requested_partner_field_names \ - contributor_allowed @@ -55,22 +45,6 @@ def process( event_dto.status_code = status.HTTP_403_FORBIDDEN return event_dto - # Type validation. - invalid_type_fields = self.__collect_invalid_type_fields( - raw, - partner_fields, - self.TYPE_VALIDATORS - ) - - if invalid_type_fields: - event_dto.errors = self.__transform_type_errors( - invalid_type_fields - ) - event_dto.status_code = status \ - .HTTP_422_UNPROCESSABLE_ENTITY - - return event_dto - return super().process(event_dto) @staticmethod @@ -88,41 +62,3 @@ def __transform_permission_errors( for field_name in fields_errors ], } - - @staticmethod - def __transform_type_errors( - invalid_type_fields: List[tuple] - ) -> Dict: - return { - 'detail': APIV1CommonErrorMessages.COMMON_REQ_BODY_ERROR, - 'errors': [ - { - 'field': name, - 'detail': ( - f'Field {name} must be {expected}, ' - f'not {type(value).__name__}.' - ), - } - for name, expected, value in invalid_type_fields - ], - } - - @staticmethod - def __collect_invalid_type_fields( - raw: Mapping[str, object], - partner_fields: Mapping[str, str], - validators: Mapping[str, callable], - ) -> List[Tuple[str, str, object]]: - - invalid_fields: List[Tuple[str, str, object]] = [] - - for name, field_type in partner_fields.items(): - value = raw.get(name) - - if value is not None and field_type in validators: - if not validators[field_type](value): - invalid_fields.append( - (name, field_type, value) - ) - - return invalid_fields diff --git a/src/django/requirements.txt b/src/django/requirements.txt index 0d243d69c..4431a3104 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -46,3 +46,4 @@ google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.1 django-bleach==2.0.0 stripe==12.2.0 +jsonschema==4.17.3 From 244dfb0e150aeea0d4b0bbd7d364b58fcd8cdd8d Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 09:37:16 +0200 Subject: [PATCH 02/21] added json formatting and validation for json schema field in admin panel --- src/django/api/admin.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/django/api/admin.py b/src/django/api/admin.py index 56d64a16f..f6c3f0c01 100644 --- a/src/django/api/admin.py +++ b/src/django/api/admin.py @@ -249,13 +249,44 @@ class PartnerFieldAdminForm(forms.ModelForm): required=False, widget=CKEditorWidget() ) + json_schema = forms.CharField( + required=False, + widget=forms.Textarea(attrs={ + 'rows': 20, + 'cols': 100, + 'style': 'font-family: monospace; font-size: 13px; white-space: pre;', + 'class': 'json-editor' + }), + help_text='JSON Schema for validating object type partner fields. Used when type is "object".' + ) class Meta: model = PartnerField fields = ['name', 'type', 'unit', 'label', 'source_by', 'json_schema'] - widgets = { - 'json_schema': forms.Textarea(attrs={'rows': 10, 'cols': 80}), - } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Format JSON for display if it exists + if self.instance and self.instance.pk and self.instance.json_schema: + try: + formatted_json = json.dumps(self.instance.json_schema, indent=2) + self.initial['json_schema'] = formatted_json + except (TypeError, ValueError): + pass + + def clean_json_schema(self): + json_schema = self.cleaned_data.get('json_schema') + if not json_schema: + return None + + # Parse and validate JSON + try: + if isinstance(json_schema, str): + parsed = json.loads(json_schema) + return parsed + return json_schema + except json.JSONDecodeError as e: + raise forms.ValidationError(f'Invalid JSON: {str(e)}') class PartnerFieldAdmin(admin.ModelAdmin): From 751fb5e29fbf3a8b5493b440d83c42b525341dfd Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 10:14:02 +0200 Subject: [PATCH 03/21] fix issue with format validation --- .../processors/partner_type_processor.py | 18 +++++++++++++++--- src/django/requirements.txt | 1 + 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py index f173157ec..72e233aa1 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py @@ -2,6 +2,7 @@ import jsonschema from jsonschema.exceptions import ValidationError as JsonSchemaValidationError +from jsonschema.validators import Draft202012Validator from rest_framework import status from api.moderation_event_actions.creation.location_contribution \ @@ -25,6 +26,8 @@ class PartnerTypeProcessor(ContributionProcessor): ), } + FORMAT_CHECKER = jsonschema.FormatChecker() + def process( self, event_dto: CreateModerationEventDTO @@ -100,12 +103,21 @@ def __validate_partner_fields( ) if use_json_schema: - # Use JSON Schema validation + # Use JSON Schema validation with format checker try: - jsonschema.validate(instance=value, schema=json_schema) + # Use Draft202012Validator which supports format validation + validator = Draft202012Validator( + schema=json_schema, + format_checker=PartnerTypeProcessor.FORMAT_CHECKER + ) + validator.validate(instance=value) except JsonSchemaValidationError as e: error_message = e.message - if e.path: + # Extract error path from the error's absolute_path + if hasattr(e, 'absolute_path') and e.absolute_path: + error_path = ".".join(str(p) for p in e.absolute_path) + error_message = f"{error_path}: {error_message}" + elif hasattr(e, 'path') and e.path: error_path = ".".join(str(p) for p in e.path) error_message = f"{error_path}: {error_message}" validation_errors.append((field_name, error_message)) diff --git a/src/django/requirements.txt b/src/django/requirements.txt index 4431a3104..702fa6aea 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -47,3 +47,4 @@ google-auth-oauthlib==1.2.1 django-bleach==2.0.0 stripe==12.2.0 jsonschema==4.17.3 +rfc3987==1.3.8 From 70f397405e1d839eabbb6e6928db44dc143299cf Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 12:26:21 +0200 Subject: [PATCH 04/21] added django json editor --- src/django/api/admin.py | 33 ++--------------- .../location_contribution.py | 6 ++-- ...sor.py => partner_field_type_processor.py} | 36 +++++-------------- src/django/oar/settings.py | 1 + src/django/requirements.txt | 1 + 5 files changed, 15 insertions(+), 62 deletions(-) rename src/django/api/moderation_event_actions/creation/location_contribution/processors/{partner_type_processor.py => partner_field_type_processor.py} (75%) diff --git a/src/django/api/admin.py b/src/django/api/admin.py index f6c3f0c01..35ca92d24 100644 --- a/src/django/api/admin.py +++ b/src/django/api/admin.py @@ -17,6 +17,7 @@ from waffle.models import Flag, Sample, Switch from waffle.admin import FlagAdmin, SampleAdmin, SwitchAdmin from ckeditor.widgets import CKEditorWidget +from jsoneditor.forms import JSONEditor from api import models @@ -251,42 +252,12 @@ class PartnerFieldAdminForm(forms.ModelForm): ) json_schema = forms.CharField( required=False, - widget=forms.Textarea(attrs={ - 'rows': 20, - 'cols': 100, - 'style': 'font-family: monospace; font-size: 13px; white-space: pre;', - 'class': 'json-editor' - }), - help_text='JSON Schema for validating object type partner fields. Used when type is "object".' + widget=JSONEditor() ) class Meta: model = PartnerField fields = ['name', 'type', 'unit', 'label', 'source_by', 'json_schema'] - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # Format JSON for display if it exists - if self.instance and self.instance.pk and self.instance.json_schema: - try: - formatted_json = json.dumps(self.instance.json_schema, indent=2) - self.initial['json_schema'] = formatted_json - except (TypeError, ValueError): - pass - - def clean_json_schema(self): - json_schema = self.cleaned_data.get('json_schema') - if not json_schema: - return None - - # Parse and validate JSON - try: - if isinstance(json_schema, str): - parsed = json.loads(json_schema) - return parsed - return json_schema - except json.JSONDecodeError as e: - raise forms.ValidationError(f'Invalid JSON: {str(e)}') class PartnerFieldAdmin(admin.ModelAdmin): diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/location_contribution.py b/src/django/api/moderation_event_actions/creation/location_contribution/location_contribution.py index 1302f828e..c45a40419 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/location_contribution.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/location_contribution.py @@ -9,8 +9,8 @@ import ProductionLocationDataProcessor from api.moderation_event_actions.creation.location_contribution \ .processors.geocoding_processor import GeocodingProcessor -from api.moderation_event_actions.creation.location_contribution \ - .processors.partner_type_processor import PartnerTypeProcessor +from api.moderation_event_actions.creation.location_contribution.processors \ + .partner_field_type_processor import PartnerFieldTypeProcessor from api.moderation_event_actions.creation.location_contribution \ .processors.permission_processor import PermissionProcessor from api.moderation_event_actions.creation.dtos.create_moderation_event_dto \ @@ -39,7 +39,7 @@ def serialize( def __setup_location_data_processors() -> ContributionProcessor: location_data_processors = ( PermissionProcessor(), - PartnerTypeProcessor(), + PartnerFieldTypeProcessor(), SourceProcessor(), ProductionLocationDataProcessor(), GeocodingProcessor() diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py similarity index 75% rename from src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py rename to src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py index 72e233aa1..8e89a8773 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_type_processor.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py @@ -13,7 +13,7 @@ from api.constants import APIV1CommonErrorMessages -class PartnerTypeProcessor(ContributionProcessor): +class PartnerFieldTypeProcessor(ContributionProcessor): TYPE_VALIDATORS = { 'int': lambda value: isinstance(value, int) @@ -53,8 +53,6 @@ def process( if not partner_fields_data: return super().process(event_dto) - # Validate all fields: use type validation or JSON Schema validation - # based on field type and json_schema presence validation_errors = self.__validate_partner_fields( raw, partner_fields_data @@ -74,18 +72,7 @@ def __validate_partner_fields( raw: Mapping[str, object], partner_fields_data: Dict[str, Dict] ) -> List[Tuple[str, str]]: - """ - Validates partner fields using either type validation or JSON Schema - validation based on field type and json_schema presence. - - Logic: - - If type is NOT object → use type validation - - If type is object AND json_schema is empty → use type validation - - If type is object AND json_schema is NOT empty → use JSON Schema validation - - Returns a list of tuples (field_name, error_message) for fields - that fail validation. - """ + validation_errors: List[Tuple[str, str]] = [] for field_name, field_info in partner_fields_data.items(): @@ -96,24 +83,21 @@ def __validate_partner_fields( if value is None: continue - # Check if we should use JSON Schema validation - # Only if type is object AND json_schema is not empty + use_json_schema = ( field_type == PartnerField.OBJECT and json_schema ) if use_json_schema: - # Use JSON Schema validation with format checker + # JSON Schema validation try: - # Use Draft202012Validator which supports format validation validator = Draft202012Validator( schema=json_schema, - format_checker=PartnerTypeProcessor.FORMAT_CHECKER + format_checker=PartnerFieldTypeProcessor.FORMAT_CHECKER ) validator.validate(instance=value) except JsonSchemaValidationError as e: error_message = e.message - # Extract error path from the error's absolute_path if hasattr(e, 'absolute_path') and e.absolute_path: error_path = ".".join(str(p) for p in e.absolute_path) error_message = f"{error_path}: {error_message}" @@ -122,14 +106,13 @@ def __validate_partner_fields( error_message = f"{error_path}: {error_message}" validation_errors.append((field_name, error_message)) except Exception as e: - # Handle schema errors (invalid JSON schema itself) validation_errors.append( (field_name, f"Schema validation error: {str(e)}") ) else: - # Use type validation - if field_type in PartnerTypeProcessor.TYPE_VALIDATORS: - validator = PartnerTypeProcessor.TYPE_VALIDATORS[field_type] + # Type validation + if field_type in PartnerFieldTypeProcessor.TYPE_VALIDATORS: + validator = PartnerFieldTypeProcessor.TYPE_VALIDATORS[field_type] if not validator(value): validation_errors.append( ( @@ -145,9 +128,6 @@ def __validate_partner_fields( def __transform_validation_errors( validation_errors: List[Tuple[str, str]] ) -> Dict: - """ - Transforms validation errors into the standard error format. - """ return { 'detail': APIV1CommonErrorMessages.COMMON_REQ_BODY_ERROR, 'errors': [ diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index f4afdaf39..73a4edc18 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -136,6 +136,7 @@ 'ecsmanage', 'django_bleach', 'ckeditor', + 'jsoneditor', ] # For allauth diff --git a/src/django/requirements.txt b/src/django/requirements.txt index 702fa6aea..4f5df6371 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -48,3 +48,4 @@ django-bleach==2.0.0 stripe==12.2.0 jsonschema==4.17.3 rfc3987==1.3.8 +django-jsoneditor==0.2.2 From 3f312e641dc780475fba8c324e4735564a5d0842 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 12:43:08 +0200 Subject: [PATCH 05/21] small fix --- src/django/api/admin.py | 4 +++- src/django/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/django/api/admin.py b/src/django/api/admin.py index 35ca92d24..150bc117e 100644 --- a/src/django/api/admin.py +++ b/src/django/api/admin.py @@ -252,7 +252,9 @@ class PartnerFieldAdminForm(forms.ModelForm): ) json_schema = forms.CharField( required=False, - widget=JSONEditor() + widget=JSONEditor( + init_options={"mode": "code", "modes": ["code", "tree"]}, + ) ) class Meta: diff --git a/src/django/requirements.txt b/src/django/requirements.txt index 4f5df6371..280404c09 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -48,4 +48,4 @@ django-bleach==2.0.0 stripe==12.2.0 jsonschema==4.17.3 rfc3987==1.3.8 -django-jsoneditor==0.2.2 +django-jsoneditor==0.2.4 From 15bf5d97b815f9fdfee1b116b1490dc392106190 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 12:49:05 +0200 Subject: [PATCH 06/21] updated cdn access to cloud front --- deployment/terraform/cdn.tf | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/deployment/terraform/cdn.tf b/deployment/terraform/cdn.tf index 003b14727..976d4579e 100644 --- a/deployment/terraform/cdn.tf +++ b/deployment/terraform/cdn.tf @@ -674,6 +674,50 @@ resource "aws_cloudfront_distribution" "cdn" { max_ttl = 300 } + ordered_cache_behavior { + path_pattern = "/static/jsoneditor/*" + allowed_methods = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"] + cached_methods = ["GET", "HEAD", "OPTIONS"] + target_origin_id = "originAlb" + + forwarded_values { + query_string = true + headers = ["*"] + + cookies { + forward = "all" + } + } + + compress = true + viewer_protocol_policy = "redirect-to-https" + min_ttl = 0 + default_ttl = 0 + max_ttl = 300 + } + + ordered_cache_behavior { + path_pattern = "/static/django-jsoneditor/*" + allowed_methods = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"] + cached_methods = ["GET", "HEAD", "OPTIONS"] + target_origin_id = "originAlb" + + forwarded_values { + query_string = true + headers = ["*"] + + cookies { + forward = "all" + } + } + + compress = true + viewer_protocol_policy = "redirect-to-https" + min_ttl = 0 + default_ttl = 0 + max_ttl = 300 + } + logging_config { include_cookies = false bucket = aws_s3_bucket.logs.bucket_domain_name From 363b2cb534e1a95119838b6d14fb82373482ac48 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 13:43:10 +0200 Subject: [PATCH 07/21] fix linter --- .../processors/partner_field_type_processor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py index 8e89a8773..f6c24e2ba 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py @@ -83,7 +83,6 @@ def __validate_partner_fields( if value is None: continue - use_json_schema = ( field_type == PartnerField.OBJECT and json_schema ) @@ -112,7 +111,9 @@ def __validate_partner_fields( else: # Type validation if field_type in PartnerFieldTypeProcessor.TYPE_VALIDATORS: - validator = PartnerFieldTypeProcessor.TYPE_VALIDATORS[field_type] + validator = PartnerFieldTypeProcessor.TYPE_VALIDATORS[ + field_type + ] if not validator(value): validation_errors.append( ( @@ -138,4 +139,3 @@ def __transform_validation_errors( for field_name, error_message in validation_errors ], } - From 20727bede3ed20c88f580ab65a9861066216e825 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 14:45:50 +0200 Subject: [PATCH 08/21] refactored --- .../partner_field_type_processor.py | 139 ++++++++++++------ 1 file changed, 97 insertions(+), 42 deletions(-) diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py index f6c24e2ba..e6b3190e2 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Mapping, Tuple +from typing import Dict, List, Mapping, Optional, Tuple import jsonschema from jsonschema.exceptions import ValidationError as JsonSchemaValidationError @@ -76,55 +76,110 @@ def __validate_partner_fields( validation_errors: List[Tuple[str, str]] = [] for field_name, field_info in partner_fields_data.items(): - field_type = field_info.get("type") - json_schema = field_info.get("json_schema") value = raw.get(field_name) - if value is None: continue - use_json_schema = ( - field_type == PartnerField.OBJECT and json_schema - ) + field_type = field_info.get("type") + json_schema = field_info.get("json_schema") - if use_json_schema: - # JSON Schema validation - try: - validator = Draft202012Validator( - schema=json_schema, - format_checker=PartnerFieldTypeProcessor.FORMAT_CHECKER - ) - validator.validate(instance=value) - except JsonSchemaValidationError as e: - error_message = e.message - if hasattr(e, 'absolute_path') and e.absolute_path: - error_path = ".".join(str(p) for p in e.absolute_path) - error_message = f"{error_path}: {error_message}" - elif hasattr(e, 'path') and e.path: - error_path = ".".join(str(p) for p in e.path) - error_message = f"{error_path}: {error_message}" - validation_errors.append((field_name, error_message)) - except Exception as e: - validation_errors.append( - (field_name, f"Schema validation error: {str(e)}") - ) - else: - # Type validation - if field_type in PartnerFieldTypeProcessor.TYPE_VALIDATORS: - validator = PartnerFieldTypeProcessor.TYPE_VALIDATORS[ - field_type - ] - if not validator(value): - validation_errors.append( - ( - field_name, - f'Field {field_name} must be {field_type}, ' - f'not {type(value).__name__}.' - ) - ) + error = PartnerFieldTypeProcessor.__validate_single_field( + field_name, value, field_type, json_schema + ) + if error: + validation_errors.append(error) return validation_errors + @staticmethod + def __validate_single_field( + field_name: str, + value: object, + field_type: str, + json_schema: object + ) -> Optional[Tuple[str, str]]: + + use_json_schema = ( + field_type == PartnerField.OBJECT and json_schema + ) + + if use_json_schema: + return PartnerFieldTypeProcessor.__validate_with_json_schema( + field_name, value, json_schema + ) + + return PartnerFieldTypeProcessor.__validate_with_type_validator( + field_name, value, field_type + ) + + @staticmethod + def __validate_with_json_schema( + field_name: str, + value: object, + json_schema: object + ) -> Optional[Tuple[str, str]]: + + try: + validator = Draft202012Validator( + schema=json_schema, + format_checker=PartnerFieldTypeProcessor.FORMAT_CHECKER + ) + validator.validate(instance=value) + return None + except JsonSchemaValidationError as e: + error_message = PartnerFieldTypeProcessor.__format_json_schema_error( + e + ) + return (field_name, error_message) + except Exception as e: + return (field_name, f"Schema validation error: {str(e)}") + + @staticmethod + def __format_json_schema_error( + error: JsonSchemaValidationError + ) -> str: + + error_message = error.message + error_path = PartnerFieldTypeProcessor.__extract_error_path(error) + + if error_path: + return f"{error_path}: {error_message}" + + return error_message + + @staticmethod + def __extract_error_path( + error: JsonSchemaValidationError + ) -> Optional[str]: + + if hasattr(error, 'absolute_path') and error.absolute_path: + return ".".join(str(p) for p in error.absolute_path) + + if hasattr(error, 'path') and error.path: + return ".".join(str(p) for p in error.path) + + return None + + @staticmethod + def __validate_with_type_validator( + field_name: str, + value: object, + field_type: str + ) -> Optional[Tuple[str, str]]: + + if field_type not in PartnerFieldTypeProcessor.TYPE_VALIDATORS: + return None + + validator = PartnerFieldTypeProcessor.TYPE_VALIDATORS[field_type] + if validator(value): + return None + + return ( + field_name, + f'Field {field_name} must be {field_type}, ' + f'not {type(value).__name__}.' + ) + @staticmethod def __transform_validation_errors( validation_errors: List[Tuple[str, str]] From dab95283df0dd67aa559c7e8466067295d9a2c03 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 15:39:18 +0200 Subject: [PATCH 09/21] fix linter --- .../processors/partner_field_type_processor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py index e6b3190e2..5f4a436ea 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py @@ -127,9 +127,10 @@ def __validate_with_json_schema( validator.validate(instance=value) return None except JsonSchemaValidationError as e: - error_message = PartnerFieldTypeProcessor.__format_json_schema_error( - e - ) + error_message = PartnerFieldTypeProcessor \ + .__format_json_schema_error( + e + ) return (field_name, error_message) except Exception as e: return (field_name, f"Schema validation error: {str(e)}") From 122c35aa69776c3082dd843e1465a46918b4bf56 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 16:29:25 +0200 Subject: [PATCH 10/21] possible fix tests --- .../api/migrations/0186_add_json_schema_to_partner_field.py | 2 +- src/django/api/models/partner_field.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/django/api/migrations/0186_add_json_schema_to_partner_field.py b/src/django/api/migrations/0186_add_json_schema_to_partner_field.py index 139b201db..281606421 100644 --- a/src/django/api/migrations/0186_add_json_schema_to_partner_field.py +++ b/src/django/api/migrations/0186_add_json_schema_to_partner_field.py @@ -13,7 +13,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='partnerfield', name='json_schema', - field=models.JSONField(blank=True, help_text='JSON Schema for validating object type partner fields. Used when type is "object".'), + field=models.JSONField(blank=True, null=True, help_text='JSON Schema for validating object type partner fields. Used when type is "object".'), ), ] diff --git a/src/django/api/models/partner_field.py b/src/django/api/models/partner_field.py index d2f27eb18..5474740d7 100644 --- a/src/django/api/models/partner_field.py +++ b/src/django/api/models/partner_field.py @@ -62,6 +62,7 @@ class Meta: json_schema = models.JSONField( blank=True, + null=True, help_text=( 'JSON Schema for validating object type partner fields. ' 'Used when type is "object".' From 23aef81e37bfa438f49cee88122705ae4c611602 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 16:44:11 +0200 Subject: [PATCH 11/21] small UI fixes --- src/django/api/admin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/django/api/admin.py b/src/django/api/admin.py index 150bc117e..5cfe4902b 100644 --- a/src/django/api/admin.py +++ b/src/django/api/admin.py @@ -254,6 +254,9 @@ class PartnerFieldAdminForm(forms.ModelForm): required=False, widget=JSONEditor( init_options={"mode": "code", "modes": ["code", "tree"]}, + attrs={ + 'style': 'width: 100%; height: 400px;' + } ) ) From 36efc2da2071efd86ae3c0c77e0422a1af2ffc5a Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 17:06:39 +0200 Subject: [PATCH 12/21] fix json format for json_schema value --- .../partner_field_type_processor.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py index 5f4a436ea..0bcc2abd0 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py @@ -1,3 +1,4 @@ +import json from typing import Dict, List, Mapping, Optional, Tuple import jsonschema @@ -45,7 +46,10 @@ def process( partner_fields_data: Dict[str, Dict] = { field["name"]: { "type": field["type"], - "json_schema": field["json_schema"] + "json_schema": PartnerFieldTypeProcessor \ + .__parse_json_schema( + field["json_schema"] + ) } for field in partner_fields_qs } @@ -181,6 +185,22 @@ def __validate_with_type_validator( f'not {type(value).__name__}.' ) + @staticmethod + def __parse_json_schema(json_schema: object) -> Optional[dict]: + if json_schema is None: + return None + + if isinstance(json_schema, dict): + return json_schema + + if isinstance(json_schema, str): + try: + return json.loads(json_schema) + except (json.JSONDecodeError, TypeError): + return None + + return None + @staticmethod def __transform_validation_errors( validation_errors: List[Tuple[str, str]] From bfcc2081ebcfd2042788ce2b40917c4b1187e545 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 17:28:03 +0200 Subject: [PATCH 13/21] properly save to db --- src/django/api/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/django/api/admin.py b/src/django/api/admin.py index 5cfe4902b..1272b6568 100644 --- a/src/django/api/admin.py +++ b/src/django/api/admin.py @@ -250,7 +250,7 @@ class PartnerFieldAdminForm(forms.ModelForm): required=False, widget=CKEditorWidget() ) - json_schema = forms.CharField( + json_schema = forms.JSONField( required=False, widget=JSONEditor( init_options={"mode": "code", "modes": ["code", "tree"]}, From 8a029c1c7b6d269d10e5a6fa0d776a209ccb4857 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Tue, 18 Nov 2025 23:28:37 +0200 Subject: [PATCH 14/21] fix linter --- .../processors/partner_field_type_processor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py index 0bcc2abd0..754e864b2 100644 --- a/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py +++ b/src/django/api/moderation_event_actions/creation/location_contribution/processors/partner_field_type_processor.py @@ -46,10 +46,11 @@ def process( partner_fields_data: Dict[str, Dict] = { field["name"]: { "type": field["type"], - "json_schema": PartnerFieldTypeProcessor \ - .__parse_json_schema( + "json_schema": ( + PartnerFieldTypeProcessor.__parse_json_schema( field["json_schema"] ) + ) } for field in partner_fields_qs } From b7e860d9f73df1cdd2da9138dbb9aeab5285d9e0 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Wed, 19 Nov 2025 11:06:42 +0200 Subject: [PATCH 15/21] added unit tests --- .../test_location_contribution_strategy.py | 269 ++++++++++++++++++ 1 file changed, 269 insertions(+) diff --git a/src/django/api/tests/test_location_contribution_strategy.py b/src/django/api/tests/test_location_contribution_strategy.py index 8d4338ec3..14060644a 100644 --- a/src/django/api/tests/test_location_contribution_strategy.py +++ b/src/django/api/tests/test_location_contribution_strategy.py @@ -1872,3 +1872,272 @@ def test_partner_field_object_type_validation(self): 'Field object_field must be object, not str.', detail ) + + def test_partner_field_json_schema_with_valid_data(self): + + existing_location_user_email = 'test7@example.com' + existing_location_user_password = '4567test' + existing_location_user = User.objects.create( + email=existing_location_user_email + ) + existing_location_user.set_password( + existing_location_user_password + ) + existing_location_user.save() + + EmailAddress.objects.create( + user=existing_location_user, + email=existing_location_user_email, + verified=True, + primary=True + ) + + json_schema = { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'age': {'type': 'integer'}, + 'email': { + 'type': 'string', + 'format': 'email' + } + }, + 'required': ['name', 'age'] + } + + schema_field = PartnerField.objects.create( + name='schema_field', + type=PartnerField.OBJECT, + label='Schema Field', + json_schema=json_schema + ) + + existing_location_contributor = Contributor.objects.create( + admin=existing_location_user, + name='test contributor 7', + contrib_type=Contributor.OTHER_CONTRIB_TYPE, + ) + + existing_location_contributor.partner_fields.add(schema_field) + existing_location_contributor.save() + + input_data = { + 'source': 'API', + 'name': 'Blue Horizon Facility', + 'address': '990 Spring Garden St., Philadelphia PA 19123', + 'country': 'US', + 'schema_field': { + 'name': 'John Doe', + 'age': 30, + 'email': 'john@example.com' + } + } + + event_dto = CreateModerationEventDTO( + contributor=existing_location_contributor, + raw_data=input_data, + request_type=ModerationEvent.RequestType.CREATE.value + ) + + result = self.moderation_event_creator.perform_event_creation( + event_dto + ) + self.assertEqual(result.status_code, status.HTTP_202_ACCEPTED) + self.assertIsNotNone(result.moderation_event) + + def test_partner_field_json_schema_with_invalid_data(self): + existing_location_user_email = 'test8@example.com' + existing_location_user_password = '4567test' + existing_location_user = User.objects.create( + email=existing_location_user_email + ) + existing_location_user.set_password( + existing_location_user_password + ) + existing_location_user.save() + + EmailAddress.objects.create( + user=existing_location_user, + email=existing_location_user_email, + verified=True, + primary=True + ) + + json_schema = { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'age': {'type': 'integer'}, + 'email': {'type': 'string', 'format': 'email'} + }, + 'required': ['name', 'age'] + } + + schema_field = PartnerField.objects.create( + name='schema_field', + type=PartnerField.OBJECT, + label='Schema Field', + json_schema=json_schema + ) + + existing_location_contributor = Contributor.objects.create( + admin=existing_location_user, + name='test contributor 8', + contrib_type=Contributor.OTHER_CONTRIB_TYPE, + ) + + existing_location_contributor.partner_fields.add(schema_field) + existing_location_contributor.save() + + input_data_missing_required = { + 'source': 'API', + 'name': 'Blue Horizon Facility', + 'address': '990 Spring Garden St., Philadelphia PA 19123', + 'country': 'US', + 'schema_field': { + 'name': 'John Doe' + } + } + + event_dto = CreateModerationEventDTO( + contributor=existing_location_contributor, + raw_data=input_data_missing_required, + request_type=ModerationEvent.RequestType.CREATE.value + ) + + result = self.moderation_event_creator.perform_event_creation( + event_dto + ) + self.assertEqual( + result.status_code, + status.HTTP_422_UNPROCESSABLE_ENTITY + ) + self.assertIn('errors', result.errors) + self.assertEqual(len(result.errors['errors']), 1) + self.assertEqual(result.errors['errors'][0]['field'], 'schema_field') + self.assertIn('age', result.errors['errors'][0]['detail']) + + input_data_wrong_type = { + 'source': 'API', + 'name': 'Blue Horizon Facility', + 'address': '990 Spring Garden St., Philadelphia PA 19123', + 'country': 'US', + 'schema_field': { + 'name': 123, + 'age': 30 + } + } + + event_dto = CreateModerationEventDTO( + contributor=existing_location_contributor, + raw_data=input_data_wrong_type, + request_type=ModerationEvent.RequestType.CREATE.value + ) + + result = self.moderation_event_creator.perform_event_creation( + event_dto + ) + self.assertEqual( + result.status_code, + status.HTTP_422_UNPROCESSABLE_ENTITY + ) + self.assertIn('errors', result.errors) + self.assertEqual(len(result.errors['errors']), 1) + self.assertEqual(result.errors['errors'][0]['field'], 'schema_field') + self.assertIn('name', result.errors['errors'][0]['detail']) + + def test_partner_field_json_schema_with_nested_errors(self): + existing_location_user_email = 'test9@example.com' + existing_location_user_password = '4567test' + existing_location_user = User.objects.create( + email=existing_location_user_email + ) + existing_location_user.set_password( + existing_location_user_password + ) + existing_location_user.save() + + EmailAddress.objects.create( + user=existing_location_user, + email=existing_location_user_email, + verified=True, + primary=True + ) + + json_schema = { + 'type': 'object', + 'properties': { + 'user': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'contact': { + 'type': 'object', + 'properties': { + 'email': { + 'type': 'string', + 'format': 'email' + } + } + } + } + } + }, + 'required': ['user'] + } + + schema_field = PartnerField.objects.create( + name='nested_schema_field', + type=PartnerField.OBJECT, + label='Nested Schema Field', + json_schema=json_schema + ) + + existing_location_contributor = Contributor.objects.create( + admin=existing_location_user, + name='test contributor 9', + contrib_type=Contributor.OTHER_CONTRIB_TYPE, + ) + + existing_location_contributor.partner_fields.add(schema_field) + existing_location_contributor.save() + + input_data = { + 'source': 'API', + 'name': 'Blue Horizon Facility', + 'address': '990 Spring Garden St., Philadelphia PA 19123', + 'country': 'US', + 'nested_schema_field': { + 'user': { + 'name': 'John Doe', + 'contact': { + 'email': 'invalid-email' + } + } + } + } + + event_dto = CreateModerationEventDTO( + contributor=existing_location_contributor, + raw_data=input_data, + request_type=ModerationEvent.RequestType.CREATE.value + ) + + result = self.moderation_event_creator.perform_event_creation( + event_dto + ) + self.assertEqual( + result.status_code, + status.HTTP_422_UNPROCESSABLE_ENTITY + ) + self.assertIn('errors', result.errors) + self.assertEqual(len(result.errors['errors']), 1) + self.assertEqual( + result.errors['errors'][0]['field'], + 'nested_schema_field' + ) + + detail = result.errors['errors'][0]['detail'] + self.assertTrue( + 'email' in detail.lower() or 'contact' in detail.lower() + ) From 5ced5c779393fd02278a601b8816ae447d850c28 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Wed, 19 Nov 2025 13:32:01 +0200 Subject: [PATCH 16/21] possible fix --- .../test_location_contribution_strategy.py | 87 ++++++++++++++++++- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/src/django/api/tests/test_location_contribution_strategy.py b/src/django/api/tests/test_location_contribution_strategy.py index 14060644a..8f6c9c19a 100644 --- a/src/django/api/tests/test_location_contribution_strategy.py +++ b/src/django/api/tests/test_location_contribution_strategy.py @@ -1921,6 +1921,31 @@ def test_partner_field_json_schema_with_valid_data(self): existing_location_contributor.partner_fields.add(schema_field) existing_location_contributor.save() + list = FacilityList.objects.create( + header='header', file_name='one', name='New List Test' + ) + source = Source.objects.create( + source_type=Source.LIST, + facility_list=list, + contributor=existing_location_contributor + ) + list_item = FacilityListItem.objects.create( + name='Gamma Tech Manufacturing Plant', + address='1574 Quantum Avenue, Building 4B, Technopolis', + country_code='YT', + sector=['Apparel'], + row_index=1, + status=FacilityListItem.CONFIRMED_MATCH, + source=source + ) + production_location = Facility.objects.create( + name=list_item.name, + address=list_item.address, + country_code=list_item.country_code, + location=Point(0, 0), + created_from=list_item + ) + input_data = { 'source': 'API', 'name': 'Blue Horizon Facility', @@ -1936,7 +1961,8 @@ def test_partner_field_json_schema_with_valid_data(self): event_dto = CreateModerationEventDTO( contributor=existing_location_contributor, raw_data=input_data, - request_type=ModerationEvent.RequestType.CREATE.value + request_type=ModerationEvent.RequestType.CREATE.value, + os=production_location ) result = self.moderation_event_creator.perform_event_creation( @@ -1989,6 +2015,31 @@ def test_partner_field_json_schema_with_invalid_data(self): existing_location_contributor.partner_fields.add(schema_field) existing_location_contributor.save() + list = FacilityList.objects.create( + header='header', file_name='one', name='New List Test' + ) + source = Source.objects.create( + source_type=Source.LIST, + facility_list=list, + contributor=existing_location_contributor + ) + list_item = FacilityListItem.objects.create( + name='Gamma Tech Manufacturing Plant', + address='1574 Quantum Avenue, Building 4B, Technopolis', + country_code='YT', + sector=['Apparel'], + row_index=1, + status=FacilityListItem.CONFIRMED_MATCH, + source=source + ) + production_location = Facility.objects.create( + name=list_item.name, + address=list_item.address, + country_code=list_item.country_code, + location=Point(0, 0), + created_from=list_item + ) + input_data_missing_required = { 'source': 'API', 'name': 'Blue Horizon Facility', @@ -2002,7 +2053,8 @@ def test_partner_field_json_schema_with_invalid_data(self): event_dto = CreateModerationEventDTO( contributor=existing_location_contributor, raw_data=input_data_missing_required, - request_type=ModerationEvent.RequestType.CREATE.value + request_type=ModerationEvent.RequestType.CREATE.value, + os=production_location ) result = self.moderation_event_creator.perform_event_creation( @@ -2031,7 +2083,8 @@ def test_partner_field_json_schema_with_invalid_data(self): event_dto = CreateModerationEventDTO( contributor=existing_location_contributor, raw_data=input_data_wrong_type, - request_type=ModerationEvent.RequestType.CREATE.value + request_type=ModerationEvent.RequestType.CREATE.value, + os=production_location ) result = self.moderation_event_creator.perform_event_creation( @@ -2102,6 +2155,31 @@ def test_partner_field_json_schema_with_nested_errors(self): existing_location_contributor.partner_fields.add(schema_field) existing_location_contributor.save() + list = FacilityList.objects.create( + header='header', file_name='one', name='New List Test' + ) + source = Source.objects.create( + source_type=Source.LIST, + facility_list=list, + contributor=existing_location_contributor + ) + list_item = FacilityListItem.objects.create( + name='Gamma Tech Manufacturing Plant', + address='1574 Quantum Avenue, Building 4B, Technopolis', + country_code='YT', + sector=['Apparel'], + row_index=1, + status=FacilityListItem.CONFIRMED_MATCH, + source=source + ) + production_location = Facility.objects.create( + name=list_item.name, + address=list_item.address, + country_code=list_item.country_code, + location=Point(0, 0), + created_from=list_item + ) + input_data = { 'source': 'API', 'name': 'Blue Horizon Facility', @@ -2120,7 +2198,8 @@ def test_partner_field_json_schema_with_nested_errors(self): event_dto = CreateModerationEventDTO( contributor=existing_location_contributor, raw_data=input_data, - request_type=ModerationEvent.RequestType.CREATE.value + request_type=ModerationEvent.RequestType.CREATE.value, + os=production_location ) result = self.moderation_event_creator.perform_event_creation( From 04804928cb7e9ce24d11873bb77aebb1fcee04eb Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Wed, 19 Nov 2025 14:13:16 +0200 Subject: [PATCH 17/21] fix tests --- .../tests/test_location_contribution_strategy.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/django/api/tests/test_location_contribution_strategy.py b/src/django/api/tests/test_location_contribution_strategy.py index 8f6c9c19a..38c1fc0f5 100644 --- a/src/django/api/tests/test_location_contribution_strategy.py +++ b/src/django/api/tests/test_location_contribution_strategy.py @@ -1955,6 +1955,10 @@ def test_partner_field_json_schema_with_valid_data(self): 'name': 'John Doe', 'age': 30, 'email': 'john@example.com' + }, + 'coordinates': { + 'lat': 51.078389, + 'lng': 16.978477 } } @@ -2047,6 +2051,10 @@ def test_partner_field_json_schema_with_invalid_data(self): 'country': 'US', 'schema_field': { 'name': 'John Doe' + }, + 'coordinates': { + 'lat': 51.078389, + 'lng': 16.978477 } } @@ -2077,6 +2085,10 @@ def test_partner_field_json_schema_with_invalid_data(self): 'schema_field': { 'name': 123, 'age': 30 + }, + 'coordinates': { + 'lat': 51.078389, + 'lng': 16.978477 } } @@ -2192,6 +2204,10 @@ def test_partner_field_json_schema_with_nested_errors(self): 'email': 'invalid-email' } } + }, + 'coordinates': { + 'lat': 51.078389, + 'lng': 16.978477 } } From 64794f0f9a185d4dadc611de1c9def88be65ac8a Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Wed, 19 Nov 2025 14:30:58 +0200 Subject: [PATCH 18/21] updated release notes --- doc/release/RELEASE-NOTES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index ef193f8da..e5b613195 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -9,9 +9,15 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * Product name: Open Supply Hub * Release date: November 29, 2025 +### Database changes + +#### Migrations +* 0186_add_json_schema_to_partner_field.py - This migration adds a `json_schema` JSONField to the `PartnerField` model, allowing administrators to define JSON schemas for validating object type partner fields. The field is optional and is used to validate the structure and content of contributed data when the partner field type is `object`. + ### What's new * [OSDEV-2112](https://opensupplyhub.atlassian.net/browse/OSDEV-2112) - Moved "Recruitment Agency" (previously classified as a location type) under the "Office / HQ" location type as a processing type. Also introduced a new processing type, "Union Headquarters/Office", under the "Office / HQ" location type. This update affects both search and newly contributed data: from now on, "Union Headquarters/Office" and "Recruitment Agency" will appear under the "Office / HQ" location type when displayed in search dropdowns or shown on location profiles for **newly** added locations. * [OSDEV-2244](https://opensupplyhub.atlassian.net/browse/OSDEV-2244) - Implemented frontend formatting for the ISIC 4 extended field to display Section, Division, Group, and Class as separate labeled entries, building the full ISIC 4 hierarchy on production location profile pages. +* [OSDEV-2266](https://opensupplyhub.atlassian.net/browse/OSDEV-2266) - Added JSON schema validation support for object type partner fields. Partner fields with object type can now have an associated JSON schema that validates the structure and content of contributed data, providing detailed error messages for validation failures. ### Release instructions * Ensure that the following commands are included in the `post_deployment` command: From 30e0fad9e96e9ebb797cd3178985b127c6bf1288 Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Wed, 19 Nov 2025 14:52:29 +0200 Subject: [PATCH 19/21] remove head issue --- doc/release/RELEASE-NOTES.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index e5b613195..5f72d4664 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -67,11 +67,8 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * [OSDEV-2259](https://opensupplyhub.atlassian.net/browse/OSDEV-2259) - Fixed an issue where the Company Phone field was being saved to the incorrect `office_phone_number` column instead of the `facility_phone_number` when submitting a claim. The Company Phone field now properly stores the value in the correct `facility_phone_number` column in `api_facilityclaim` table. * [OSDEV-2262](https://opensupplyhub.atlassian.net/browse/OSDEV-2262) - Prevented unintended submission of the last-step claim form when pressing Enter in an input while the Submit button is not focused. Updated `src/react/src/components/InitialClaimFlow/ClaimForm/ClaimForm.jsx` to remove implicit form submission and trigger Formik submission explicitly via the Submit button, aligning with Material UI semantics. * [OSDEV-2231](https://opensupplyhub.atlassian.net/browse/OSDEV-2231) - Fixed Django admin panel not displaying location type values for claims when they didn't match the predefined taxonomy. Removed the restrictive `choices` constraint to allow all location types to be visible. -<<<<<<< HEAD * [OSDEV-2260](https://opensupplyhub.atlassian.net/browse/OSDEV-2260) - Removed the unnecessary `facility_type` checks in the `facilities_view_set.py` to support proper saving of the data. The function assumed that the value is an array and that caused errors in the creation of extended fields. On front-end data format has been changed to send array values as a pipe-separated (`|`) string instead of a comma-separated one. The `extended_fields.py` file has been updated to correctly parse this new pipe-delimited format, ensuring data is handled correctly. -======= * [OSDEV-2264](https://opensupplyhub.atlassian.net/browse/OSDEV-2264) - Fixed an issue where the processing type field was being saved to the database (`api_extendedfield` table) as an empty list. Now it's not saved if the list is empty. ->>>>>>> d009243c ([OSDEV-2264] Prevent saving empty lists for facility production types (#805)) ### What's new * [OSDEV-2200](https://opensupplyhub.atlassian.net/browse/OSDEV-2200) - Implements a new claim introduction page for the new facility claiming process, accessible via `/claim/:osId`, which can be enabled or activated through a feature flag. From 640c807175ecd73bcd08ae136a4db43176f6189f Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Thu, 20 Nov 2025 14:19:35 +0200 Subject: [PATCH 20/21] addressed Vadim comments --- deployment/terraform/cdn.tf | 24 +------ doc/release/RELEASE-NOTES.md | 3 + src/django/api/admin.py | 11 +++- .../static/admin/js/partner_field_admin.js | 63 +++++++++++++++++++ 4 files changed, 77 insertions(+), 24 deletions(-) create mode 100644 src/django/api/static/admin/js/partner_field_admin.js diff --git a/deployment/terraform/cdn.tf b/deployment/terraform/cdn.tf index 976d4579e..e9d2375e2 100644 --- a/deployment/terraform/cdn.tf +++ b/deployment/terraform/cdn.tf @@ -675,29 +675,7 @@ resource "aws_cloudfront_distribution" "cdn" { } ordered_cache_behavior { - path_pattern = "/static/jsoneditor/*" - allowed_methods = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"] - cached_methods = ["GET", "HEAD", "OPTIONS"] - target_origin_id = "originAlb" - - forwarded_values { - query_string = true - headers = ["*"] - - cookies { - forward = "all" - } - } - - compress = true - viewer_protocol_policy = "redirect-to-https" - min_ttl = 0 - default_ttl = 0 - max_ttl = 300 - } - - ordered_cache_behavior { - path_pattern = "/static/django-jsoneditor/*" + path_pattern = "/static/*jsoneditor/*" allowed_methods = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"] cached_methods = ["GET", "HEAD", "OPTIONS"] target_origin_id = "originAlb" diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index 5f72d4664..ed0afe175 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -14,6 +14,9 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html #### Migrations * 0186_add_json_schema_to_partner_field.py - This migration adds a `json_schema` JSONField to the `PartnerField` model, allowing administrators to define JSON schemas for validating object type partner fields. The field is optional and is used to validate the structure and content of contributed data when the partner field type is `object`. +#### Schema changes +* [OSDEV-2266](https://opensupplyhub.atlassian.net/browse/OSDEV-2266) - The `PartnerField` model has been updated. Field `json_schema` added with a type JSONField. + ### What's new * [OSDEV-2112](https://opensupplyhub.atlassian.net/browse/OSDEV-2112) - Moved "Recruitment Agency" (previously classified as a location type) under the "Office / HQ" location type as a processing type. Also introduced a new processing type, "Union Headquarters/Office", under the "Office / HQ" location type. This update affects both search and newly contributed data: from now on, "Union Headquarters/Office" and "Recruitment Agency" will appear under the "Office / HQ" location type when displayed in search dropdowns or shown on location profiles for **newly** added locations. * [OSDEV-2244](https://opensupplyhub.atlassian.net/browse/OSDEV-2244) - Implemented frontend formatting for the ISIC 4 extended field to display Section, Division, Group, and Class as separate labeled entries, building the full ISIC 4 hierarchy on production location profile pages. diff --git a/src/django/api/admin.py b/src/django/api/admin.py index 1272b6568..e5e16ec62 100644 --- a/src/django/api/admin.py +++ b/src/django/api/admin.py @@ -255,7 +255,7 @@ class PartnerFieldAdminForm(forms.ModelForm): widget=JSONEditor( init_options={"mode": "code", "modes": ["code", "tree"]}, attrs={ - 'style': 'width: 100%; height: 400px;' + 'style': 'width: 800px; height: 400px;' } ) ) @@ -264,6 +264,9 @@ class Meta: model = PartnerField fields = ['name', 'type', 'unit', 'label', 'source_by', 'json_schema'] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + class PartnerFieldAdmin(admin.ModelAdmin): form = PartnerFieldAdminForm @@ -271,6 +274,12 @@ class PartnerFieldAdmin(admin.ModelAdmin): search_fields = ('name', 'type', 'label', 'unit', 'source_by') readonly_fields = ('uuid', 'created_at', 'updated_at') + class Media: + js = ( + 'admin/js/jquery.init.js', + 'admin/js/partner_field_admin.js', + ) + class EmailAddressAdmin(admin.ModelAdmin): list_display = ('email', 'user', 'primary', 'verified') diff --git a/src/django/api/static/admin/js/partner_field_admin.js b/src/django/api/static/admin/js/partner_field_admin.js new file mode 100644 index 000000000..0b488f423 --- /dev/null +++ b/src/django/api/static/admin/js/partner_field_admin.js @@ -0,0 +1,63 @@ +(function($) { + 'use strict'; + + function findJsonSchemaFieldRow() { + var jsonSchemaField = $('#id_json_schema'); + + if (jsonSchemaField.length) { + return jsonSchemaField.closest('.field-json_schema, tr, .form-row').first(); + } + + return $('.field-json_schema').first(); + } + + function toggleJsonSchemaField() { + var typeField = $('#id_type'); + var jsonSchemaFieldRow = findJsonSchemaFieldRow(); + + if (!typeField.length) { + return false; + } + + if (!jsonSchemaFieldRow.length) { + return false; + } + + var currentType = typeField.val(); + + if (currentType === 'object') { + jsonSchemaFieldRow.show(); + return true; + } else { + jsonSchemaFieldRow.hide(); + return true; + } + } + + function setupJsonSchemaToggle() { + var typeField = $('#id_type'); + var jsonSchemaField = $('#id_json_schema'); + + if (!typeField.length || !jsonSchemaField.length) { + return false; + } + + var jsonSchemaFieldRow = findJsonSchemaFieldRow(); + if (!jsonSchemaFieldRow.length) { + return false; + } + + toggleJsonSchemaField(); + + typeField.on('change', function() { + toggleJsonSchemaField(); + }); + + return true; + } + + $(document).ready(function() { + setupJsonSchemaToggle(); + }); +})(django.jQuery || jQuery); + From e1a581ece9dcdaf0de888f1b09efcd2894bdc9de Mon Sep 17 00:00:00 2001 From: Roman Stolar Date: Thu, 20 Nov 2025 14:27:18 +0200 Subject: [PATCH 21/21] small fix --- .../api/static/admin/js/partner_field_admin.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/django/api/static/admin/js/partner_field_admin.js b/src/django/api/static/admin/js/partner_field_admin.js index 0b488f423..b035c1b6d 100644 --- a/src/django/api/static/admin/js/partner_field_admin.js +++ b/src/django/api/static/admin/js/partner_field_admin.js @@ -2,7 +2,7 @@ 'use strict'; function findJsonSchemaFieldRow() { - var jsonSchemaField = $('#id_json_schema'); + const jsonSchemaField = $('#id_json_schema'); if (jsonSchemaField.length) { return jsonSchemaField.closest('.field-json_schema, tr, .form-row').first(); @@ -12,8 +12,8 @@ } function toggleJsonSchemaField() { - var typeField = $('#id_type'); - var jsonSchemaFieldRow = findJsonSchemaFieldRow(); + const typeField = $('#id_type'); + const jsonSchemaFieldRow = findJsonSchemaFieldRow(); if (!typeField.length) { return false; @@ -23,7 +23,7 @@ return false; } - var currentType = typeField.val(); + const currentType = typeField.val(); if (currentType === 'object') { jsonSchemaFieldRow.show(); @@ -35,14 +35,14 @@ } function setupJsonSchemaToggle() { - var typeField = $('#id_type'); - var jsonSchemaField = $('#id_json_schema'); + const typeField = $('#id_type'); + const jsonSchemaField = $('#id_json_schema'); if (!typeField.length || !jsonSchemaField.length) { return false; } - var jsonSchemaFieldRow = findJsonSchemaFieldRow(); + const jsonSchemaFieldRow = findJsonSchemaFieldRow(); if (!jsonSchemaFieldRow.length) { return false; }