From 006d2e6b6bcadd8f2d5e241e40551dd7fa1b780f Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 14:20:37 +0400 Subject: [PATCH 01/56] Upgrage to Django 4.2.6, Python 3.11 --- src/django/Dockerfile | 7 +- .../user_password_reset_confirm_serializer.py | 2 +- .../user/user_password_reset_serializer.py | 2 +- .../api/views/auth/login_to_oar_client.py | 2 +- .../api/views/auth/logout_to_oar_client.py | 2 +- src/django/oar/settings.py | 6 +- src/django/oar/urls.py | 10 ++- src/django/requirements.txt | 71 ++++++++++--------- 8 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/django/Dockerfile b/src/django/Dockerfile index fabe30129..dbc992e24 100644 --- a/src/django/Dockerfile +++ b/src/django/Dockerfile @@ -1,13 +1,10 @@ -FROM --platform=linux/amd64 python:3.8-slim-bookworm +FROM --platform=linux/amd64 python:3.11-slim-bookworm RUN mkdir -p /usr/local/src/static/static WORKDIR /usr/local/src COPY requirements.txt /usr/local/src/ -# [OSDEV-1785][https://github.com/aio-libs/aiokafka/issues/1093] - Remove aiokafka in requirements.txt to avoid conflicts. -RUN sed -i '/aiokafka/d' requirements.txt - RUN set -ex \ && buildDeps=" \ build-essential \ @@ -24,8 +21,6 @@ RUN set -ex \ " \ && apt-get update && apt-get install -y $buildDeps $deps --no-install-recommends \ && pip install --no-cache-dir -r requirements.txt \ - # [OSDEV-1785][https://github.com/aio-libs/aiokafka/issues/1093] - Install aiokafka from the forked repo. - && pip install --no-cache-dir git+https://github.com/opensupplyhub/aiokafka.git@kafka-python-2.0.3 \ && apt-get purge -y --auto-remove $buildDeps \ && rm -rf /usr/local/src/requirements.txt /var/lib/apt/lists/* diff --git a/src/django/api/serializers/user/user_password_reset_confirm_serializer.py b/src/django/api/serializers/user/user_password_reset_confirm_serializer.py index 567010c9e..bc584ca4f 100644 --- a/src/django/api/serializers/user/user_password_reset_confirm_serializer.py +++ b/src/django/api/serializers/user/user_password_reset_confirm_serializer.py @@ -1,4 +1,4 @@ -from rest_auth.serializers import PasswordResetConfirmSerializer +from dj_rest_auth.serializers import PasswordResetConfirmSerializer from django.db import transaction diff --git a/src/django/api/serializers/user/user_password_reset_serializer.py b/src/django/api/serializers/user/user_password_reset_serializer.py index 69c21d5be..5e6ce96c4 100644 --- a/src/django/api/serializers/user/user_password_reset_serializer.py +++ b/src/django/api/serializers/user/user_password_reset_serializer.py @@ -1,4 +1,4 @@ -from rest_auth.serializers import PasswordResetSerializer +from dj_rest_auth.serializers import PasswordResetSerializer from rest_framework.exceptions import ValidationError from rest_framework.serializers import EmailField from django.conf import settings diff --git a/src/django/api/views/auth/login_to_oar_client.py b/src/django/api/views/auth/login_to_oar_client.py index 501b71567..d8fb4a2b3 100644 --- a/src/django/api/views/auth/login_to_oar_client.py +++ b/src/django/api/views/auth/login_to_oar_client.py @@ -1,7 +1,7 @@ from allauth.account.models import EmailAddress from rest_framework.exceptions import AuthenticationFailed from rest_framework.response import Response -from rest_auth.views import LoginView +from dj_rest_auth.views import LoginView from django.contrib.auth import authenticate, login from ...serializers.user.user_serializer import UserSerializer diff --git a/src/django/api/views/auth/logout_to_oar_client.py b/src/django/api/views/auth/logout_to_oar_client.py index 63fc0a644..8e2b97162 100644 --- a/src/django/api/views/auth/logout_to_oar_client.py +++ b/src/django/api/views/auth/logout_to_oar_client.py @@ -1,6 +1,6 @@ from rest_framework import status from rest_framework.response import Response -from rest_auth.views import LogoutView +from dj_rest_auth.views import LogoutView from django.contrib.auth import logout from ...serializers.user.user_serializer import UserSerializer diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 4d3cd2494..e22240004 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -124,11 +124,11 @@ 'rest_framework.authtoken', 'rest_framework_gis', 'drf_yasg', - 'rest_auth', + 'dj_rest_auth', 'allauth', 'allauth.account', 'allauth.socialaccount', - 'rest_auth.registration', + 'dj_rest_auth.registration', 'watchman', 'simple_history', 'waffle', @@ -210,6 +210,7 @@ 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'allauth.account.middleware.AccountMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', # Clickjacking protection is turned off to allow iframes: # 'django.middleware.clickjacking.XFrameOptionsMiddleware', @@ -582,7 +583,6 @@ ] # CORS_ALLOWED_ORIGIN_REGEXES = json.loads(os.getenv('CORS_ALLOWED_ORIGIN_REGEXES')) -CORS_REPLACE_HTTPS_REFERER = True # django-storages # Reference # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html diff --git a/src/django/oar/urls.py b/src/django/oar/urls.py index c90c6620a..c6706b0fe 100644 --- a/src/django/oar/urls.py +++ b/src/django/oar/urls.py @@ -145,8 +145,8 @@ path('admin/', admin_site.urls), path('health-check/', include('watchman.urls')), path('api-auth/', include('rest_framework.urls')), - path('rest-auth/', include('rest_auth.urls')), - path('rest-auth/registration/', include('rest_auth.registration.urls')), + path('rest-auth/', include('dj_rest_auth.urls')), + path('rest-auth/registration/', include('dj_rest_auth.registration.urls')), path('user-login/', views.LoginToOARClient.as_view(), name='login_to_oar_client'), path('user-logout/', views.LogoutOfOARClient.as_view(), @@ -165,7 +165,11 @@ path('tile/////.', views.get_tile, name='tile'), path('api/current_tile_cache_key/', views.current_tile_cache_key), - path('api-blocks/', views.ApiBlockViewSet, 'api-block'), + path( + 'api-blocks/', + views.ApiBlockViewSet.as_view({'get': 'list'}), + name='api-block', + ), path('api/admin-facility-lists/', views.AdminFacilityListView.as_view(), name='admin-facility-lists'), path('api/geocoder/', views.get_geocoding, name='get_geocoding'), diff --git a/src/django/requirements.txt b/src/django/requirements.txt index 280404c09..a1d2bea5d 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -1,51 +1,52 @@ aiokafka==0.8.0 +kafka-python==2.0.2 base32-crockford==0.3.0 -boto3==1.24.23 +boto3==1.35.24 coreapi==2.3.3 -coverage -debugpy==1.5.1 -defusedxml==0.6.0 +coverage==7.6.1 +debugpy==1.8.5 +defusedxml==0.7.1 django-amazon-ses==4.0.1 -django-cors-headers==3.13.0 +django-cors-headers==4.4.0 django-ecsmanage==2.0.1 django-elasticache==1.0.3 -django-extensions==3.1.4 +django-extensions==3.2.3 django-jsonview==2.0.0 -django-rest-auth[with_social]==0.9.5 -django-simple-history==3.1.1 +dj-rest-auth[with_social]==6.0.0 +django-simple-history==3.4.0 django-spa==0.3.5 -django-storages==1.13.1 -django-waffle==2.5.0 +django-storages==1.14.4 +django-waffle==4.1.0 django-watchman==1.3.0 -drf-yasg==1.20.0 -django-ckeditor==6.5.1 -djangorestframework-gis==1.0.0 -djangorestframework==3.13.1 -flake8==4.0.1 -mccabe==0.6.1 -mercantile==1.1.2 -openpyxl==3.0.9 -pycodestyle==2.8.0 -pyflakes==2.4.0 -pylibmc==1.6.0 -pytz==2023.3 -pymemcache==3.5.2 -requests==2.32.4 +drf-yasg==1.21.7 +django-ckeditor==6.7.1 +djangorestframework-gis==1.1.0 +djangorestframework==3.15.1 +flake8==7.1.0 +mccabe==0.7.0 +mercantile==1.2.1 +openpyxl==3.1.5 +pycodestyle==2.12.1 +pyflakes==3.2.0 +pylibmc==1.6.3 +pytz==2024.1 +pymemcache==4.0.0 +requests==2.32.3 rollbar==0.16.3 -thefuzz==0.19.0 -Unidecode==1.0.23 -django-slowtests -django-allauth==0.54.0 +thefuzz==0.22.1 +Unidecode==1.3.8 +django-slowtests==1.1.1 +django-allauth==0.61.1 opensearch-py==2.8.0 -gunicorn==20.1.0 -django==3.2.17 -psycopg2==2.9.4 -gevent==22.8.0 +gunicorn==22.0.0 +django==4.2.16 +psycopg2==2.9.9 +gevent==24.2.1 google-api-python-client==2.156.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.1 -django-bleach==2.0.0 -stripe==12.2.0 -jsonschema==4.17.3 +django-bleach==3.0.0 +stripe==12.5.0 +jsonschema==4.23.0 rfc3987==1.3.8 django-jsoneditor==0.2.4 From cf97fc8ae45dc55d97adbccd6d7aaa01236ebafe Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 15:18:28 +0400 Subject: [PATCH 02/56] Upgrage to Django 5.1.3 --- src/django/api/apps.py | 10 +++++++++ .../api/migrations/0190_add_event_index.py | 22 +++++++++++++++++++ src/django/api/models/event.py | 7 +++++- src/django/api/processing.py | 2 +- src/django/api/serializers/facility/utils.py | 5 ++--- src/django/api/throttles.py | 2 +- src/django/oar/settings.py | 5 +++++ src/django/requirements.txt | 12 +++++----- 8 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 src/django/api/migrations/0190_add_event_index.py diff --git a/src/django/api/apps.py b/src/django/api/apps.py index 37b3be955..a9c365960 100644 --- a/src/django/api/apps.py +++ b/src/django/api/apps.py @@ -1,9 +1,19 @@ from django.apps import AppConfig +from django.db.models.options import Options class ApiConfig(AppConfig): name = 'api' def ready(self): + names = getattr(Options, 'DEFAULT_NAMES', None) + if names is None: + names = getattr(Options, 'default_names', ()) + if 'index_together' not in names: + new_names = tuple(names) + ('index_together',) + if hasattr(Options, 'DEFAULT_NAMES'): + Options.DEFAULT_NAMES = new_names + else: + Options.default_names = new_names # Implicitly connect signal handlers decorated with @receiver. from api import signals # noqa: F401 diff --git a/src/django/api/migrations/0190_add_event_index.py b/src/django/api/migrations/0190_add_event_index.py new file mode 100644 index 000000000..82622eec6 --- /dev/null +++ b/src/django/api/migrations/0190_add_event_index.py @@ -0,0 +1,22 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0189_add_base_url_and_text_field_to_partner_field"), + ] + + operations = [ + migrations.AlterModelOptions( + name="event", + options={ + "indexes": [ + models.Index( + fields=["content_type", "object_id"], + name="api_event_content_object_idx", + ) + ] + }, + ), + ] + diff --git a/src/django/api/models/event.py b/src/django/api/models/event.py index fdf25f715..19585c106 100644 --- a/src/django/api/models/event.py +++ b/src/django/api/models/event.py @@ -7,7 +7,12 @@ class Event(models.Model): class Meta: - index_together = (('content_type', 'object_id'),) + indexes = [ + models.Index( + fields=['content_type', 'object_id'], + name='api_event_content_object_idx', + ), + ] EVENT_TYPE_CHOICES = ( (FacilityHistoryActions.CREATE, FacilityHistoryActions.CREATE), diff --git a/src/django/api/processing.py b/src/django/api/processing.py index e73e95b2c..66e34bcd0 100644 --- a/src/django/api/processing.py +++ b/src/django/api/processing.py @@ -78,7 +78,7 @@ def parse_production_location_list(location_list: FacilityList): def geocode_facility_list_item(item): started = str(timezone.now()) - if type(item) != FacilityListItem: + if not isinstance(item, FacilityListItem): raise ValueError('Argument must be a FacilityListItem') if item.status == FacilityListItem.ITEM_REMOVED: raise ItemRemovedException() diff --git a/src/django/api/serializers/facility/utils.py b/src/django/api/serializers/facility/utils.py index 1d4cd3941..62916c2e2 100644 --- a/src/django/api/serializers/facility/utils.py +++ b/src/django/api/serializers/facility/utils.py @@ -1,13 +1,12 @@ from typing import (Union) from itertools import groupby +from datetime import timezone as dt_timezone from api.constants import ( FacilityClaimStatuses, FacilitiesQueryParams ) from dateutil import parser -from django.utils import timezone - from ...helpers.helpers import ( cleanup_data, replace_invalid_data, @@ -265,7 +264,7 @@ def format_date(date: str) -> str: original_datetime = parser.isoparse(date) # Convert the datetime object to UTC. - utc_datetime = original_datetime.astimezone(timezone.utc) + utc_datetime = original_datetime.astimezone(dt_timezone.utc) # Format the UTC datetime with the "Z" notation. formatted_datetime_str = utc_datetime.strftime('%Y-%m-%dT%H:%M:%S.%fZ') diff --git a/src/django/api/throttles.py b/src/django/api/throttles.py index 42ef93bda..6cbb30d10 100644 --- a/src/django/api/throttles.py +++ b/src/django/api/throttles.py @@ -11,7 +11,7 @@ class UserCustomRateThrottle(UserRateThrottle): - """Allow custom per-user throttle rates defined on custom Django user model. + """Allow per-user throttle rates defined on the custom user model. `model_rate_field`: Specify the throttle rate field. Required. """ diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index e22240004..7c34f81e2 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -14,6 +14,7 @@ import requests import sys +from django.db.models import options from django.core.exceptions import ImproperlyConfigured from corsheaders.defaults import default_headers @@ -22,6 +23,10 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +# Compatibility: allow legacy `index_together` meta option for older migrations. +if 'index_together' not in options.DEFAULT_NAMES: + options.DEFAULT_NAMES = options.DEFAULT_NAMES + ('index_together',) + # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ diff --git a/src/django/requirements.txt b/src/django/requirements.txt index a1d2bea5d..c6c943981 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -1,7 +1,7 @@ aiokafka==0.8.0 kafka-python==2.0.2 base32-crockford==0.3.0 -boto3==1.35.24 +boto3==1.35.37 coreapi==2.3.3 coverage==7.6.1 debugpy==1.8.5 @@ -12,16 +12,16 @@ django-ecsmanage==2.0.1 django-elasticache==1.0.3 django-extensions==3.2.3 django-jsonview==2.0.0 -dj-rest-auth[with_social]==6.0.0 +dj-rest-auth[with_social]==7.0.1 django-simple-history==3.4.0 django-spa==0.3.5 django-storages==1.14.4 django-waffle==4.1.0 django-watchman==1.3.0 -drf-yasg==1.21.7 +drf-yasg==1.21.8 django-ckeditor==6.7.1 djangorestframework-gis==1.1.0 -djangorestframework==3.15.1 +djangorestframework==3.15.2 flake8==7.1.0 mccabe==0.7.0 mercantile==1.2.1 @@ -36,10 +36,10 @@ rollbar==0.16.3 thefuzz==0.22.1 Unidecode==1.3.8 django-slowtests==1.1.1 -django-allauth==0.61.1 +django-allauth==64.0.0 opensearch-py==2.8.0 gunicorn==22.0.0 -django==4.2.16 +django==5.1.3 psycopg2==2.9.9 gevent==24.2.1 google-api-python-client==2.156.0 From 06e8e960e0f92c47b829927c97de56fc3e4cd050 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 16:09:02 +0400 Subject: [PATCH 03/56] Fix default timezone issue --- src/django/api/close_list.py | 2 +- .../api/limitation/date/monthly_date_limitation.py | 2 +- .../api/limitation/date/yearly_date_limitation.py | 2 +- src/django/api/management/commands/sync_databases.py | 2 +- src/django/api/middleware.py | 2 +- src/django/api/reports.py | 10 ++++++++-- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/django/api/close_list.py b/src/django/api/close_list.py index 15b39bad1..cce2fe210 100644 --- a/src/django/api/close_list.py +++ b/src/django/api/close_list.py @@ -10,7 +10,7 @@ def close_list(list_id, user_id): user = User.objects.get(id=user_id) contributor = Contributor.objects.get(admin=user) - now = datetime.now(tz=timezone.utc) + now = datetime.now(tz=timezone.get_default_timezone()) reason = "Closed via bulk list closure" facilities = Facility.objects.filter( facilitylistitem__source__facility_list_id=list_id) diff --git a/src/django/api/limitation/date/monthly_date_limitation.py b/src/django/api/limitation/date/monthly_date_limitation.py index 0f932da14..277423feb 100644 --- a/src/django/api/limitation/date/monthly_date_limitation.py +++ b/src/django/api/limitation/date/monthly_date_limitation.py @@ -10,7 +10,7 @@ class MonthlyDateLimitation(DateLimitation): def execute(self, period_start_date: datetime): - utc = timezone.utc + utc = timezone.get_default_timezone() self.start_date = period_start_date one_month_in_past = datetime.now(tz=utc) - relativedelta(months=1) diff --git a/src/django/api/limitation/date/yearly_date_limitation.py b/src/django/api/limitation/date/yearly_date_limitation.py index 6a597b423..8c369da59 100644 --- a/src/django/api/limitation/date/yearly_date_limitation.py +++ b/src/django/api/limitation/date/yearly_date_limitation.py @@ -10,7 +10,7 @@ class YearlyDateLimitation(DateLimitation): def execute(self, period_start_date: datetime): - utc = timezone.utc + utc = timezone.get_default_timezone() self.start_date = period_start_date one_year_in_past = datetime.now(tz=utc) - relativedelta(years=1) diff --git a/src/django/api/management/commands/sync_databases.py b/src/django/api/management/commands/sync_databases.py index 7bf34c142..534d72849 100644 --- a/src/django/api/management/commands/sync_databases.py +++ b/src/django/api/management/commands/sync_databases.py @@ -965,7 +965,7 @@ def __get_last_run_timestamp(self, model_name, suffix=''): f'for {model_name}: {e}.') # Return a very old date if no last run file exists. - return datetime(1970, 1, 1, tzinfo=timezone.utc) + return datetime(1970, 1, 1, tzinfo=timezone.get_default_timezone()) def __save_last_run_timestamp(self, model_name, timestamp, suffix=''): '''Save the last run timestamp for a specific model.''' diff --git a/src/django/api/middleware.py b/src/django/api/middleware.py index cb8999657..1056b4096 100644 --- a/src/django/api/middleware.py +++ b/src/django/api/middleware.py @@ -78,7 +78,7 @@ def has_active_block(request): contributor = token.user.contributor apiBlock = get_api_block(contributor) - at_datetime = datetime.datetime.now(tz=timezone.utc) + at_datetime = datetime.datetime.now(tz=timezone.get_default_timezone()) return (apiBlock is not None and apiBlock.until > at_datetime and apiBlock.active) except ObjectDoesNotExist: diff --git a/src/django/api/reports.py b/src/django/api/reports.py index 842ca9eb2..58ff39f55 100644 --- a/src/django/api/reports.py +++ b/src/django/api/reports.py @@ -50,7 +50,10 @@ def monthly_promoted_name_and_address(): def geocoding_time_without_queue(date_format): temp_data = dict() - start_date = datetime.now(tz=timezone.utc) - relativedelta(months=4) + start_date = ( + datetime.now(tz=timezone.get_default_timezone()) + - relativedelta(months=4) + ) processing_results = FacilityListItem.objects.filter( created_at__gte=start_date, processing_results__contains=[{"action": "geocode"}] @@ -89,7 +92,10 @@ def weekly_geocoding_time_without_queue(): def geocoding_time_with_queue(date_format): temp_data = dict() - start_date = datetime.now(tz=timezone.utc) - relativedelta(months=4) + start_date = ( + datetime.now(tz=timezone.get_default_timezone()) + - relativedelta(months=4) + ) listitems = FacilityListItem.objects.filter( created_at__gte=start_date, processing_results__contains=[{"action": "geocode"}] From b5cea92fe8bfad582e2c8ce3f9e478832f6bd568 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 16:26:44 +0400 Subject: [PATCH 04/56] Migrate to django-ckeditor-5 --- src/django/api/admin.py | 4 ++-- ...1_switch_partner_field_source_by_editor.py | 22 +++++++++++++++++++ src/django/api/models/partner_field.py | 5 ++--- src/django/ckeditor/__init__.py | 2 ++ src/django/ckeditor/fields.py | 11 ++++++++++ src/django/ckeditor/widgets.py | 11 ++++++++++ src/django/oar/settings.py | 12 +++++++++- src/django/requirements.txt | 2 +- 8 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 src/django/api/migrations/0191_switch_partner_field_source_by_editor.py create mode 100644 src/django/ckeditor/__init__.py create mode 100644 src/django/ckeditor/fields.py create mode 100644 src/django/ckeditor/widgets.py diff --git a/src/django/api/admin.py b/src/django/api/admin.py index 9efb67406..dfb348297 100644 --- a/src/django/api/admin.py +++ b/src/django/api/admin.py @@ -22,7 +22,7 @@ from simple_history.admin import SimpleHistoryAdmin from waffle.models import Flag, Sample, Switch from waffle.admin import FlagAdmin, SampleAdmin, SwitchAdmin -from ckeditor.widgets import CKEditorWidget +from django_ckeditor_5.widgets import CKEditor5Widget from jsoneditor.forms import JSONEditor from api import models @@ -256,7 +256,7 @@ def get_ordering(self, request): class PartnerFieldAdminForm(forms.ModelForm): source_by = forms.CharField( required=False, - widget=CKEditorWidget() + widget=CKEditor5Widget(config_name='default') ) json_schema = forms.JSONField( required=False, diff --git a/src/django/api/migrations/0191_switch_partner_field_source_by_editor.py b/src/django/api/migrations/0191_switch_partner_field_source_by_editor.py new file mode 100644 index 000000000..16d316032 --- /dev/null +++ b/src/django/api/migrations/0191_switch_partner_field_source_by_editor.py @@ -0,0 +1,22 @@ +from django.db import migrations +import django_ckeditor_5.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0190_add_event_index'), + ] + + operations = [ + migrations.AlterField( + model_name='partnerfield', + name='source_by', + field=django_ckeditor_5.fields.CKEditor5Field( + blank=True, + help_text='Rich text field describing the source of this partner field.', + null=True, + ), + ), + ] + diff --git a/src/django/api/models/partner_field.py b/src/django/api/models/partner_field.py index 639bc157c..3cf6adcd9 100644 --- a/src/django/api/models/partner_field.py +++ b/src/django/api/models/partner_field.py @@ -3,7 +3,7 @@ from django.db import models from django.core.cache import cache -from ckeditor.fields import RichTextField +from django_ckeditor_5.fields import CKEditor5Field from api.constants import PARTNER_FIELD_LIST_KEY, PARTNER_FIELD_NAMES_KEY from api.models.partner_field_manager import PartnerFieldManager @@ -68,10 +68,9 @@ class Meta: max_length=500, blank=True, ) - source_by = RichTextField( + source_by = CKEditor5Field( blank=True, null=True, - config_name="default", help_text=( "Rich text field describing the " "source of this partner field." diff --git a/src/django/ckeditor/__init__.py b/src/django/ckeditor/__init__.py new file mode 100644 index 000000000..262beb8ab --- /dev/null +++ b/src/django/ckeditor/__init__.py @@ -0,0 +1,2 @@ +"""Compatibility shim for legacy migrations that import `ckeditor`.""" + diff --git a/src/django/ckeditor/fields.py b/src/django/ckeditor/fields.py new file mode 100644 index 000000000..1b77c8b0a --- /dev/null +++ b/src/django/ckeditor/fields.py @@ -0,0 +1,11 @@ +from django.db import models + + +class RichTextField(models.TextField): + """Minimal stub to satisfy legacy migrations.""" + + def __init__(self, *args, **kwargs): + # Legacy migrations pass config_name; ignore it for compatibility. + kwargs.pop("config_name", None) + super().__init__(*args, **kwargs) + diff --git a/src/django/ckeditor/widgets.py b/src/django/ckeditor/widgets.py new file mode 100644 index 000000000..927b1d2e5 --- /dev/null +++ b/src/django/ckeditor/widgets.py @@ -0,0 +1,11 @@ +from django.forms import Textarea + + +class CKEditorWidget(Textarea): + """Minimal stub to satisfy any legacy imports.""" + + def __init__(self, *args, **kwargs): + # Ignore legacy config_name argument if provided. + kwargs.pop("config_name", None) + super().__init__(*args, **kwargs) + diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 7c34f81e2..220be12d8 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -33,6 +33,16 @@ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'secret') +CKEDITOR_5_CONFIGS = { + 'default': { + 'toolbar': [ + 'heading', '|', + 'bold', 'italic', 'link', 'bulletedList', 'numberedList', + 'blockQuote', + ], + }, +} + # Set environment ENVIRONMENT = os.getenv('DJANGO_ENV', 'Local') @@ -141,7 +151,7 @@ 'web', 'ecsmanage', 'django_bleach', - 'ckeditor', + 'django_ckeditor_5', 'jsoneditor', ] diff --git a/src/django/requirements.txt b/src/django/requirements.txt index c6c943981..5423068e4 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -19,7 +19,7 @@ django-storages==1.14.4 django-waffle==4.1.0 django-watchman==1.3.0 drf-yasg==1.21.8 -django-ckeditor==6.7.1 +django-ckeditor-5==0.2.13 djangorestframework-gis==1.1.0 djangorestframework==3.15.2 flake8==7.1.0 From 99368bc6e1eef943a02c48fe6a703e075685d8ed Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 16:31:26 +0400 Subject: [PATCH 05/56] Consolidate duplicate index_together compatibility workarounds --- src/django/api/apps.py | 10 ---------- src/django/oar/settings.py | 3 ++- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/django/api/apps.py b/src/django/api/apps.py index a9c365960..37b3be955 100644 --- a/src/django/api/apps.py +++ b/src/django/api/apps.py @@ -1,19 +1,9 @@ from django.apps import AppConfig -from django.db.models.options import Options class ApiConfig(AppConfig): name = 'api' def ready(self): - names = getattr(Options, 'DEFAULT_NAMES', None) - if names is None: - names = getattr(Options, 'default_names', ()) - if 'index_together' not in names: - new_names = tuple(names) + ('index_together',) - if hasattr(Options, 'DEFAULT_NAMES'): - Options.DEFAULT_NAMES = new_names - else: - Options.default_names = new_names # Implicitly connect signal handlers decorated with @receiver. from api import signals # noqa: F401 diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 220be12d8..a3295b159 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -23,7 +23,8 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -# Compatibility: allow legacy `index_together` meta option for older migrations. +# Compatibility: allow legacy `index_together` meta option so immutable older +# migrations (e.g., Django<5 era) continue to load without edits. if 'index_together' not in options.DEFAULT_NAMES: options.DEFAULT_NAMES = options.DEFAULT_NAMES + ('index_together',) From f2370ba8afd64830bbfb2a11642d5a608f973bdc Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 16:35:36 +0400 Subject: [PATCH 06/56] Fix api-block route name collision --- src/django/oar/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/django/oar/urls.py b/src/django/oar/urls.py index c6706b0fe..28a04b7a0 100644 --- a/src/django/oar/urls.py +++ b/src/django/oar/urls.py @@ -168,7 +168,7 @@ path( 'api-blocks/', views.ApiBlockViewSet.as_view({'get': 'list'}), - name='api-block', + name='api-block-internal', ), path('api/admin-facility-lists/', views.AdminFacilityListView.as_view(), name='admin-facility-lists'), From 2059c648573c1a8401f56286aa925c7223514da1 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 16:40:24 +0400 Subject: [PATCH 07/56] Use proper utc timezone --- .../date/monthly_date_limitation.py | 19 +++++++++++++------ .../limitation/date/yearly_date_limitation.py | 4 ++-- .../api/management/commands/sync_databases.py | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/django/api/limitation/date/monthly_date_limitation.py b/src/django/api/limitation/date/monthly_date_limitation.py index 277423feb..5322d042d 100644 --- a/src/django/api/limitation/date/monthly_date_limitation.py +++ b/src/django/api/limitation/date/monthly_date_limitation.py @@ -10,12 +10,19 @@ class MonthlyDateLimitation(DateLimitation): def execute(self, period_start_date: datetime): - utc = timezone.get_default_timezone() - self.start_date = period_start_date - - one_month_in_past = datetime.now(tz=utc) - relativedelta(months=1) - while (self.start_date < one_month_in_past): - self.start_date = self.start_date + relativedelta(months=1) + default_tz = timezone.get_default_timezone() + start_date = period_start_date + if timezone.is_naive(start_date): + start_date = start_date.replace(tzinfo=default_tz) + start_date_utc = start_date.astimezone(timezone.utc) + + one_month_ago_utc = ( + datetime.now(tz=timezone.utc) - relativedelta(months=1) + ) + while start_date_utc < one_month_ago_utc: + start_date_utc = start_date_utc + relativedelta(months=1) + + self.start_date = start_date_utc return self diff --git a/src/django/api/limitation/date/yearly_date_limitation.py b/src/django/api/limitation/date/yearly_date_limitation.py index 8c369da59..8497be4a0 100644 --- a/src/django/api/limitation/date/yearly_date_limitation.py +++ b/src/django/api/limitation/date/yearly_date_limitation.py @@ -10,10 +10,10 @@ class YearlyDateLimitation(DateLimitation): def execute(self, period_start_date: datetime): - utc = timezone.get_default_timezone() + default_tz = timezone.get_default_timezone() self.start_date = period_start_date - one_year_in_past = datetime.now(tz=utc) - relativedelta(years=1) + one_year_in_past = datetime.now(tz=default_tz) - relativedelta(years=1) while (self.start_date < one_year_in_past): self.start_date = self.start_date + relativedelta(years=1) diff --git a/src/django/api/management/commands/sync_databases.py b/src/django/api/management/commands/sync_databases.py index 534d72849..7bf34c142 100644 --- a/src/django/api/management/commands/sync_databases.py +++ b/src/django/api/management/commands/sync_databases.py @@ -965,7 +965,7 @@ def __get_last_run_timestamp(self, model_name, suffix=''): f'for {model_name}: {e}.') # Return a very old date if no last run file exists. - return datetime(1970, 1, 1, tzinfo=timezone.get_default_timezone()) + return datetime(1970, 1, 1, tzinfo=timezone.utc) def __save_last_run_timestamp(self, model_name, timestamp, suffix=''): '''Save the last run timestamp for a specific model.''' From 19e7eb10f5d827fc38edb7481e5a40be1cd374ab Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 16:47:29 +0400 Subject: [PATCH 08/56] Fix flake8 issues --- src/django/ckeditor/__init__.py | 1 - src/django/ckeditor/fields.py | 1 - src/django/ckeditor/widgets.py | 1 - 3 files changed, 3 deletions(-) diff --git a/src/django/ckeditor/__init__.py b/src/django/ckeditor/__init__.py index 262beb8ab..516a596ca 100644 --- a/src/django/ckeditor/__init__.py +++ b/src/django/ckeditor/__init__.py @@ -1,2 +1 @@ """Compatibility shim for legacy migrations that import `ckeditor`.""" - diff --git a/src/django/ckeditor/fields.py b/src/django/ckeditor/fields.py index 1b77c8b0a..29b8db33e 100644 --- a/src/django/ckeditor/fields.py +++ b/src/django/ckeditor/fields.py @@ -8,4 +8,3 @@ def __init__(self, *args, **kwargs): # Legacy migrations pass config_name; ignore it for compatibility. kwargs.pop("config_name", None) super().__init__(*args, **kwargs) - diff --git a/src/django/ckeditor/widgets.py b/src/django/ckeditor/widgets.py index 927b1d2e5..dd598a974 100644 --- a/src/django/ckeditor/widgets.py +++ b/src/django/ckeditor/widgets.py @@ -8,4 +8,3 @@ def __init__(self, *args, **kwargs): # Ignore legacy config_name argument if provided. kwargs.pop("config_name", None) super().__init__(*args, **kwargs) - From 8212503b96baeda4de8b098ced81cb21c5706597 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 17:09:44 +0400 Subject: [PATCH 09/56] Use timezone.make_aware() instead of replace(tzinfo=...) --- src/django/api/limitation/date/monthly_date_limitation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/django/api/limitation/date/monthly_date_limitation.py b/src/django/api/limitation/date/monthly_date_limitation.py index 5322d042d..ae4390db2 100644 --- a/src/django/api/limitation/date/monthly_date_limitation.py +++ b/src/django/api/limitation/date/monthly_date_limitation.py @@ -13,7 +13,7 @@ def execute(self, period_start_date: datetime): default_tz = timezone.get_default_timezone() start_date = period_start_date if timezone.is_naive(start_date): - start_date = start_date.replace(tzinfo=default_tz) + start_date = timezone.make_aware(start_date, default_tz) start_date_utc = start_date.astimezone(timezone.utc) one_month_ago_utc = ( From ab75ffd1a06a92e09a73091ef111256c8ee29c82 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 17:14:21 +0400 Subject: [PATCH 10/56] Remove the last timezone.utc usage --- src/django/api/limitation/date/monthly_date_limitation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/django/api/limitation/date/monthly_date_limitation.py b/src/django/api/limitation/date/monthly_date_limitation.py index ae4390db2..5120bc6f9 100644 --- a/src/django/api/limitation/date/monthly_date_limitation.py +++ b/src/django/api/limitation/date/monthly_date_limitation.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone as dt_timezone from dateutil.relativedelta import relativedelta from django.utils import timezone @@ -14,10 +14,10 @@ def execute(self, period_start_date: datetime): start_date = period_start_date if timezone.is_naive(start_date): start_date = timezone.make_aware(start_date, default_tz) - start_date_utc = start_date.astimezone(timezone.utc) + start_date_utc = start_date.astimezone(dt_timezone.utc) one_month_ago_utc = ( - datetime.now(tz=timezone.utc) - relativedelta(months=1) + datetime.now(tz=dt_timezone.utc) - relativedelta(months=1) ) while start_date_utc < one_month_ago_utc: start_date_utc = start_date_utc + relativedelta(months=1) From b040ec58ca44e4ab16679a94ee7f7ee357cc2f6a Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 17:31:11 +0400 Subject: [PATCH 11/56] Update release notes --- doc/release/RELEASE-NOTES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index 1e2f03e04..a64afd65c 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -48,6 +48,10 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * Removed the FE request to `GET /api/parent-companies/`. * Refactored `Parent Company / Supplier Group` to act as a regular text input field (not a dropdown). Prepopulate only value that has been assisgned to a particular claim. * [OSDEV-2295](https://opensupplyhub.atlassian.net/browse/OSDEV-2295) - UI: added divider between each record of the `isic-4` field from the same contribution. +* [OSDEV-814](https://opensupplyhub.atlassian.net/browse/OSDEV-814) - Major upgrade of Django application backend services: + * Upgraded Python from `3.8` to `3.11`. + * Upgraded Django from `3.2.17` to `5.1.3`. + * Upgraded Python and Django packages to conform compatibility. ### Release instructions * Ensure that the following commands are included in the `post_deployment` command: From 6bcb0feb1cb51e2c1e2eb2fe3fcb47ce92007bcf Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 17:35:09 +0400 Subject: [PATCH 12/56] Upd Release notes --- doc/release/RELEASE-NOTES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index a64afd65c..4d8673472 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -34,6 +34,10 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * [OSDEV-2047](https://opensupplyhub.atlassian.net/browse/OSDEV-2047) - Removed all Terraform configurations and ECS service definitions related to the deprecated standalone ContriCleaner service. Cleaned up the repository by deleting unused code and references, as ContriCleaner now operates exclusively as an internal Django library. * [OSDEV-2318](https://opensupplyhub.atlassian.net/browse/OSDEV-2318) - Updated Terraform version from `1.5` to `1.13.3`. Upgraded Kafka from the `3.4.0` to `3.9.0` to align with the current AWS MSK supported version. * [OSDEV-2328](https://opensupplyhub.atlassian.net/browse/OSDEV-2328) - Added CloudFront caching for the facilities and production-location OS ID endpoints, refactored the Terraform config to use endpoint-specific TTL variables, and set per-environment durations (30 minutes for Prod/RBA/Preprod/Staging, 1 minute for Dev/Test). CloudFront still caches only GET/HEAD/OPTIONS while allowing all HTTP methods to reach the origin. +* [OSDEV-814](https://opensupplyhub.atlassian.net/browse/OSDEV-814) - Major upgrade of Django application backend services: + * Upgraded Python from `3.8` to `3.11`. + * Upgraded Django from `3.2.17` to `5.1.3`. + * Upgraded Python and Django packages to conform compatibility. ### Bugfix * [OSDEV-2047](https://opensupplyhub.atlassian.net/browse/OSDEV-2047) - Previously, there were two security groups with the same tags: one for the Django app and another for ContriCleaner. After removing the ContriCleaner service infrastructure, a bug was eliminated in which the Django CLI task in the Development environment selected the wrong security group - the one without database access, belonging to ContriCleaner - which prevented Django management commands from running against the database in the Development environment. @@ -48,10 +52,6 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * Removed the FE request to `GET /api/parent-companies/`. * Refactored `Parent Company / Supplier Group` to act as a regular text input field (not a dropdown). Prepopulate only value that has been assisgned to a particular claim. * [OSDEV-2295](https://opensupplyhub.atlassian.net/browse/OSDEV-2295) - UI: added divider between each record of the `isic-4` field from the same contribution. -* [OSDEV-814](https://opensupplyhub.atlassian.net/browse/OSDEV-814) - Major upgrade of Django application backend services: - * Upgraded Python from `3.8` to `3.11`. - * Upgraded Django from `3.2.17` to `5.1.3`. - * Upgraded Python and Django packages to conform compatibility. ### Release instructions * Ensure that the following commands are included in the `post_deployment` command: From 255dc05fa0e555b39b54e63bc616f18d0dad9fd7 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 24 Dec 2025 17:41:05 +0400 Subject: [PATCH 13/56] Refine phrase in release notes --- doc/release/RELEASE-NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index 4d8673472..d49370fc3 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -37,7 +37,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * [OSDEV-814](https://opensupplyhub.atlassian.net/browse/OSDEV-814) - Major upgrade of Django application backend services: * Upgraded Python from `3.8` to `3.11`. * Upgraded Django from `3.2.17` to `5.1.3`. - * Upgraded Python and Django packages to conform compatibility. + * Upgraded Python and Django packages to maintain compatibility. ### Bugfix * [OSDEV-2047](https://opensupplyhub.atlassian.net/browse/OSDEV-2047) - Previously, there were two security groups with the same tags: one for the Django app and another for ContriCleaner. After removing the ContriCleaner service infrastructure, a bug was eliminated in which the Django CLI task in the Development environment selected the wrong security group - the one without database access, belonging to ContriCleaner - which prevented Django management commands from running against the database in the Development environment. From 59a16b5fd02d3bb75e38f57edd8fc1f710627d0d Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Mon, 29 Dec 2025 10:23:07 +0400 Subject: [PATCH 14/56] Update migrations graph --- ...editor.py => 0194_switch_partner_field_source_by_editor.py} | 2 +- .../{0190_add_event_index.py => 0195_add_event_index.py} | 2 +- src/django/api/models/partner_field.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) rename src/django/api/migrations/{0191_switch_partner_field_source_by_editor.py => 0194_switch_partner_field_source_by_editor.py} (90%) rename src/django/api/migrations/{0190_add_event_index.py => 0195_add_event_index.py} (87%) diff --git a/src/django/api/migrations/0191_switch_partner_field_source_by_editor.py b/src/django/api/migrations/0194_switch_partner_field_source_by_editor.py similarity index 90% rename from src/django/api/migrations/0191_switch_partner_field_source_by_editor.py rename to src/django/api/migrations/0194_switch_partner_field_source_by_editor.py index 16d316032..ba0bf10d7 100644 --- a/src/django/api/migrations/0191_switch_partner_field_source_by_editor.py +++ b/src/django/api/migrations/0194_switch_partner_field_source_by_editor.py @@ -5,7 +5,7 @@ class Migration(migrations.Migration): dependencies = [ - ('api', '0190_add_event_index'), + ('api', '0193_populate_wage_indicator_data'), ] operations = [ diff --git a/src/django/api/migrations/0190_add_event_index.py b/src/django/api/migrations/0195_add_event_index.py similarity index 87% rename from src/django/api/migrations/0190_add_event_index.py rename to src/django/api/migrations/0195_add_event_index.py index 82622eec6..ff5c62c6b 100644 --- a/src/django/api/migrations/0190_add_event_index.py +++ b/src/django/api/migrations/0195_add_event_index.py @@ -3,7 +3,7 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0189_add_base_url_and_text_field_to_partner_field"), + ("api", "0194_switch_partner_field_source_by_editor"), ] operations = [ diff --git a/src/django/api/models/partner_field.py b/src/django/api/models/partner_field.py index 3cf6adcd9..120bbaba1 100644 --- a/src/django/api/models/partner_field.py +++ b/src/django/api/models/partner_field.py @@ -72,8 +72,7 @@ class Meta: blank=True, null=True, help_text=( - "Rich text field describing the " - "source of this partner field." + "Rich text field describing the source of this partner field." ), ) json_schema = models.JSONField( From 2f010cea5550100bf80fb9d4c71c45e93dda6302 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Mon, 29 Dec 2025 15:06:38 +0400 Subject: [PATCH 15/56] Switch the throttling cache backend to django.core.cache.backends.memcached.PyLibMCCache for all environments --- src/django/oar/settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index a3295b159..97f14dff9 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -306,10 +306,10 @@ # https://docs.djangoproject.com/en/3.2/topics/cache/ MEMCACHED_LOCATION = f"{os.getenv('CACHE_HOST')}:{os.getenv('CACHE_PORT')}" -if DEBUG: - CACHE_BACKEND = 'django.core.cache.backends.memcached.PyLibMCCache' -else: - CACHE_BACKEND = 'django_elasticache.memcached.ElastiCache' +# Use PyLibMCCache everywhere; django_elasticache is incompatible with Django 4 +# because it still imports smart_text. This keeps throttling cache working in +# jobs/containers that upgrade Django. +CACHE_BACKEND = 'django.core.cache.backends.memcached.PyLibMCCache' CACHES = { 'default': { From 7d076f978c716f78e5b55856602b9781019f9fad Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Mon, 29 Dec 2025 18:05:54 +0400 Subject: [PATCH 16/56] Add explicit logging --- src/django/contricleaner/lib/parsers/source_parser_xlsx.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/django/contricleaner/lib/parsers/source_parser_xlsx.py b/src/django/contricleaner/lib/parsers/source_parser_xlsx.py index e1a8ef704..b118a69e4 100644 --- a/src/django/contricleaner/lib/parsers/source_parser_xlsx.py +++ b/src/django/contricleaner/lib/parsers/source_parser_xlsx.py @@ -9,6 +9,10 @@ from contricleaner.lib.parsers.abstractions.file_parser import FileParser from contricleaner.lib.exceptions.parsing_error import ParsingError +import logging +import traceback +logger = logging.getLogger(__name__) + class SourceParserXLSX(SourceParser, FileParser): def get_parsed_rows(self) -> List[dict]: @@ -47,6 +51,7 @@ def _parse(file: File) -> List[dict]: return rows except Exception: + logger.error(f"Error parsing XLSX file: {traceback.format_exc()}") raise ParsingError( 'There was an error within your file and our team needs to ' 'take a look. Please send your file to ' From 905ffce00f5e0a93311633da9f741cb1a5cc6659 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Tue, 30 Dec 2025 10:45:46 +0400 Subject: [PATCH 17/56] Fix migration graph --- ...rline.py => 0196_create_and_populate_us_county_tigerline.py} | 2 +- ...artner_field.py => 0197_add_mit_livingwage_partner_field.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/django/api/migrations/{0194_create_and_populate_us_county_tigerline.py => 0196_create_and_populate_us_county_tigerline.py} (97%) rename src/django/api/migrations/{0195_add_mit_livingwage_partner_field.py => 0197_add_mit_livingwage_partner_field.py} (96%) diff --git a/src/django/api/migrations/0194_create_and_populate_us_county_tigerline.py b/src/django/api/migrations/0196_create_and_populate_us_county_tigerline.py similarity index 97% rename from src/django/api/migrations/0194_create_and_populate_us_county_tigerline.py rename to src/django/api/migrations/0196_create_and_populate_us_county_tigerline.py index bda053d45..dab4d1248 100644 --- a/src/django/api/migrations/0194_create_and_populate_us_county_tigerline.py +++ b/src/django/api/migrations/0196_create_and_populate_us_county_tigerline.py @@ -27,7 +27,7 @@ def populate_tigerline_data_wrapper(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('api', '0193_populate_wage_indicator_data'), + ('api', '0195_add_event_index'), ] operations = [ diff --git a/src/django/api/migrations/0195_add_mit_livingwage_partner_field.py b/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py similarity index 96% rename from src/django/api/migrations/0195_add_mit_livingwage_partner_field.py rename to src/django/api/migrations/0197_add_mit_livingwage_partner_field.py index 36ce33ab1..3430281b9 100644 --- a/src/django/api/migrations/0195_add_mit_livingwage_partner_field.py +++ b/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py @@ -49,7 +49,7 @@ def remove_mit_living_wage_partner_field(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('api', '0194_create_and_populate_us_county_tigerline'), + ('api', '0196_create_and_populate_us_county_tigerline'), ] operations = [ From 6c7d60de28c7c4f70b89f3967cfd3a2f1593f71c Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Tue, 30 Dec 2025 12:05:42 +0400 Subject: [PATCH 18/56] Tighten the storage config to always use S3 --- src/django/oar/settings.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 97f14dff9..bd5ad3636 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -614,11 +614,14 @@ TESTING = 'test' in sys.argv -if not DEBUG or (AWS_S3_ENDPOINT_URL and not TESTING): - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - +# Always force S3 when a bucket is configured (except during tests). This avoids +# accidentally falling back to local filesystem when DEBUG is False but an env +# is mis-set. Still allow Local+tests to use the default. AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') +if not TESTING and AWS_STORAGE_BUCKET_NAME: + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + if AWS_STORAGE_BUCKET_NAME is None and not DEBUG: raise ImproperlyConfigured( 'Invalid AWS_STORAGE_BUCKET_NAME provided, must be set in the environment' From 996c25e7250be84f066fca4d8ae3d301e7c578cc Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Tue, 30 Dec 2025 17:19:19 +0400 Subject: [PATCH 19/56] Use explicit image tag --- .github/workflows/deploy_to_aws.yml | 4 ++++ deployment/infra | 11 ++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy_to_aws.yml b/.github/workflows/deploy_to_aws.yml index c74693c04..3feead227 100644 --- a/.github/workflows/deploy_to_aws.yml +++ b/.github/workflows/deploy_to_aws.yml @@ -62,6 +62,8 @@ jobs: needs: check-base if: needs.check-base.outputs.skip == 'false' runs-on: ubuntu-latest + env: + IMAGE_TAG: 6ea4840 environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') }} steps: - name: Get Environment Name for ${{ vars.ENV_NAME }} @@ -179,6 +181,8 @@ jobs: apply: needs: [init-and-plan, detach-waf-if-needed] runs-on: ubuntu-latest + env: + IMAGE_TAG: 6ea4840 environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') }} if: ${{ inputs.deploy-plan-only == false }} steps: diff --git a/deployment/infra b/deployment/infra index e755efc8f..5aa782606 100755 --- a/deployment/infra +++ b/deployment/infra @@ -12,15 +12,16 @@ Execute Terraform subcommands with remote state management. " } -# # Temporary solution -# GIT_COMMIT="latest" - -if [[ -n "${GIT_COMMIT}" ]]; then +# Prefer an explicit IMAGE_TAG (e.g., from CI), otherwise fall back to a provided +# GIT_COMMIT or the current repo commit. Always shorten to 7 characters to match +# our ECR tagging convention. +if [[ -n "${IMAGE_TAG}" ]]; then + GIT_COMMIT="${IMAGE_TAG:0:7}" +elif [[ -n "${GIT_COMMIT}" ]]; then GIT_COMMIT="${GIT_COMMIT:0:7}" else SHORT_SHA="$(git rev-parse --short HEAD)" GIT_COMMIT="${SHORT_SHA:0:7}" - fi if [ "${BASH_SOURCE[0]}" = "${0}" ]; then From 6c1b1cb359dc9aaf5fbc0980a6dd7e0d7ae61f9e Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 31 Dec 2025 09:47:24 +0400 Subject: [PATCH 20/56] Use dynamic image tag --- ... | 258 ++++++++++++++++++ .github/workflows/deploy_to_aws.yml | 4 +- ...equest canceled, context deadline exceeded | 258 ++++++++++++++++++ src/django/oar/settings.py | 2 +- 4 files changed, 519 insertions(+), 3 deletions(-) create mode 100644 ... create mode 100644 h cached credentials, no EC2 IMDS role found, operation error ec2imds: GetMetadata, request canceled, context deadline exceeded diff --git a/... b/... new file mode 100644 index 000000000..333a0b576 --- /dev/null +++ b/... @@ -0,0 +1,258 @@ + + SSUUMMMMAARRYY OOFF LLEESSSS CCOOMMMMAANNDDSS + + Commands marked with * may be preceded by a number, _N. + Notes in parentheses indicate the behavior if _N is given. + A key preceded by a caret indicates the Ctrl key; thus ^K is ctrl-K. + + h H Display this help. + q :q Q :Q ZZ Exit. + --------------------------------------------------------------------------- + + MMOOVVIINNGG + + e ^E j ^N CR * Forward one line (or _N lines). + y ^Y k ^K ^P * Backward one line (or _N lines). + f ^F ^V SPACE * Forward one window (or _N lines). + b ^B ESC-v * Backward one window (or _N lines). + z * Forward one window (and set window to _N). + w * Backward one window (and set window to _N). + ESC-SPACE * Forward one window, but don't stop at end-of-file. + d ^D * Forward one half-window (and set half-window to _N). + u ^U * Backward one half-window (and set half-window to _N). + ESC-) RightArrow * Right one half screen width (or _N positions). + ESC-( LeftArrow * Left one half screen width (or _N positions). + ESC-} ^RightArrow Right to last column displayed. + ESC-{ ^LeftArrow Left to first column. + F Forward forever; like "tail -f". + ESC-F Like F but stop when search pattern is found. + r ^R ^L Repaint screen. + R Repaint screen, discarding buffered input. + --------------------------------------------------- + Default "window" is the screen height. + Default "half-window" is half of the screen height. + --------------------------------------------------------------------------- + + SSEEAARRCCHHIINNGG + + /_p_a_t_t_e_r_n * Search forward for (_N-th) matching line. + ?_p_a_t_t_e_r_n * Search backward for (_N-th) matching line. + n * Repeat previous search (for _N-th occurrence). + N * Repeat previous search in reverse direction. + ESC-n * Repeat previous search, spanning files. + ESC-N * Repeat previous search, reverse dir. & spanning files. + ESC-u Undo (toggle) search highlighting. + ESC-U Clear search highlighting. + &_p_a_t_t_e_r_n * Display only matching lines. + --------------------------------------------------- + A search pattern may begin with one or more of: + ^N or ! Search for NON-matching lines. + ^E or * Search multiple files (pass thru END OF FILE). + ^F or @ Start search at FIRST file (for /) or last file (for ?). + ^K Highlight matches, but don't move (KEEP position). + ^R Don't use REGULAR EXPRESSIONS. + ^W WRAP search if no match found. + --------------------------------------------------------------------------- + + JJUUMMPPIINNGG + + g < ESC-< * Go to first line in file (or line _N). + G > ESC-> * Go to last line in file (or line _N). + p % * Go to beginning of file (or _N percent into file). + t * Go to the (_N-th) next tag. + T * Go to the (_N-th) previous tag. + { ( [ * Find close bracket } ) ]. + } ) ] * Find open bracket { ( [. + ESC-^F _<_c_1_> _<_c_2_> * Find close bracket _<_c_2_>. + ESC-^B _<_c_1_> _<_c_2_> * Find open bracket _<_c_1_>. + --------------------------------------------------- + Each "find close bracket" command goes forward to the close bracket + matching the (_N-th) open bracket in the top line. + Each "find open bracket" command goes backward to the open bracket + matching the (_N-th) close bracket in the bottom line. + + m_<_l_e_t_t_e_r_> Mark the current top line with . + M_<_l_e_t_t_e_r_> Mark the current bottom line with . + '_<_l_e_t_t_e_r_> Go to a previously marked position. + '' Go to the previous position. + ^X^X Same as '. + ESC-M_<_l_e_t_t_e_r_> Clear a mark. + --------------------------------------------------- + A mark is any upper-case or lower-case letter. + Certain marks are predefined: + ^ means beginning of the file + $ means end of the file + --------------------------------------------------------------------------- + + CCHHAANNGGIINNGG FFIILLEESS + + :e [_f_i_l_e] Examine a new file. + ^X^V Same as :e. + :n * Examine the (_N-th) next file from the command line. + :p * Examine the (_N-th) previous file from the command line. + :x * Examine the first (or _N-th) file from the command line. + :d Delete the current file from the command line list. + = ^G :f Print current file name. + --------------------------------------------------------------------------- + + MMIISSCCEELLLLAANNEEOOUUSS CCOOMMMMAANNDDSS + + -_<_f_l_a_g_> Toggle a command line option [see OPTIONS below]. + --_<_n_a_m_e_> Toggle a command line option, by name. + __<_f_l_a_g_> Display the setting of a command line option. + ___<_n_a_m_e_> Display the setting of an option, by name. + +_c_m_d Execute the less cmd each time a new file is examined. + + !_c_o_m_m_a_n_d Execute the shell command with $SHELL. + |XX_c_o_m_m_a_n_d Pipe file between current pos & mark XX to shell command. + s _f_i_l_e Save input to a file. + v Edit the current file with $VISUAL or $EDITOR. + V Print version number of "less". + --------------------------------------------------------------------------- + + OOPPTTIIOONNSS + + Most options may be changed either on the command line, + or from within less by using the - or -- command. + Options may be given in one of two forms: either a single + character preceded by a -, or a name preceded by --. + + -? ........ --help + Display help (from command line). + -a ........ --search-skip-screen + Search skips current screen. + -A ........ --SEARCH-SKIP-SCREEN + Search starts just after target line. + -b [_N] .... --buffers=[_N] + Number of buffers. + -B ........ --auto-buffers + Don't automatically allocate buffers for pipes. + -c ........ --clear-screen + Repaint by clearing rather than scrolling. + -d ........ --dumb + Dumb terminal. + -D xx_c_o_l_o_r . --color=xx_c_o_l_o_r + Set screen colors. + -e -E .... --quit-at-eof --QUIT-AT-EOF + Quit at end of file. + -f ........ --force + Force open non-regular files. + -F ........ --quit-if-one-screen + Quit if entire file fits on first screen. + -g ........ --hilite-search + Highlight only last match for searches. + -G ........ --HILITE-SEARCH + Don't highlight any matches for searches. + -h [_N] .... --max-back-scroll=[_N] + Backward scroll limit. + -i ........ --ignore-case + Ignore case in searches that do not contain uppercase. + -I ........ --IGNORE-CASE + Ignore case in all searches. + -j [_N] .... --jump-target=[_N] + Screen position of target lines. + -J ........ --status-column + Display a status column at left edge of screen. + -k [_f_i_l_e] . --lesskey-file=[_f_i_l_e] + Use a lesskey file. + -K ........ --quit-on-intr + Exit less in response to ctrl-C. + -L ........ --no-lessopen + Ignore the LESSOPEN environment variable. + -m -M .... --long-prompt --LONG-PROMPT + Set prompt style. + -n -N .... --line-numbers --LINE-NUMBERS + Don't use line numbers. + -o [_f_i_l_e] . --log-file=[_f_i_l_e] + Copy to log file (standard input only). + -O [_f_i_l_e] . --LOG-FILE=[_f_i_l_e] + Copy to log file (unconditionally overwrite). + -p [_p_a_t_t_e_r_n] --pattern=[_p_a_t_t_e_r_n] + Start at pattern (from command line). + -P [_p_r_o_m_p_t] --prompt=[_p_r_o_m_p_t] + Define new prompt. + -q -Q .... --quiet --QUIET --silent --SILENT + Quiet the terminal bell. + -r -R .... --raw-control-chars --RAW-CONTROL-CHARS + Output "raw" control characters. + -s ........ --squeeze-blank-lines + Squeeze multiple blank lines. + -S ........ --chop-long-lines + Chop (truncate) long lines rather than wrapping. + -t [_t_a_g] .. --tag=[_t_a_g] + Find a tag. + -T [_t_a_g_s_f_i_l_e] --tag-file=[_t_a_g_s_f_i_l_e] + Use an alternate tags file. + -u -U .... --underline-special --UNDERLINE-SPECIAL + Change handling of backspaces. + -V ........ --version + Display the version number of "less". + -w ........ --hilite-unread + Highlight first new line after forward-screen. + -W ........ --HILITE-UNREAD + Highlight first new line after any forward movement. + -x [_N[,...]] --tabs=[_N[,...]] + Set tab stops. + -X ........ --no-init + Don't use termcap init/deinit strings. + -y [_N] .... --max-forw-scroll=[_N] + Forward scroll limit. + -z [_N] .... --window=[_N] + Set size of window. + -" [_c[_c]] . --quotes=[_c[_c]] + Set shell quote characters. + -~ ........ --tilde + Don't display tildes after end of file. + -# [_N] .... --shift=[_N] + Set horizontal scroll amount (0 = one half screen width). + --file-size + Automatically determine the size of the input file. + --follow-name + The F command changes files if the input file is renamed. + --incsearch + Search file as each pattern character is typed in. + --line-num-width=N + Set the width of the -N line number field to N characters. + --mouse + Enable mouse input. + --no-keypad + Don't send termcap keypad init/deinit strings. + --no-histdups + Remove duplicates from command history. + --rscroll=C + Set the character used to mark truncated lines. + --save-marks + Retain marks across invocations of less. + --status-col-width=N + Set the width of the -J status column to N characters. + --use-backslash + Subsequent options use backslash as escape char. + --use-color + Enables colored text. + --wheel-lines=N + Each click of the mouse wheel moves N lines. + + + --------------------------------------------------------------------------- + + LLIINNEE EEDDIITTIINNGG + + These keys can be used to edit text being entered + on the "command line" at the bottom of the screen. + + RightArrow ..................... ESC-l ... Move cursor right one character. + LeftArrow ...................... ESC-h ... Move cursor left one character. + ctrl-RightArrow ESC-RightArrow ESC-w ... Move cursor right one word. + ctrl-LeftArrow ESC-LeftArrow ESC-b ... Move cursor left one word. + HOME ........................... ESC-0 ... Move cursor to start of line. + END ............................ ESC-$ ... Move cursor to end of line. + BACKSPACE ................................ Delete char to left of cursor. + DELETE ......................... ESC-x ... Delete char under cursor. + ctrl-BACKSPACE ESC-BACKSPACE ........... Delete word to left of cursor. + ctrl-DELETE .... ESC-DELETE .... ESC-X ... Delete word under cursor. + ctrl-U ......... ESC (MS-DOS only) ....... Delete entire line. + UpArrow ........................ ESC-k ... Retrieve previous command line. + DownArrow ...................... ESC-j ... Retrieve next command line. + TAB ...................................... Complete filename & cycle. + SHIFT-TAB ...................... ESC-TAB Complete filename & reverse cycle. + ctrl-L ................................... Complete filename, list all. diff --git a/.github/workflows/deploy_to_aws.yml b/.github/workflows/deploy_to_aws.yml index 3feead227..0a1ced0b7 100644 --- a/.github/workflows/deploy_to_aws.yml +++ b/.github/workflows/deploy_to_aws.yml @@ -63,7 +63,7 @@ jobs: if: needs.check-base.outputs.skip == 'false' runs-on: ubuntu-latest env: - IMAGE_TAG: 6ea4840 + IMAGE_TAG: ${{ github.sha }} environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') }} steps: - name: Get Environment Name for ${{ vars.ENV_NAME }} @@ -182,7 +182,7 @@ jobs: needs: [init-and-plan, detach-waf-if-needed] runs-on: ubuntu-latest env: - IMAGE_TAG: 6ea4840 + IMAGE_TAG: ${{ github.sha }} environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') }} if: ${{ inputs.deploy-plan-only == false }} steps: diff --git a/h cached credentials, no EC2 IMDS role found, operation error ec2imds: GetMetadata, request canceled, context deadline exceeded b/h cached credentials, no EC2 IMDS role found, operation error ec2imds: GetMetadata, request canceled, context deadline exceeded new file mode 100644 index 000000000..333a0b576 --- /dev/null +++ b/h cached credentials, no EC2 IMDS role found, operation error ec2imds: GetMetadata, request canceled, context deadline exceeded @@ -0,0 +1,258 @@ + + SSUUMMMMAARRYY OOFF LLEESSSS CCOOMMMMAANNDDSS + + Commands marked with * may be preceded by a number, _N. + Notes in parentheses indicate the behavior if _N is given. + A key preceded by a caret indicates the Ctrl key; thus ^K is ctrl-K. + + h H Display this help. + q :q Q :Q ZZ Exit. + --------------------------------------------------------------------------- + + MMOOVVIINNGG + + e ^E j ^N CR * Forward one line (or _N lines). + y ^Y k ^K ^P * Backward one line (or _N lines). + f ^F ^V SPACE * Forward one window (or _N lines). + b ^B ESC-v * Backward one window (or _N lines). + z * Forward one window (and set window to _N). + w * Backward one window (and set window to _N). + ESC-SPACE * Forward one window, but don't stop at end-of-file. + d ^D * Forward one half-window (and set half-window to _N). + u ^U * Backward one half-window (and set half-window to _N). + ESC-) RightArrow * Right one half screen width (or _N positions). + ESC-( LeftArrow * Left one half screen width (or _N positions). + ESC-} ^RightArrow Right to last column displayed. + ESC-{ ^LeftArrow Left to first column. + F Forward forever; like "tail -f". + ESC-F Like F but stop when search pattern is found. + r ^R ^L Repaint screen. + R Repaint screen, discarding buffered input. + --------------------------------------------------- + Default "window" is the screen height. + Default "half-window" is half of the screen height. + --------------------------------------------------------------------------- + + SSEEAARRCCHHIINNGG + + /_p_a_t_t_e_r_n * Search forward for (_N-th) matching line. + ?_p_a_t_t_e_r_n * Search backward for (_N-th) matching line. + n * Repeat previous search (for _N-th occurrence). + N * Repeat previous search in reverse direction. + ESC-n * Repeat previous search, spanning files. + ESC-N * Repeat previous search, reverse dir. & spanning files. + ESC-u Undo (toggle) search highlighting. + ESC-U Clear search highlighting. + &_p_a_t_t_e_r_n * Display only matching lines. + --------------------------------------------------- + A search pattern may begin with one or more of: + ^N or ! Search for NON-matching lines. + ^E or * Search multiple files (pass thru END OF FILE). + ^F or @ Start search at FIRST file (for /) or last file (for ?). + ^K Highlight matches, but don't move (KEEP position). + ^R Don't use REGULAR EXPRESSIONS. + ^W WRAP search if no match found. + --------------------------------------------------------------------------- + + JJUUMMPPIINNGG + + g < ESC-< * Go to first line in file (or line _N). + G > ESC-> * Go to last line in file (or line _N). + p % * Go to beginning of file (or _N percent into file). + t * Go to the (_N-th) next tag. + T * Go to the (_N-th) previous tag. + { ( [ * Find close bracket } ) ]. + } ) ] * Find open bracket { ( [. + ESC-^F _<_c_1_> _<_c_2_> * Find close bracket _<_c_2_>. + ESC-^B _<_c_1_> _<_c_2_> * Find open bracket _<_c_1_>. + --------------------------------------------------- + Each "find close bracket" command goes forward to the close bracket + matching the (_N-th) open bracket in the top line. + Each "find open bracket" command goes backward to the open bracket + matching the (_N-th) close bracket in the bottom line. + + m_<_l_e_t_t_e_r_> Mark the current top line with . + M_<_l_e_t_t_e_r_> Mark the current bottom line with . + '_<_l_e_t_t_e_r_> Go to a previously marked position. + '' Go to the previous position. + ^X^X Same as '. + ESC-M_<_l_e_t_t_e_r_> Clear a mark. + --------------------------------------------------- + A mark is any upper-case or lower-case letter. + Certain marks are predefined: + ^ means beginning of the file + $ means end of the file + --------------------------------------------------------------------------- + + CCHHAANNGGIINNGG FFIILLEESS + + :e [_f_i_l_e] Examine a new file. + ^X^V Same as :e. + :n * Examine the (_N-th) next file from the command line. + :p * Examine the (_N-th) previous file from the command line. + :x * Examine the first (or _N-th) file from the command line. + :d Delete the current file from the command line list. + = ^G :f Print current file name. + --------------------------------------------------------------------------- + + MMIISSCCEELLLLAANNEEOOUUSS CCOOMMMMAANNDDSS + + -_<_f_l_a_g_> Toggle a command line option [see OPTIONS below]. + --_<_n_a_m_e_> Toggle a command line option, by name. + __<_f_l_a_g_> Display the setting of a command line option. + ___<_n_a_m_e_> Display the setting of an option, by name. + +_c_m_d Execute the less cmd each time a new file is examined. + + !_c_o_m_m_a_n_d Execute the shell command with $SHELL. + |XX_c_o_m_m_a_n_d Pipe file between current pos & mark XX to shell command. + s _f_i_l_e Save input to a file. + v Edit the current file with $VISUAL or $EDITOR. + V Print version number of "less". + --------------------------------------------------------------------------- + + OOPPTTIIOONNSS + + Most options may be changed either on the command line, + or from within less by using the - or -- command. + Options may be given in one of two forms: either a single + character preceded by a -, or a name preceded by --. + + -? ........ --help + Display help (from command line). + -a ........ --search-skip-screen + Search skips current screen. + -A ........ --SEARCH-SKIP-SCREEN + Search starts just after target line. + -b [_N] .... --buffers=[_N] + Number of buffers. + -B ........ --auto-buffers + Don't automatically allocate buffers for pipes. + -c ........ --clear-screen + Repaint by clearing rather than scrolling. + -d ........ --dumb + Dumb terminal. + -D xx_c_o_l_o_r . --color=xx_c_o_l_o_r + Set screen colors. + -e -E .... --quit-at-eof --QUIT-AT-EOF + Quit at end of file. + -f ........ --force + Force open non-regular files. + -F ........ --quit-if-one-screen + Quit if entire file fits on first screen. + -g ........ --hilite-search + Highlight only last match for searches. + -G ........ --HILITE-SEARCH + Don't highlight any matches for searches. + -h [_N] .... --max-back-scroll=[_N] + Backward scroll limit. + -i ........ --ignore-case + Ignore case in searches that do not contain uppercase. + -I ........ --IGNORE-CASE + Ignore case in all searches. + -j [_N] .... --jump-target=[_N] + Screen position of target lines. + -J ........ --status-column + Display a status column at left edge of screen. + -k [_f_i_l_e] . --lesskey-file=[_f_i_l_e] + Use a lesskey file. + -K ........ --quit-on-intr + Exit less in response to ctrl-C. + -L ........ --no-lessopen + Ignore the LESSOPEN environment variable. + -m -M .... --long-prompt --LONG-PROMPT + Set prompt style. + -n -N .... --line-numbers --LINE-NUMBERS + Don't use line numbers. + -o [_f_i_l_e] . --log-file=[_f_i_l_e] + Copy to log file (standard input only). + -O [_f_i_l_e] . --LOG-FILE=[_f_i_l_e] + Copy to log file (unconditionally overwrite). + -p [_p_a_t_t_e_r_n] --pattern=[_p_a_t_t_e_r_n] + Start at pattern (from command line). + -P [_p_r_o_m_p_t] --prompt=[_p_r_o_m_p_t] + Define new prompt. + -q -Q .... --quiet --QUIET --silent --SILENT + Quiet the terminal bell. + -r -R .... --raw-control-chars --RAW-CONTROL-CHARS + Output "raw" control characters. + -s ........ --squeeze-blank-lines + Squeeze multiple blank lines. + -S ........ --chop-long-lines + Chop (truncate) long lines rather than wrapping. + -t [_t_a_g] .. --tag=[_t_a_g] + Find a tag. + -T [_t_a_g_s_f_i_l_e] --tag-file=[_t_a_g_s_f_i_l_e] + Use an alternate tags file. + -u -U .... --underline-special --UNDERLINE-SPECIAL + Change handling of backspaces. + -V ........ --version + Display the version number of "less". + -w ........ --hilite-unread + Highlight first new line after forward-screen. + -W ........ --HILITE-UNREAD + Highlight first new line after any forward movement. + -x [_N[,...]] --tabs=[_N[,...]] + Set tab stops. + -X ........ --no-init + Don't use termcap init/deinit strings. + -y [_N] .... --max-forw-scroll=[_N] + Forward scroll limit. + -z [_N] .... --window=[_N] + Set size of window. + -" [_c[_c]] . --quotes=[_c[_c]] + Set shell quote characters. + -~ ........ --tilde + Don't display tildes after end of file. + -# [_N] .... --shift=[_N] + Set horizontal scroll amount (0 = one half screen width). + --file-size + Automatically determine the size of the input file. + --follow-name + The F command changes files if the input file is renamed. + --incsearch + Search file as each pattern character is typed in. + --line-num-width=N + Set the width of the -N line number field to N characters. + --mouse + Enable mouse input. + --no-keypad + Don't send termcap keypad init/deinit strings. + --no-histdups + Remove duplicates from command history. + --rscroll=C + Set the character used to mark truncated lines. + --save-marks + Retain marks across invocations of less. + --status-col-width=N + Set the width of the -J status column to N characters. + --use-backslash + Subsequent options use backslash as escape char. + --use-color + Enables colored text. + --wheel-lines=N + Each click of the mouse wheel moves N lines. + + + --------------------------------------------------------------------------- + + LLIINNEE EEDDIITTIINNGG + + These keys can be used to edit text being entered + on the "command line" at the bottom of the screen. + + RightArrow ..................... ESC-l ... Move cursor right one character. + LeftArrow ...................... ESC-h ... Move cursor left one character. + ctrl-RightArrow ESC-RightArrow ESC-w ... Move cursor right one word. + ctrl-LeftArrow ESC-LeftArrow ESC-b ... Move cursor left one word. + HOME ........................... ESC-0 ... Move cursor to start of line. + END ............................ ESC-$ ... Move cursor to end of line. + BACKSPACE ................................ Delete char to left of cursor. + DELETE ......................... ESC-x ... Delete char under cursor. + ctrl-BACKSPACE ESC-BACKSPACE ........... Delete word to left of cursor. + ctrl-DELETE .... ESC-DELETE .... ESC-X ... Delete word under cursor. + ctrl-U ......... ESC (MS-DOS only) ....... Delete entire line. + UpArrow ........................ ESC-k ... Retrieve previous command line. + DownArrow ...................... ESC-j ... Retrieve next command line. + TAB ...................................... Complete filename & cycle. + SHIFT-TAB ...................... ESC-TAB Complete filename & reverse cycle. + ctrl-L ................................... Complete filename, list all. diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index bd5ad3636..59f3cd7b5 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -309,7 +309,7 @@ # Use PyLibMCCache everywhere; django_elasticache is incompatible with Django 4 # because it still imports smart_text. This keeps throttling cache working in # jobs/containers that upgrade Django. -CACHE_BACKEND = 'django.core.cache.backends.memcached.PyLibMCCache' + CACHE_BACKEND = 'django.core.cache.backends.memcached.PyLibMCCache' CACHES = { 'default': { From 6ca768dcf45763952c35843948875a62c341288d Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 31 Dec 2025 10:03:18 +0400 Subject: [PATCH 21/56] Fix indentation error --- src/django/oar/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 59f3cd7b5..bd5ad3636 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -309,7 +309,7 @@ # Use PyLibMCCache everywhere; django_elasticache is incompatible with Django 4 # because it still imports smart_text. This keeps throttling cache working in # jobs/containers that upgrade Django. - CACHE_BACKEND = 'django.core.cache.backends.memcached.PyLibMCCache' +CACHE_BACKEND = 'django.core.cache.backends.memcached.PyLibMCCache' CACHES = { 'default': { From 2735b9de878a2b332f5acda9f339e7598465d35f Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 31 Dec 2025 14:57:58 +0400 Subject: [PATCH 22/56] Force S3 bucket storage --- src/django/oar/settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index bd5ad3636..53e6b5f27 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -619,8 +619,12 @@ # is mis-set. Still allow Local+tests to use the default. AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') -if not TESTING and AWS_STORAGE_BUCKET_NAME: +# Force S3-backed storage whenever a bucket is provided (including non-test +# local runs), so default_storage/Model FileField storage cannot fall back to +# the filesystem. +if AWS_STORAGE_BUCKET_NAME: DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + FILE_UPLOAD_STORAGE = DEFAULT_FILE_STORAGE if AWS_STORAGE_BUCKET_NAME is None and not DEBUG: raise ImproperlyConfigured( From f7f321882dc62c20a5322e393d96a8017f381317 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 31 Dec 2025 14:59:54 +0400 Subject: [PATCH 23/56] Remove cursor artifacts --- ... | 258 ------------------ ...equest canceled, context deadline exceeded | 258 ------------------ 2 files changed, 516 deletions(-) delete mode 100644 ... delete mode 100644 h cached credentials, no EC2 IMDS role found, operation error ec2imds: GetMetadata, request canceled, context deadline exceeded diff --git a/... b/... deleted file mode 100644 index 333a0b576..000000000 --- a/... +++ /dev/null @@ -1,258 +0,0 @@ - - SSUUMMMMAARRYY OOFF LLEESSSS CCOOMMMMAANNDDSS - - Commands marked with * may be preceded by a number, _N. - Notes in parentheses indicate the behavior if _N is given. - A key preceded by a caret indicates the Ctrl key; thus ^K is ctrl-K. - - h H Display this help. - q :q Q :Q ZZ Exit. - --------------------------------------------------------------------------- - - MMOOVVIINNGG - - e ^E j ^N CR * Forward one line (or _N lines). - y ^Y k ^K ^P * Backward one line (or _N lines). - f ^F ^V SPACE * Forward one window (or _N lines). - b ^B ESC-v * Backward one window (or _N lines). - z * Forward one window (and set window to _N). - w * Backward one window (and set window to _N). - ESC-SPACE * Forward one window, but don't stop at end-of-file. - d ^D * Forward one half-window (and set half-window to _N). - u ^U * Backward one half-window (and set half-window to _N). - ESC-) RightArrow * Right one half screen width (or _N positions). - ESC-( LeftArrow * Left one half screen width (or _N positions). - ESC-} ^RightArrow Right to last column displayed. - ESC-{ ^LeftArrow Left to first column. - F Forward forever; like "tail -f". - ESC-F Like F but stop when search pattern is found. - r ^R ^L Repaint screen. - R Repaint screen, discarding buffered input. - --------------------------------------------------- - Default "window" is the screen height. - Default "half-window" is half of the screen height. - --------------------------------------------------------------------------- - - SSEEAARRCCHHIINNGG - - /_p_a_t_t_e_r_n * Search forward for (_N-th) matching line. - ?_p_a_t_t_e_r_n * Search backward for (_N-th) matching line. - n * Repeat previous search (for _N-th occurrence). - N * Repeat previous search in reverse direction. - ESC-n * Repeat previous search, spanning files. - ESC-N * Repeat previous search, reverse dir. & spanning files. - ESC-u Undo (toggle) search highlighting. - ESC-U Clear search highlighting. - &_p_a_t_t_e_r_n * Display only matching lines. - --------------------------------------------------- - A search pattern may begin with one or more of: - ^N or ! Search for NON-matching lines. - ^E or * Search multiple files (pass thru END OF FILE). - ^F or @ Start search at FIRST file (for /) or last file (for ?). - ^K Highlight matches, but don't move (KEEP position). - ^R Don't use REGULAR EXPRESSIONS. - ^W WRAP search if no match found. - --------------------------------------------------------------------------- - - JJUUMMPPIINNGG - - g < ESC-< * Go to first line in file (or line _N). - G > ESC-> * Go to last line in file (or line _N). - p % * Go to beginning of file (or _N percent into file). - t * Go to the (_N-th) next tag. - T * Go to the (_N-th) previous tag. - { ( [ * Find close bracket } ) ]. - } ) ] * Find open bracket { ( [. - ESC-^F _<_c_1_> _<_c_2_> * Find close bracket _<_c_2_>. - ESC-^B _<_c_1_> _<_c_2_> * Find open bracket _<_c_1_>. - --------------------------------------------------- - Each "find close bracket" command goes forward to the close bracket - matching the (_N-th) open bracket in the top line. - Each "find open bracket" command goes backward to the open bracket - matching the (_N-th) close bracket in the bottom line. - - m_<_l_e_t_t_e_r_> Mark the current top line with . - M_<_l_e_t_t_e_r_> Mark the current bottom line with . - '_<_l_e_t_t_e_r_> Go to a previously marked position. - '' Go to the previous position. - ^X^X Same as '. - ESC-M_<_l_e_t_t_e_r_> Clear a mark. - --------------------------------------------------- - A mark is any upper-case or lower-case letter. - Certain marks are predefined: - ^ means beginning of the file - $ means end of the file - --------------------------------------------------------------------------- - - CCHHAANNGGIINNGG FFIILLEESS - - :e [_f_i_l_e] Examine a new file. - ^X^V Same as :e. - :n * Examine the (_N-th) next file from the command line. - :p * Examine the (_N-th) previous file from the command line. - :x * Examine the first (or _N-th) file from the command line. - :d Delete the current file from the command line list. - = ^G :f Print current file name. - --------------------------------------------------------------------------- - - MMIISSCCEELLLLAANNEEOOUUSS CCOOMMMMAANNDDSS - - -_<_f_l_a_g_> Toggle a command line option [see OPTIONS below]. - --_<_n_a_m_e_> Toggle a command line option, by name. - __<_f_l_a_g_> Display the setting of a command line option. - ___<_n_a_m_e_> Display the setting of an option, by name. - +_c_m_d Execute the less cmd each time a new file is examined. - - !_c_o_m_m_a_n_d Execute the shell command with $SHELL. - |XX_c_o_m_m_a_n_d Pipe file between current pos & mark XX to shell command. - s _f_i_l_e Save input to a file. - v Edit the current file with $VISUAL or $EDITOR. - V Print version number of "less". - --------------------------------------------------------------------------- - - OOPPTTIIOONNSS - - Most options may be changed either on the command line, - or from within less by using the - or -- command. - Options may be given in one of two forms: either a single - character preceded by a -, or a name preceded by --. - - -? ........ --help - Display help (from command line). - -a ........ --search-skip-screen - Search skips current screen. - -A ........ --SEARCH-SKIP-SCREEN - Search starts just after target line. - -b [_N] .... --buffers=[_N] - Number of buffers. - -B ........ --auto-buffers - Don't automatically allocate buffers for pipes. - -c ........ --clear-screen - Repaint by clearing rather than scrolling. - -d ........ --dumb - Dumb terminal. - -D xx_c_o_l_o_r . --color=xx_c_o_l_o_r - Set screen colors. - -e -E .... --quit-at-eof --QUIT-AT-EOF - Quit at end of file. - -f ........ --force - Force open non-regular files. - -F ........ --quit-if-one-screen - Quit if entire file fits on first screen. - -g ........ --hilite-search - Highlight only last match for searches. - -G ........ --HILITE-SEARCH - Don't highlight any matches for searches. - -h [_N] .... --max-back-scroll=[_N] - Backward scroll limit. - -i ........ --ignore-case - Ignore case in searches that do not contain uppercase. - -I ........ --IGNORE-CASE - Ignore case in all searches. - -j [_N] .... --jump-target=[_N] - Screen position of target lines. - -J ........ --status-column - Display a status column at left edge of screen. - -k [_f_i_l_e] . --lesskey-file=[_f_i_l_e] - Use a lesskey file. - -K ........ --quit-on-intr - Exit less in response to ctrl-C. - -L ........ --no-lessopen - Ignore the LESSOPEN environment variable. - -m -M .... --long-prompt --LONG-PROMPT - Set prompt style. - -n -N .... --line-numbers --LINE-NUMBERS - Don't use line numbers. - -o [_f_i_l_e] . --log-file=[_f_i_l_e] - Copy to log file (standard input only). - -O [_f_i_l_e] . --LOG-FILE=[_f_i_l_e] - Copy to log file (unconditionally overwrite). - -p [_p_a_t_t_e_r_n] --pattern=[_p_a_t_t_e_r_n] - Start at pattern (from command line). - -P [_p_r_o_m_p_t] --prompt=[_p_r_o_m_p_t] - Define new prompt. - -q -Q .... --quiet --QUIET --silent --SILENT - Quiet the terminal bell. - -r -R .... --raw-control-chars --RAW-CONTROL-CHARS - Output "raw" control characters. - -s ........ --squeeze-blank-lines - Squeeze multiple blank lines. - -S ........ --chop-long-lines - Chop (truncate) long lines rather than wrapping. - -t [_t_a_g] .. --tag=[_t_a_g] - Find a tag. - -T [_t_a_g_s_f_i_l_e] --tag-file=[_t_a_g_s_f_i_l_e] - Use an alternate tags file. - -u -U .... --underline-special --UNDERLINE-SPECIAL - Change handling of backspaces. - -V ........ --version - Display the version number of "less". - -w ........ --hilite-unread - Highlight first new line after forward-screen. - -W ........ --HILITE-UNREAD - Highlight first new line after any forward movement. - -x [_N[,...]] --tabs=[_N[,...]] - Set tab stops. - -X ........ --no-init - Don't use termcap init/deinit strings. - -y [_N] .... --max-forw-scroll=[_N] - Forward scroll limit. - -z [_N] .... --window=[_N] - Set size of window. - -" [_c[_c]] . --quotes=[_c[_c]] - Set shell quote characters. - -~ ........ --tilde - Don't display tildes after end of file. - -# [_N] .... --shift=[_N] - Set horizontal scroll amount (0 = one half screen width). - --file-size - Automatically determine the size of the input file. - --follow-name - The F command changes files if the input file is renamed. - --incsearch - Search file as each pattern character is typed in. - --line-num-width=N - Set the width of the -N line number field to N characters. - --mouse - Enable mouse input. - --no-keypad - Don't send termcap keypad init/deinit strings. - --no-histdups - Remove duplicates from command history. - --rscroll=C - Set the character used to mark truncated lines. - --save-marks - Retain marks across invocations of less. - --status-col-width=N - Set the width of the -J status column to N characters. - --use-backslash - Subsequent options use backslash as escape char. - --use-color - Enables colored text. - --wheel-lines=N - Each click of the mouse wheel moves N lines. - - - --------------------------------------------------------------------------- - - LLIINNEE EEDDIITTIINNGG - - These keys can be used to edit text being entered - on the "command line" at the bottom of the screen. - - RightArrow ..................... ESC-l ... Move cursor right one character. - LeftArrow ...................... ESC-h ... Move cursor left one character. - ctrl-RightArrow ESC-RightArrow ESC-w ... Move cursor right one word. - ctrl-LeftArrow ESC-LeftArrow ESC-b ... Move cursor left one word. - HOME ........................... ESC-0 ... Move cursor to start of line. - END ............................ ESC-$ ... Move cursor to end of line. - BACKSPACE ................................ Delete char to left of cursor. - DELETE ......................... ESC-x ... Delete char under cursor. - ctrl-BACKSPACE ESC-BACKSPACE ........... Delete word to left of cursor. - ctrl-DELETE .... ESC-DELETE .... ESC-X ... Delete word under cursor. - ctrl-U ......... ESC (MS-DOS only) ....... Delete entire line. - UpArrow ........................ ESC-k ... Retrieve previous command line. - DownArrow ...................... ESC-j ... Retrieve next command line. - TAB ...................................... Complete filename & cycle. - SHIFT-TAB ...................... ESC-TAB Complete filename & reverse cycle. - ctrl-L ................................... Complete filename, list all. diff --git a/h cached credentials, no EC2 IMDS role found, operation error ec2imds: GetMetadata, request canceled, context deadline exceeded b/h cached credentials, no EC2 IMDS role found, operation error ec2imds: GetMetadata, request canceled, context deadline exceeded deleted file mode 100644 index 333a0b576..000000000 --- a/h cached credentials, no EC2 IMDS role found, operation error ec2imds: GetMetadata, request canceled, context deadline exceeded +++ /dev/null @@ -1,258 +0,0 @@ - - SSUUMMMMAARRYY OOFF LLEESSSS CCOOMMMMAANNDDSS - - Commands marked with * may be preceded by a number, _N. - Notes in parentheses indicate the behavior if _N is given. - A key preceded by a caret indicates the Ctrl key; thus ^K is ctrl-K. - - h H Display this help. - q :q Q :Q ZZ Exit. - --------------------------------------------------------------------------- - - MMOOVVIINNGG - - e ^E j ^N CR * Forward one line (or _N lines). - y ^Y k ^K ^P * Backward one line (or _N lines). - f ^F ^V SPACE * Forward one window (or _N lines). - b ^B ESC-v * Backward one window (or _N lines). - z * Forward one window (and set window to _N). - w * Backward one window (and set window to _N). - ESC-SPACE * Forward one window, but don't stop at end-of-file. - d ^D * Forward one half-window (and set half-window to _N). - u ^U * Backward one half-window (and set half-window to _N). - ESC-) RightArrow * Right one half screen width (or _N positions). - ESC-( LeftArrow * Left one half screen width (or _N positions). - ESC-} ^RightArrow Right to last column displayed. - ESC-{ ^LeftArrow Left to first column. - F Forward forever; like "tail -f". - ESC-F Like F but stop when search pattern is found. - r ^R ^L Repaint screen. - R Repaint screen, discarding buffered input. - --------------------------------------------------- - Default "window" is the screen height. - Default "half-window" is half of the screen height. - --------------------------------------------------------------------------- - - SSEEAARRCCHHIINNGG - - /_p_a_t_t_e_r_n * Search forward for (_N-th) matching line. - ?_p_a_t_t_e_r_n * Search backward for (_N-th) matching line. - n * Repeat previous search (for _N-th occurrence). - N * Repeat previous search in reverse direction. - ESC-n * Repeat previous search, spanning files. - ESC-N * Repeat previous search, reverse dir. & spanning files. - ESC-u Undo (toggle) search highlighting. - ESC-U Clear search highlighting. - &_p_a_t_t_e_r_n * Display only matching lines. - --------------------------------------------------- - A search pattern may begin with one or more of: - ^N or ! Search for NON-matching lines. - ^E or * Search multiple files (pass thru END OF FILE). - ^F or @ Start search at FIRST file (for /) or last file (for ?). - ^K Highlight matches, but don't move (KEEP position). - ^R Don't use REGULAR EXPRESSIONS. - ^W WRAP search if no match found. - --------------------------------------------------------------------------- - - JJUUMMPPIINNGG - - g < ESC-< * Go to first line in file (or line _N). - G > ESC-> * Go to last line in file (or line _N). - p % * Go to beginning of file (or _N percent into file). - t * Go to the (_N-th) next tag. - T * Go to the (_N-th) previous tag. - { ( [ * Find close bracket } ) ]. - } ) ] * Find open bracket { ( [. - ESC-^F _<_c_1_> _<_c_2_> * Find close bracket _<_c_2_>. - ESC-^B _<_c_1_> _<_c_2_> * Find open bracket _<_c_1_>. - --------------------------------------------------- - Each "find close bracket" command goes forward to the close bracket - matching the (_N-th) open bracket in the top line. - Each "find open bracket" command goes backward to the open bracket - matching the (_N-th) close bracket in the bottom line. - - m_<_l_e_t_t_e_r_> Mark the current top line with . - M_<_l_e_t_t_e_r_> Mark the current bottom line with . - '_<_l_e_t_t_e_r_> Go to a previously marked position. - '' Go to the previous position. - ^X^X Same as '. - ESC-M_<_l_e_t_t_e_r_> Clear a mark. - --------------------------------------------------- - A mark is any upper-case or lower-case letter. - Certain marks are predefined: - ^ means beginning of the file - $ means end of the file - --------------------------------------------------------------------------- - - CCHHAANNGGIINNGG FFIILLEESS - - :e [_f_i_l_e] Examine a new file. - ^X^V Same as :e. - :n * Examine the (_N-th) next file from the command line. - :p * Examine the (_N-th) previous file from the command line. - :x * Examine the first (or _N-th) file from the command line. - :d Delete the current file from the command line list. - = ^G :f Print current file name. - --------------------------------------------------------------------------- - - MMIISSCCEELLLLAANNEEOOUUSS CCOOMMMMAANNDDSS - - -_<_f_l_a_g_> Toggle a command line option [see OPTIONS below]. - --_<_n_a_m_e_> Toggle a command line option, by name. - __<_f_l_a_g_> Display the setting of a command line option. - ___<_n_a_m_e_> Display the setting of an option, by name. - +_c_m_d Execute the less cmd each time a new file is examined. - - !_c_o_m_m_a_n_d Execute the shell command with $SHELL. - |XX_c_o_m_m_a_n_d Pipe file between current pos & mark XX to shell command. - s _f_i_l_e Save input to a file. - v Edit the current file with $VISUAL or $EDITOR. - V Print version number of "less". - --------------------------------------------------------------------------- - - OOPPTTIIOONNSS - - Most options may be changed either on the command line, - or from within less by using the - or -- command. - Options may be given in one of two forms: either a single - character preceded by a -, or a name preceded by --. - - -? ........ --help - Display help (from command line). - -a ........ --search-skip-screen - Search skips current screen. - -A ........ --SEARCH-SKIP-SCREEN - Search starts just after target line. - -b [_N] .... --buffers=[_N] - Number of buffers. - -B ........ --auto-buffers - Don't automatically allocate buffers for pipes. - -c ........ --clear-screen - Repaint by clearing rather than scrolling. - -d ........ --dumb - Dumb terminal. - -D xx_c_o_l_o_r . --color=xx_c_o_l_o_r - Set screen colors. - -e -E .... --quit-at-eof --QUIT-AT-EOF - Quit at end of file. - -f ........ --force - Force open non-regular files. - -F ........ --quit-if-one-screen - Quit if entire file fits on first screen. - -g ........ --hilite-search - Highlight only last match for searches. - -G ........ --HILITE-SEARCH - Don't highlight any matches for searches. - -h [_N] .... --max-back-scroll=[_N] - Backward scroll limit. - -i ........ --ignore-case - Ignore case in searches that do not contain uppercase. - -I ........ --IGNORE-CASE - Ignore case in all searches. - -j [_N] .... --jump-target=[_N] - Screen position of target lines. - -J ........ --status-column - Display a status column at left edge of screen. - -k [_f_i_l_e] . --lesskey-file=[_f_i_l_e] - Use a lesskey file. - -K ........ --quit-on-intr - Exit less in response to ctrl-C. - -L ........ --no-lessopen - Ignore the LESSOPEN environment variable. - -m -M .... --long-prompt --LONG-PROMPT - Set prompt style. - -n -N .... --line-numbers --LINE-NUMBERS - Don't use line numbers. - -o [_f_i_l_e] . --log-file=[_f_i_l_e] - Copy to log file (standard input only). - -O [_f_i_l_e] . --LOG-FILE=[_f_i_l_e] - Copy to log file (unconditionally overwrite). - -p [_p_a_t_t_e_r_n] --pattern=[_p_a_t_t_e_r_n] - Start at pattern (from command line). - -P [_p_r_o_m_p_t] --prompt=[_p_r_o_m_p_t] - Define new prompt. - -q -Q .... --quiet --QUIET --silent --SILENT - Quiet the terminal bell. - -r -R .... --raw-control-chars --RAW-CONTROL-CHARS - Output "raw" control characters. - -s ........ --squeeze-blank-lines - Squeeze multiple blank lines. - -S ........ --chop-long-lines - Chop (truncate) long lines rather than wrapping. - -t [_t_a_g] .. --tag=[_t_a_g] - Find a tag. - -T [_t_a_g_s_f_i_l_e] --tag-file=[_t_a_g_s_f_i_l_e] - Use an alternate tags file. - -u -U .... --underline-special --UNDERLINE-SPECIAL - Change handling of backspaces. - -V ........ --version - Display the version number of "less". - -w ........ --hilite-unread - Highlight first new line after forward-screen. - -W ........ --HILITE-UNREAD - Highlight first new line after any forward movement. - -x [_N[,...]] --tabs=[_N[,...]] - Set tab stops. - -X ........ --no-init - Don't use termcap init/deinit strings. - -y [_N] .... --max-forw-scroll=[_N] - Forward scroll limit. - -z [_N] .... --window=[_N] - Set size of window. - -" [_c[_c]] . --quotes=[_c[_c]] - Set shell quote characters. - -~ ........ --tilde - Don't display tildes after end of file. - -# [_N] .... --shift=[_N] - Set horizontal scroll amount (0 = one half screen width). - --file-size - Automatically determine the size of the input file. - --follow-name - The F command changes files if the input file is renamed. - --incsearch - Search file as each pattern character is typed in. - --line-num-width=N - Set the width of the -N line number field to N characters. - --mouse - Enable mouse input. - --no-keypad - Don't send termcap keypad init/deinit strings. - --no-histdups - Remove duplicates from command history. - --rscroll=C - Set the character used to mark truncated lines. - --save-marks - Retain marks across invocations of less. - --status-col-width=N - Set the width of the -J status column to N characters. - --use-backslash - Subsequent options use backslash as escape char. - --use-color - Enables colored text. - --wheel-lines=N - Each click of the mouse wheel moves N lines. - - - --------------------------------------------------------------------------- - - LLIINNEE EEDDIITTIINNGG - - These keys can be used to edit text being entered - on the "command line" at the bottom of the screen. - - RightArrow ..................... ESC-l ... Move cursor right one character. - LeftArrow ...................... ESC-h ... Move cursor left one character. - ctrl-RightArrow ESC-RightArrow ESC-w ... Move cursor right one word. - ctrl-LeftArrow ESC-LeftArrow ESC-b ... Move cursor left one word. - HOME ........................... ESC-0 ... Move cursor to start of line. - END ............................ ESC-$ ... Move cursor to end of line. - BACKSPACE ................................ Delete char to left of cursor. - DELETE ......................... ESC-x ... Delete char under cursor. - ctrl-BACKSPACE ESC-BACKSPACE ........... Delete word to left of cursor. - ctrl-DELETE .... ESC-DELETE .... ESC-X ... Delete word under cursor. - ctrl-U ......... ESC (MS-DOS only) ....... Delete entire line. - UpArrow ........................ ESC-k ... Retrieve previous command line. - DownArrow ...................... ESC-j ... Retrieve next command line. - TAB ...................................... Complete filename & cycle. - SHIFT-TAB ...................... ESC-TAB Complete filename & reverse cycle. - ctrl-L ................................... Complete filename, list all. From fc6ccba32a39d0e160deded5e8274a8f52b5476c Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 31 Dec 2025 16:49:07 +0400 Subject: [PATCH 24/56] Add DJANGO_SETTINGS_MODULE to job configs --- deployment/terraform/job-definitions/default.json | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/terraform/job-definitions/default.json b/deployment/terraform/job-definitions/default.json index b0ef2d114..62c8c980c 100644 --- a/deployment/terraform/job-definitions/default.json +++ b/deployment/terraform/job-definitions/default.json @@ -10,6 +10,7 @@ { "name": "POSTGRES_USER", "value": "${postgres_user}" }, { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, + { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, { "name": "DJANGO_ENV", "value": "${environment}" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, From 831a6a178a3de95d70929ed47ee7afb0ea476ed3 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Thu, 1 Jan 2026 17:07:59 +0400 Subject: [PATCH 25/56] Add logs to the list parsing --- .../lib/parsers/source_parser_xlsx.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/django/contricleaner/lib/parsers/source_parser_xlsx.py b/src/django/contricleaner/lib/parsers/source_parser_xlsx.py index b118a69e4..aad17bf23 100644 --- a/src/django/contricleaner/lib/parsers/source_parser_xlsx.py +++ b/src/django/contricleaner/lib/parsers/source_parser_xlsx.py @@ -4,6 +4,8 @@ from openpyxl.utils import get_column_letter from openpyxl.worksheet.worksheet import Worksheet from django.core.files.base import File +from django.conf import settings +import os from contricleaner.lib.parsers.abstractions.source_parser import SourceParser from contricleaner.lib.parsers.abstractions.file_parser import FileParser @@ -21,6 +23,18 @@ def get_parsed_rows(self) -> List[dict]: @staticmethod def _parse(file: File) -> List[dict]: try: + logger.info( + "XLSX parse: file.name=%s storage=%s storage_class=%s " + "DEFAULT_FILE_STORAGE=%s AWS_STORAGE_BUCKET_NAME=%s " + "DJANGO_SETTINGS_MODULE=%s DEBUG=%s", + getattr(file, "name", ""), + getattr(file, "storage", None), + getattr(getattr(file, "storage", None), "__class__", None), + getattr(settings, "DEFAULT_FILE_STORAGE", None), + getattr(settings, "AWS_STORAGE_BUCKET_NAME", None), + os.getenv("DJANGO_SETTINGS_MODULE"), + settings.DEBUG, + ) worksheet = SourceParserXLSX.__get_xlsx_sheet(file) # openpyxl package is 1-indexed From 2c32b24362396313fb080936c9bc2e4b1a61a8e6 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Thu, 1 Jan 2026 18:05:28 +0400 Subject: [PATCH 26/56] Add DJANGO_SETTINGS_MODULE: oar.settings to all job definitions --- deployment/terraform/job-definitions/db_sync.json | 1 + .../terraform/job-definitions/direct_data_load.json | 1 + .../terraform/job-definitions/export_csv.json | 1 + .../terraform/job-definitions/notifications.json | 1 + src/django/oar/settings.py | 13 +++---------- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/deployment/terraform/job-definitions/db_sync.json b/deployment/terraform/job-definitions/db_sync.json index bc35bd18a..80d45e0b1 100644 --- a/deployment/terraform/job-definitions/db_sync.json +++ b/deployment/terraform/job-definitions/db_sync.json @@ -22,6 +22,7 @@ { "name": "POSTGRES_USER", "value": "${postgres_user}" }, { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, + { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, { "name": "DJANGO_ENV", "value": "${environment}" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, diff --git a/deployment/terraform/job-definitions/direct_data_load.json b/deployment/terraform/job-definitions/direct_data_load.json index afba6c904..86d174976 100644 --- a/deployment/terraform/job-definitions/direct_data_load.json +++ b/deployment/terraform/job-definitions/direct_data_load.json @@ -34,6 +34,7 @@ { "name": "POSTGRES_USER", "value": "${postgres_user}" }, { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, + { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, { "name": "BATCH_MODE", "value": "True" }, diff --git a/deployment/terraform/job-definitions/export_csv.json b/deployment/terraform/job-definitions/export_csv.json index 1afee777a..e2cd8ec39 100644 --- a/deployment/terraform/job-definitions/export_csv.json +++ b/deployment/terraform/job-definitions/export_csv.json @@ -15,6 +15,7 @@ { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, { "name": "BATCH_MODE", "value": "True" }, { "name": "DJANGO_ENV", "value": "${environment}" }, + { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, { "name": "DJANGO_SECRET_KEY", "value": "${django_secret_key}" }, { "name": "GOOGLE_SERVER_SIDE_API_KEY", "value": "${google_server_side_api_key}" }, { "name": "OAR_CLIENT_KEY", "value": "${oar_client_key}" }, diff --git a/deployment/terraform/job-definitions/notifications.json b/deployment/terraform/job-definitions/notifications.json index 4973a12d1..a5fc70691 100644 --- a/deployment/terraform/job-definitions/notifications.json +++ b/deployment/terraform/job-definitions/notifications.json @@ -11,6 +11,7 @@ { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, { "name": "DJANGO_ENV", "value": "${environment}" }, + { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, { "name": "DJANGO_SECRET_KEY", "value": "${django_secret_key}" }, diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 53e6b5f27..97f14dff9 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -614,17 +614,10 @@ TESTING = 'test' in sys.argv -# Always force S3 when a bucket is configured (except during tests). This avoids -# accidentally falling back to local filesystem when DEBUG is False but an env -# is mis-set. Still allow Local+tests to use the default. -AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') - -# Force S3-backed storage whenever a bucket is provided (including non-test -# local runs), so default_storage/Model FileField storage cannot fall back to -# the filesystem. -if AWS_STORAGE_BUCKET_NAME: +if not DEBUG or (AWS_S3_ENDPOINT_URL and not TESTING): DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - FILE_UPLOAD_STORAGE = DEFAULT_FILE_STORAGE + +AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') if AWS_STORAGE_BUCKET_NAME is None and not DEBUG: raise ImproperlyConfigured( From b784da856788cd75145c0bbbf24c8d827b5b7660 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Thu, 1 Jan 2026 22:04:52 +0400 Subject: [PATCH 27/56] Introduce FILE_UPLOAD_STORAGE --- src/django/oar/settings.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 97f14dff9..90f8affb9 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -614,11 +614,13 @@ TESTING = 'test' in sys.argv -if not DEBUG or (AWS_S3_ENDPOINT_URL and not TESTING): - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') +# Force S3-backed storage whenever a bucket is provided (unless running tests). +if AWS_STORAGE_BUCKET_NAME and not TESTING: + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + FILE_UPLOAD_STORAGE = DEFAULT_FILE_STORAGE + if AWS_STORAGE_BUCKET_NAME is None and not DEBUG: raise ImproperlyConfigured( 'Invalid AWS_STORAGE_BUCKET_NAME provided, must be set in the environment' From 9cc14250accbc34aefe287cf01e942a5a04b8e95 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Thu, 1 Jan 2026 22:46:43 +0400 Subject: [PATCH 28/56] Explicit add of DEFAULT_FILE_STORAGE --- deployment/terraform/job-definitions/db_sync.json | 1 + deployment/terraform/job-definitions/default.json | 1 + deployment/terraform/job-definitions/direct_data_load.json | 1 + deployment/terraform/job-definitions/export_csv.json | 1 + deployment/terraform/job-definitions/notifications.json | 1 + 5 files changed, 5 insertions(+) diff --git a/deployment/terraform/job-definitions/db_sync.json b/deployment/terraform/job-definitions/db_sync.json index 80d45e0b1..09d95bcac 100644 --- a/deployment/terraform/job-definitions/db_sync.json +++ b/deployment/terraform/job-definitions/db_sync.json @@ -23,6 +23,7 @@ { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, + { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "DJANGO_ENV", "value": "${environment}" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, diff --git a/deployment/terraform/job-definitions/default.json b/deployment/terraform/job-definitions/default.json index 62c8c980c..43df12ddf 100644 --- a/deployment/terraform/job-definitions/default.json +++ b/deployment/terraform/job-definitions/default.json @@ -11,6 +11,7 @@ { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, + { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "DJANGO_ENV", "value": "${environment}" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, diff --git a/deployment/terraform/job-definitions/direct_data_load.json b/deployment/terraform/job-definitions/direct_data_load.json index 86d174976..6e85e5d3d 100644 --- a/deployment/terraform/job-definitions/direct_data_load.json +++ b/deployment/terraform/job-definitions/direct_data_load.json @@ -35,6 +35,7 @@ { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, + { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, { "name": "BATCH_MODE", "value": "True" }, diff --git a/deployment/terraform/job-definitions/export_csv.json b/deployment/terraform/job-definitions/export_csv.json index e2cd8ec39..5f8586d4f 100644 --- a/deployment/terraform/job-definitions/export_csv.json +++ b/deployment/terraform/job-definitions/export_csv.json @@ -16,6 +16,7 @@ { "name": "BATCH_MODE", "value": "True" }, { "name": "DJANGO_ENV", "value": "${environment}" }, { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, + { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "DJANGO_SECRET_KEY", "value": "${django_secret_key}" }, { "name": "GOOGLE_SERVER_SIDE_API_KEY", "value": "${google_server_side_api_key}" }, { "name": "OAR_CLIENT_KEY", "value": "${oar_client_key}" }, diff --git a/deployment/terraform/job-definitions/notifications.json b/deployment/terraform/job-definitions/notifications.json index a5fc70691..878e2bb8b 100644 --- a/deployment/terraform/job-definitions/notifications.json +++ b/deployment/terraform/job-definitions/notifications.json @@ -12,6 +12,7 @@ { "name": "POSTGRES_DB", "value": "${postgres_db}" }, { "name": "DJANGO_ENV", "value": "${environment}" }, { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, + { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, { "name": "DJANGO_SECRET_KEY", "value": "${django_secret_key}" }, From 40a9b42ef63b9d664e0f78101e0ecebc98279206 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 2 Jan 2026 09:52:44 +0400 Subject: [PATCH 29/56] Refactor storage init settings --- .../contricleaner/lib/parsers/source_parser_xlsx.py | 4 ++++ src/django/oar/settings.py | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/django/contricleaner/lib/parsers/source_parser_xlsx.py b/src/django/contricleaner/lib/parsers/source_parser_xlsx.py index aad17bf23..6920d4ead 100644 --- a/src/django/contricleaner/lib/parsers/source_parser_xlsx.py +++ b/src/django/contricleaner/lib/parsers/source_parser_xlsx.py @@ -5,6 +5,7 @@ from openpyxl.worksheet.worksheet import Worksheet from django.core.files.base import File from django.conf import settings +from django.core.files.storage import default_storage import os from contricleaner.lib.parsers.abstractions.source_parser import SourceParser @@ -25,11 +26,14 @@ def _parse(file: File) -> List[dict]: try: logger.info( "XLSX parse: file.name=%s storage=%s storage_class=%s " + "default_storage=%s default_storage_class=%s " "DEFAULT_FILE_STORAGE=%s AWS_STORAGE_BUCKET_NAME=%s " "DJANGO_SETTINGS_MODULE=%s DEBUG=%s", getattr(file, "name", ""), getattr(file, "storage", None), getattr(getattr(file, "storage", None), "__class__", None), + default_storage, + default_storage.__class__, getattr(settings, "DEFAULT_FILE_STORAGE", None), getattr(settings, "AWS_STORAGE_BUCKET_NAME", None), os.getenv("DJANGO_SETTINGS_MODULE"), diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 90f8affb9..55c14d369 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -619,7 +619,13 @@ # Force S3-backed storage whenever a bucket is provided (unless running tests). if AWS_STORAGE_BUCKET_NAME and not TESTING: DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - FILE_UPLOAD_STORAGE = DEFAULT_FILE_STORAGE + # Explicitly set STORAGES.default to avoid Django falling back to the + # filesystem default_storage (per Django 6 storage API docs). + STORAGES = { + "default": { + "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", + }, + } if AWS_STORAGE_BUCKET_NAME is None and not DEBUG: raise ImproperlyConfigured( From f0b818deaf8f489343b83c933f3fdfee1bdc8de3 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 2 Jan 2026 10:45:28 +0400 Subject: [PATCH 30/56] Updated storage settings --- src/django/api/views/facility/facility_list_view_set.py | 6 ++++++ src/django/oar/settings.py | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/django/api/views/facility/facility_list_view_set.py b/src/django/api/views/facility/facility_list_view_set.py index c79d2400e..31245df89 100644 --- a/src/django/api/views/facility/facility_list_view_set.py +++ b/src/django/api/views/facility/facility_list_view_set.py @@ -161,6 +161,12 @@ def create(self, request): replaces=replaces, match_responsibility=contributor.match_responsibility) log.info(f'[List Upload] FacilityList created. ID {new_list.id}.') + log.info( + '[List Upload] File saved: name=%s bucket=%s storage=%s', + uploaded_file.name, + # getattr(settings, "AWS_STORAGE_BUCKET_NAME", None), + # getattr(settings, "DEFAULT_FILE_STORAGE", None), + ) source = Source.objects.create( contributor=contributor, diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 55c14d369..e323a2d76 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -615,16 +615,20 @@ TESTING = 'test' in sys.argv AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') +# For local/MinIO dev: if endpoint задан и бакет не передан, подставляем локальный. +if DEBUG and AWS_S3_ENDPOINT_URL and not AWS_STORAGE_BUCKET_NAME: + AWS_STORAGE_BUCKET_NAME = 'local-dev-bucket' # Force S3-backed storage whenever a bucket is provided (unless running tests). if AWS_STORAGE_BUCKET_NAME and not TESTING: DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - # Explicitly set STORAGES.default to avoid Django falling back to the - # filesystem default_storage (per Django 6 storage API docs). STORAGES = { "default": { "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, } if AWS_STORAGE_BUCKET_NAME is None and not DEBUG: From 2ae400a679e9eddffa7c22a1d64524a519667cf4 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 2 Jan 2026 12:20:10 +0400 Subject: [PATCH 31/56] Revome redundant logs and settings --- .github/workflows/deploy_to_aws.yml | 4 ---- deployment/infra | 11 ++++----- .../terraform/job-definitions/db_sync.json | 2 -- .../terraform/job-definitions/default.json | 2 -- .../job-definitions/direct_data_load.json | 2 -- .../terraform/job-definitions/export_csv.json | 2 -- .../job-definitions/notifications.json | 2 -- .../views/facility/facility_list_view_set.py | 6 ----- .../lib/parsers/source_parser_xlsx.py | 23 ------------------- src/django/oar/settings.py | 11 +++------ 10 files changed, 8 insertions(+), 57 deletions(-) diff --git a/.github/workflows/deploy_to_aws.yml b/.github/workflows/deploy_to_aws.yml index 0a1ced0b7..c74693c04 100644 --- a/.github/workflows/deploy_to_aws.yml +++ b/.github/workflows/deploy_to_aws.yml @@ -62,8 +62,6 @@ jobs: needs: check-base if: needs.check-base.outputs.skip == 'false' runs-on: ubuntu-latest - env: - IMAGE_TAG: ${{ github.sha }} environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') }} steps: - name: Get Environment Name for ${{ vars.ENV_NAME }} @@ -181,8 +179,6 @@ jobs: apply: needs: [init-and-plan, detach-waf-if-needed] runs-on: ubuntu-latest - env: - IMAGE_TAG: ${{ github.sha }} environment: ${{ inputs.deploy-env || (github.ref_name == 'main' && 'Development') || (startsWith(github.ref_name, 'releases/') && 'Pre-prod') }} if: ${{ inputs.deploy-plan-only == false }} steps: diff --git a/deployment/infra b/deployment/infra index 5aa782606..e755efc8f 100755 --- a/deployment/infra +++ b/deployment/infra @@ -12,16 +12,15 @@ Execute Terraform subcommands with remote state management. " } -# Prefer an explicit IMAGE_TAG (e.g., from CI), otherwise fall back to a provided -# GIT_COMMIT or the current repo commit. Always shorten to 7 characters to match -# our ECR tagging convention. -if [[ -n "${IMAGE_TAG}" ]]; then - GIT_COMMIT="${IMAGE_TAG:0:7}" -elif [[ -n "${GIT_COMMIT}" ]]; then +# # Temporary solution +# GIT_COMMIT="latest" + +if [[ -n "${GIT_COMMIT}" ]]; then GIT_COMMIT="${GIT_COMMIT:0:7}" else SHORT_SHA="$(git rev-parse --short HEAD)" GIT_COMMIT="${SHORT_SHA:0:7}" + fi if [ "${BASH_SOURCE[0]}" = "${0}" ]; then diff --git a/deployment/terraform/job-definitions/db_sync.json b/deployment/terraform/job-definitions/db_sync.json index 09d95bcac..bc35bd18a 100644 --- a/deployment/terraform/job-definitions/db_sync.json +++ b/deployment/terraform/job-definitions/db_sync.json @@ -22,8 +22,6 @@ { "name": "POSTGRES_USER", "value": "${postgres_user}" }, { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, - { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, - { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "DJANGO_ENV", "value": "${environment}" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, diff --git a/deployment/terraform/job-definitions/default.json b/deployment/terraform/job-definitions/default.json index 43df12ddf..b0ef2d114 100644 --- a/deployment/terraform/job-definitions/default.json +++ b/deployment/terraform/job-definitions/default.json @@ -10,8 +10,6 @@ { "name": "POSTGRES_USER", "value": "${postgres_user}" }, { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, - { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, - { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "DJANGO_ENV", "value": "${environment}" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, diff --git a/deployment/terraform/job-definitions/direct_data_load.json b/deployment/terraform/job-definitions/direct_data_load.json index 6e85e5d3d..afba6c904 100644 --- a/deployment/terraform/job-definitions/direct_data_load.json +++ b/deployment/terraform/job-definitions/direct_data_load.json @@ -34,8 +34,6 @@ { "name": "POSTGRES_USER", "value": "${postgres_user}" }, { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, - { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, - { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, { "name": "BATCH_MODE", "value": "True" }, diff --git a/deployment/terraform/job-definitions/export_csv.json b/deployment/terraform/job-definitions/export_csv.json index 5f8586d4f..1afee777a 100644 --- a/deployment/terraform/job-definitions/export_csv.json +++ b/deployment/terraform/job-definitions/export_csv.json @@ -15,8 +15,6 @@ { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, { "name": "BATCH_MODE", "value": "True" }, { "name": "DJANGO_ENV", "value": "${environment}" }, - { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, - { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "DJANGO_SECRET_KEY", "value": "${django_secret_key}" }, { "name": "GOOGLE_SERVER_SIDE_API_KEY", "value": "${google_server_side_api_key}" }, { "name": "OAR_CLIENT_KEY", "value": "${oar_client_key}" }, diff --git a/deployment/terraform/job-definitions/notifications.json b/deployment/terraform/job-definitions/notifications.json index 878e2bb8b..4973a12d1 100644 --- a/deployment/terraform/job-definitions/notifications.json +++ b/deployment/terraform/job-definitions/notifications.json @@ -11,8 +11,6 @@ { "name": "POSTGRES_PASSWORD", "value": "${postgres_password}" }, { "name": "POSTGRES_DB", "value": "${postgres_db}" }, { "name": "DJANGO_ENV", "value": "${environment}" }, - { "name": "DJANGO_SETTINGS_MODULE", "value": "oar.settings" }, - { "name": "DEFAULT_FILE_STORAGE", "value": "storages.backends.s3boto3.S3Boto3Storage" }, { "name": "BATCH_JOB_QUEUE_NAME", "value": "${batch_job_queue_name}" }, { "name": "BATCH_JOB_DEF_NAME", "value": "${batch_job_def_name}" }, { "name": "DJANGO_SECRET_KEY", "value": "${django_secret_key}" }, diff --git a/src/django/api/views/facility/facility_list_view_set.py b/src/django/api/views/facility/facility_list_view_set.py index 31245df89..c79d2400e 100644 --- a/src/django/api/views/facility/facility_list_view_set.py +++ b/src/django/api/views/facility/facility_list_view_set.py @@ -161,12 +161,6 @@ def create(self, request): replaces=replaces, match_responsibility=contributor.match_responsibility) log.info(f'[List Upload] FacilityList created. ID {new_list.id}.') - log.info( - '[List Upload] File saved: name=%s bucket=%s storage=%s', - uploaded_file.name, - # getattr(settings, "AWS_STORAGE_BUCKET_NAME", None), - # getattr(settings, "DEFAULT_FILE_STORAGE", None), - ) source = Source.objects.create( contributor=contributor, diff --git a/src/django/contricleaner/lib/parsers/source_parser_xlsx.py b/src/django/contricleaner/lib/parsers/source_parser_xlsx.py index 6920d4ead..e1a8ef704 100644 --- a/src/django/contricleaner/lib/parsers/source_parser_xlsx.py +++ b/src/django/contricleaner/lib/parsers/source_parser_xlsx.py @@ -4,18 +4,11 @@ from openpyxl.utils import get_column_letter from openpyxl.worksheet.worksheet import Worksheet from django.core.files.base import File -from django.conf import settings -from django.core.files.storage import default_storage -import os from contricleaner.lib.parsers.abstractions.source_parser import SourceParser from contricleaner.lib.parsers.abstractions.file_parser import FileParser from contricleaner.lib.exceptions.parsing_error import ParsingError -import logging -import traceback -logger = logging.getLogger(__name__) - class SourceParserXLSX(SourceParser, FileParser): def get_parsed_rows(self) -> List[dict]: @@ -24,21 +17,6 @@ def get_parsed_rows(self) -> List[dict]: @staticmethod def _parse(file: File) -> List[dict]: try: - logger.info( - "XLSX parse: file.name=%s storage=%s storage_class=%s " - "default_storage=%s default_storage_class=%s " - "DEFAULT_FILE_STORAGE=%s AWS_STORAGE_BUCKET_NAME=%s " - "DJANGO_SETTINGS_MODULE=%s DEBUG=%s", - getattr(file, "name", ""), - getattr(file, "storage", None), - getattr(getattr(file, "storage", None), "__class__", None), - default_storage, - default_storage.__class__, - getattr(settings, "DEFAULT_FILE_STORAGE", None), - getattr(settings, "AWS_STORAGE_BUCKET_NAME", None), - os.getenv("DJANGO_SETTINGS_MODULE"), - settings.DEBUG, - ) worksheet = SourceParserXLSX.__get_xlsx_sheet(file) # openpyxl package is 1-indexed @@ -69,7 +47,6 @@ def _parse(file: File) -> List[dict]: return rows except Exception: - logger.error(f"Error parsing XLSX file: {traceback.format_exc()}") raise ParsingError( 'There was an error within your file and our team needs to ' 'take a look. Please send your file to ' diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index e323a2d76..7d1dcca17 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -614,23 +614,18 @@ TESTING = 'test' in sys.argv -AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') -# For local/MinIO dev: if endpoint задан и бакет не передан, подставляем локальный. -if DEBUG and AWS_S3_ENDPOINT_URL and not AWS_STORAGE_BUCKET_NAME: - AWS_STORAGE_BUCKET_NAME = 'local-dev-bucket' - -# Force S3-backed storage whenever a bucket is provided (unless running tests). -if AWS_STORAGE_BUCKET_NAME and not TESTING: +if not DEBUG or (AWS_S3_ENDPOINT_URL and not TESTING): DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' STORAGES = { "default": { - "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", + "BACKEND": DEFAULT_FILE_STORAGE, }, "staticfiles": { "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", }, } +AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') if AWS_STORAGE_BUCKET_NAME is None and not DEBUG: raise ImproperlyConfigured( 'Invalid AWS_STORAGE_BUCKET_NAME provided, must be set in the environment' From 0c74d700724f9c6a64919df65148278ae5c67886 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 2 Jan 2026 12:21:56 +0400 Subject: [PATCH 32/56] Remove legacy CKEDITOR_CONFIGS --- src/django/oar/settings.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 7d1dcca17..f2e62d14d 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -521,24 +521,6 @@ }, } -CKEDITOR_CONFIGS = { - 'default': { - 'toolbar': [ - ['Bold', 'Italic', 'Underline', 'Strike'], - ['NumberedList', 'BulletedList'], - ['Link', 'Unlink'], - ['RemoveFormat', 'Source'], - ], - 'height': 250, - 'width': '100%', - 'removePlugins': 'uploadimage,uploadfile,image,flash,smiley', - 'enterMode': 2, - 'shiftEnterMode': 2, - 'autoParagraph': False, - 'fillEmptyBlocks': False, - } -} - # Application settings MAX_UPLOADED_FILE_SIZE_IN_BYTES = 5242880 MAX_ATTACHMENT_SIZE_IN_BYTES = 5 * 1024 * 1024 # 5 MB From 9d1f73c05443eddbe6632af0552c1a5564a9381f Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 2 Jan 2026 12:43:25 +0400 Subject: [PATCH 33/56] Use REST_AUTH instead of REST_AUTH_SERIALIZER, use tuple unpacking for DEFAULT_NAMES --- src/django/oar/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index f2e62d14d..e0c3c1abf 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -26,7 +26,7 @@ # Compatibility: allow legacy `index_together` meta option so immutable older # migrations (e.g., Django<5 era) continue to load without edits. if 'index_together' not in options.DEFAULT_NAMES: - options.DEFAULT_NAMES = options.DEFAULT_NAMES + ('index_together',) + options.DEFAULT_NAMES = (*options.DEFAULT_NAMES, 'index_together') # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ @@ -173,7 +173,7 @@ SESSION_COOKIE_SECURE = True -REST_AUTH_SERIALIZERS = { +REST_AUTH = { 'USER_DETAILS_SERIALIZER': 'api.serializers.UserSerializer', 'PASSWORD_RESET_SERIALIZER': 'api.serializers.UserPasswordResetSerializer', 'PASSWORD_RESET_CONFIRM_SERIALIZER': 'api.serializers.UserPasswordResetConfirmSerializer', @@ -306,7 +306,7 @@ # https://docs.djangoproject.com/en/3.2/topics/cache/ MEMCACHED_LOCATION = f"{os.getenv('CACHE_HOST')}:{os.getenv('CACHE_PORT')}" -# Use PyLibMCCache everywhere; django_elasticache is incompatible with Django 4 +# Use PyLibMCCache everywhere; django_elasticache is incompatible with Django 5 # because it still imports smart_text. This keeps throttling cache working in # jobs/containers that upgrade Django. CACHE_BACKEND = 'django.core.cache.backends.memcached.PyLibMCCache' From b94c06b5c101a585b225d8b90c28be40dc095b2d Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Thu, 8 Jan 2026 16:39:47 +0400 Subject: [PATCH 34/56] Resolve merge conflicts --- .../0197_add_mit_livingwage_partner_field.py | 61 ------------------- 1 file changed, 61 deletions(-) delete mode 100644 src/django/api/migrations/0197_add_mit_livingwage_partner_field.py diff --git a/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py b/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py deleted file mode 100644 index 3430281b9..000000000 --- a/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py +++ /dev/null @@ -1,61 +0,0 @@ -# Generated by Django 3.2.17 on 2025-12-16 - -from django.db import migrations - - -def create_mit_living_wage_partner_field(apps, schema_editor): - ''' - Create the mit_living_wage partner field. - ''' - partner_field = apps.get_model('api', 'PartnerField') - - if partner_field.objects.filter(name='mit_living_wage').exists(): - raise ValueError( - 'The mit_living_wage partner field already exists.' - ) - - json_schema = { - "type": "object", - "title": "Some Data", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "properties": { - "county_id": { - "type": "string", - "title": "County Id", - "format": "uri-reference" - } - } - } - - partner_field.objects.create( - name='mit_living_wage', - type='object', - json_schema=json_schema, - base_url='https://livingwage.mit.edu/counties/', - display_text='View Living Wage Data for this County', - active=True, - system_field=True - ) - - -def remove_mit_living_wage_partner_field(apps, schema_editor): - ''' - Remove the mit_living_wage partner field. - ''' - partner_field = apps.get_model('api', 'PartnerField') - partner_field.objects.filter(name='mit_living_wage').delete() - - -class Migration(migrations.Migration): - - dependencies = [ - ('api', '0196_create_and_populate_us_county_tigerline'), - ] - - operations = [ - migrations.RunPython( - create_mit_living_wage_partner_field, - reverse_code=remove_mit_living_wage_partner_field - ), - ] - From c85c20c4b5becb3405036f2f95451690f1cafb15 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Thu, 8 Jan 2026 16:40:46 +0400 Subject: [PATCH 35/56] Resolve merge conflicts --- .../0197_add_mit_livingwage_partner_field.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 src/django/api/migrations/0197_add_mit_livingwage_partner_field.py diff --git a/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py b/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py new file mode 100644 index 000000000..3430281b9 --- /dev/null +++ b/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py @@ -0,0 +1,61 @@ +# Generated by Django 3.2.17 on 2025-12-16 + +from django.db import migrations + + +def create_mit_living_wage_partner_field(apps, schema_editor): + ''' + Create the mit_living_wage partner field. + ''' + partner_field = apps.get_model('api', 'PartnerField') + + if partner_field.objects.filter(name='mit_living_wage').exists(): + raise ValueError( + 'The mit_living_wage partner field already exists.' + ) + + json_schema = { + "type": "object", + "title": "Some Data", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "county_id": { + "type": "string", + "title": "County Id", + "format": "uri-reference" + } + } + } + + partner_field.objects.create( + name='mit_living_wage', + type='object', + json_schema=json_schema, + base_url='https://livingwage.mit.edu/counties/', + display_text='View Living Wage Data for this County', + active=True, + system_field=True + ) + + +def remove_mit_living_wage_partner_field(apps, schema_editor): + ''' + Remove the mit_living_wage partner field. + ''' + partner_field = apps.get_model('api', 'PartnerField') + partner_field.objects.filter(name='mit_living_wage').delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0196_create_and_populate_us_county_tigerline'), + ] + + operations = [ + migrations.RunPython( + create_mit_living_wage_partner_field, + reverse_code=remove_mit_living_wage_partner_field + ), + ] + From c324f4c852b85773309643608e163f40af050bca Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 9 Jan 2026 11:29:34 +0400 Subject: [PATCH 36/56] Refactor migration graph, update release notes --- doc/release/RELEASE-NOTES.md | 28 +++++++-- ...create_and_populate_us_county_tigerline.py | 62 +++++++++++++++++++ ... 0195_add_mit_livingwage_partner_field.py} | 2 +- ..._switch_partner_field_source_by_editor.py} | 2 +- ...event_index.py => 0197_add_event_index.py} | 2 +- 5 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 src/django/api/migrations/0194_create_and_populate_us_county_tigerline.py rename src/django/api/migrations/{0197_add_mit_livingwage_partner_field.py => 0195_add_mit_livingwage_partner_field.py} (96%) rename src/django/api/migrations/{0194_switch_partner_field_source_by_editor.py => 0196_switch_partner_field_source_by_editor.py} (89%) rename src/django/api/migrations/{0195_add_event_index.py => 0197_add_event_index.py} (88%) diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index d49370fc3..e22c5380a 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -3,6 +3,30 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). The format is based on the `RELEASE-NOTES-TEMPLATE.md` file. +## Release 2.19.0 + +## Introduction +* Product name: Open Supply Hub +* Release date: *Provide release date* + +### Database changes + +#### Migrations +* 0196_switch_partner_field_source_by_editor.py - Migrates `PartnerField.source_by` to CKEditor5 so the rich-text content works after replacing `django-ckeditor` with `django-ckeditor-5` (required for Django 5); keeps existing source descriptions editable with formatting and links. +* 0197_add_event_index.py - Adds an explicit index on `Event(content_type, object_id)` to replace the legacy `index_together` removed in Django 5, keeping the existing schema intact without editing old migrations. + +### What's new +* [OSDEV-814](https://opensupplyhub.atlassian.net/browse/OSDEV-814) - Major upgrade of Django application backend services: + * Upgraded Python from `3.8` to `3.11`. + * Upgraded Django from `3.2.17` to `5.1.3`. + * Upgraded Python and Django packages to maintain compatibility. + +### Release instructions +* Ensure that the following commands are included in the `post_deployment` command: + * `migrate` + * `reindex_database` + + ## Release 2.18.0 ## Introduction @@ -34,10 +58,6 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * [OSDEV-2047](https://opensupplyhub.atlassian.net/browse/OSDEV-2047) - Removed all Terraform configurations and ECS service definitions related to the deprecated standalone ContriCleaner service. Cleaned up the repository by deleting unused code and references, as ContriCleaner now operates exclusively as an internal Django library. * [OSDEV-2318](https://opensupplyhub.atlassian.net/browse/OSDEV-2318) - Updated Terraform version from `1.5` to `1.13.3`. Upgraded Kafka from the `3.4.0` to `3.9.0` to align with the current AWS MSK supported version. * [OSDEV-2328](https://opensupplyhub.atlassian.net/browse/OSDEV-2328) - Added CloudFront caching for the facilities and production-location OS ID endpoints, refactored the Terraform config to use endpoint-specific TTL variables, and set per-environment durations (30 minutes for Prod/RBA/Preprod/Staging, 1 minute for Dev/Test). CloudFront still caches only GET/HEAD/OPTIONS while allowing all HTTP methods to reach the origin. -* [OSDEV-814](https://opensupplyhub.atlassian.net/browse/OSDEV-814) - Major upgrade of Django application backend services: - * Upgraded Python from `3.8` to `3.11`. - * Upgraded Django from `3.2.17` to `5.1.3`. - * Upgraded Python and Django packages to maintain compatibility. ### Bugfix * [OSDEV-2047](https://opensupplyhub.atlassian.net/browse/OSDEV-2047) - Previously, there were two security groups with the same tags: one for the Django app and another for ContriCleaner. After removing the ContriCleaner service infrastructure, a bug was eliminated in which the Django CLI task in the Development environment selected the wrong security group - the one without database access, belonging to ContriCleaner - which prevented Django management commands from running against the database in the Development environment. diff --git a/src/django/api/migrations/0194_create_and_populate_us_county_tigerline.py b/src/django/api/migrations/0194_create_and_populate_us_county_tigerline.py new file mode 100644 index 000000000..bda053d45 --- /dev/null +++ b/src/django/api/migrations/0194_create_and_populate_us_county_tigerline.py @@ -0,0 +1,62 @@ +# Generated by Django 3.2.17 on 2025-12-16 + +from django.db import migrations, models +from django.contrib.gis.db import models as gis_models +from api.migrations._tigerline_helper import ( + populate_tigerline_data, + clear_tigerline_data +) + +S3_CSV_KEY = 'data/us_county_tigerline_2021.csv' + + +def populate_tigerline_data_wrapper(apps, schema_editor): + ''' + Populate the USCountyTigerline table with 2021 data from CSV file. + Geometry data is in EPSG:4326 (WGS84). + ''' + populate_tigerline_data( + apps, + s3_key=S3_CSV_KEY, + source_srid=4326, + clear_existing=False, + production_envs=['Production', 'Preprod', 'Staging'] + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0193_populate_wage_indicator_data'), + ] + + operations = [ + migrations.CreateModel( + name='USCountyTigerline', + fields=[ + ('geoid', models.CharField( + help_text='The Geo ID of the county (US)', + max_length=50, + primary_key=True + )), + ('name', models.CharField( + help_text='The name of the county', + max_length=255 + )), + ('geometry', gis_models.MultiPolygonField( + help_text='The MultiPolygon geometry of the county in EPSG:5070', + srid=5070 + )), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + options={ + 'verbose_name': 'County (US)', + 'verbose_name_plural': 'Counties (US)', + }, + ), + migrations.RunPython( + populate_tigerline_data_wrapper, + reverse_code=clear_tigerline_data + ), + ] diff --git a/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py b/src/django/api/migrations/0195_add_mit_livingwage_partner_field.py similarity index 96% rename from src/django/api/migrations/0197_add_mit_livingwage_partner_field.py rename to src/django/api/migrations/0195_add_mit_livingwage_partner_field.py index 3430281b9..36ce33ab1 100644 --- a/src/django/api/migrations/0197_add_mit_livingwage_partner_field.py +++ b/src/django/api/migrations/0195_add_mit_livingwage_partner_field.py @@ -49,7 +49,7 @@ def remove_mit_living_wage_partner_field(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('api', '0196_create_and_populate_us_county_tigerline'), + ('api', '0194_create_and_populate_us_county_tigerline'), ] operations = [ diff --git a/src/django/api/migrations/0194_switch_partner_field_source_by_editor.py b/src/django/api/migrations/0196_switch_partner_field_source_by_editor.py similarity index 89% rename from src/django/api/migrations/0194_switch_partner_field_source_by_editor.py rename to src/django/api/migrations/0196_switch_partner_field_source_by_editor.py index ba0bf10d7..0f8db9326 100644 --- a/src/django/api/migrations/0194_switch_partner_field_source_by_editor.py +++ b/src/django/api/migrations/0196_switch_partner_field_source_by_editor.py @@ -5,7 +5,7 @@ class Migration(migrations.Migration): dependencies = [ - ('api', '0193_populate_wage_indicator_data'), + ('api', '0195_add_mit_livingwage_partner_field'), ] operations = [ diff --git a/src/django/api/migrations/0195_add_event_index.py b/src/django/api/migrations/0197_add_event_index.py similarity index 88% rename from src/django/api/migrations/0195_add_event_index.py rename to src/django/api/migrations/0197_add_event_index.py index ff5c62c6b..303fcfc0a 100644 --- a/src/django/api/migrations/0195_add_event_index.py +++ b/src/django/api/migrations/0197_add_event_index.py @@ -3,7 +3,7 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0194_switch_partner_field_source_by_editor"), + ("api", "0196_switch_partner_field_source_by_editor"), ] operations = [ From 29140b6e43dcec3091edd690db9665776f57bc25 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 9 Jan 2026 11:36:14 +0400 Subject: [PATCH 37/56] Remove duplicated migration --- ...create_and_populate_us_county_tigerline.py | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 src/django/api/migrations/0196_create_and_populate_us_county_tigerline.py diff --git a/src/django/api/migrations/0196_create_and_populate_us_county_tigerline.py b/src/django/api/migrations/0196_create_and_populate_us_county_tigerline.py deleted file mode 100644 index dab4d1248..000000000 --- a/src/django/api/migrations/0196_create_and_populate_us_county_tigerline.py +++ /dev/null @@ -1,62 +0,0 @@ -# Generated by Django 3.2.17 on 2025-12-16 - -from django.db import migrations, models -from django.contrib.gis.db import models as gis_models -from api.migrations._tigerline_helper import ( - populate_tigerline_data, - clear_tigerline_data -) - -S3_CSV_KEY = 'data/us_county_tigerline_2021.csv' - - -def populate_tigerline_data_wrapper(apps, schema_editor): - ''' - Populate the USCountyTigerline table with 2021 data from CSV file. - Geometry data is in EPSG:4326 (WGS84). - ''' - populate_tigerline_data( - apps, - s3_key=S3_CSV_KEY, - source_srid=4326, - clear_existing=False, - production_envs=['Production', 'Preprod', 'Staging'] - ) - - -class Migration(migrations.Migration): - - dependencies = [ - ('api', '0195_add_event_index'), - ] - - operations = [ - migrations.CreateModel( - name='USCountyTigerline', - fields=[ - ('geoid', models.CharField( - help_text='The Geo ID of the county (US)', - max_length=50, - primary_key=True - )), - ('name', models.CharField( - help_text='The name of the county', - max_length=255 - )), - ('geometry', gis_models.MultiPolygonField( - help_text='The MultiPolygon geometry of the county in EPSG:5070', - srid=5070 - )), - ('created_at', models.DateTimeField(auto_now_add=True)), - ('updated_at', models.DateTimeField(auto_now=True)), - ], - options={ - 'verbose_name': 'County (US)', - 'verbose_name_plural': 'Counties (US)', - }, - ), - migrations.RunPython( - populate_tigerline_data_wrapper, - reverse_code=clear_tigerline_data - ), - ] From 78b9330f8d7e9ab0b93bf49f3bbf9904e5b87648 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 21 Jan 2026 14:44:49 +0400 Subject: [PATCH 38/56] Drop the legacy index_together index in migration 0195 --- src/django/api/migrations/0197_add_event_index.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/django/api/migrations/0197_add_event_index.py b/src/django/api/migrations/0197_add_event_index.py index 303fcfc0a..82129acd5 100644 --- a/src/django/api/migrations/0197_add_event_index.py +++ b/src/django/api/migrations/0197_add_event_index.py @@ -7,6 +7,10 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RemoveIndex( + model_name="event", + name="api_event_content_type_id_object_id_idx", + ), migrations.AlterModelOptions( name="event", options={ @@ -19,4 +23,3 @@ class Migration(migrations.Migration): }, ), ] - From c46daf1c5717c32df9e56d2c0a2c1261ad0bdd56 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 21 Jan 2026 16:29:04 +0400 Subject: [PATCH 39/56] Remove legacy index and and use AddIndex to create the new api_event_content_object_idx --- .../api/migrations/0197_add_event_index.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/django/api/migrations/0197_add_event_index.py b/src/django/api/migrations/0197_add_event_index.py index 82129acd5..31085844c 100644 --- a/src/django/api/migrations/0197_add_event_index.py +++ b/src/django/api/migrations/0197_add_event_index.py @@ -11,15 +11,11 @@ class Migration(migrations.Migration): model_name="event", name="api_event_content_type_id_object_id_idx", ), - migrations.AlterModelOptions( - name="event", - options={ - "indexes": [ - models.Index( - fields=["content_type", "object_id"], - name="api_event_content_object_idx", - ) - ] - }, + migrations.AddIndex( + model_name="event", + index=models.Index( + fields=["content_type", "object_id"], + name="api_event_content_object_idx", + ), ), ] From 621e455bd3fc1d0c052c73c69f9f588ea1bf4922 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Wed, 21 Jan 2026 16:55:19 +0400 Subject: [PATCH 40/56] Update 0197_add_event_index to drop legacy index_together hashes defensively and avoid hard-coded names --- src/django/api/migrations/0197_add_event_index.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/django/api/migrations/0197_add_event_index.py b/src/django/api/migrations/0197_add_event_index.py index 31085844c..bf64aaf07 100644 --- a/src/django/api/migrations/0197_add_event_index.py +++ b/src/django/api/migrations/0197_add_event_index.py @@ -7,9 +7,12 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RemoveIndex( - model_name="event", - name="api_event_content_type_id_object_id_idx", + migrations.RunSQL( + sql=[ + "DROP INDEX IF EXISTS api_event_content_type_id_object_id_idx", + "DROP INDEX IF EXISTS api_event_content_type_id_object_id_b046420a_idx", + ], + reverse_sql=migrations.RunSQL.noop, ), migrations.AddIndex( model_name="event", From 352bd3676aa531f61dc09318d84fc12d2c2cd245 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 23 Jan 2026 10:54:27 +0400 Subject: [PATCH 41/56] Upgrade Django to v.5.2 --- src/django/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/django/requirements.txt b/src/django/requirements.txt index 5423068e4..c5c53c68a 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -39,7 +39,7 @@ django-slowtests==1.1.1 django-allauth==64.0.0 opensearch-py==2.8.0 gunicorn==22.0.0 -django==5.1.3 +django==5.2.10 psycopg2==2.9.9 gevent==24.2.1 google-api-python-client==2.156.0 From 4d1ea25ab15f37fa8d9392a5658a246645cd7f32 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 23 Jan 2026 11:30:40 +0400 Subject: [PATCH 42/56] Proper configs and import of ckeditor, remove DEFAULT_FILE_STORAGE --- src/django/oar/settings.py | 25 +++++++++++++++++-------- src/django/oar/urls.py | 1 + 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index e0c3c1abf..61f786a64 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -35,13 +35,23 @@ SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'secret') CKEDITOR_5_CONFIGS = { - 'default': { - 'toolbar': [ - 'heading', '|', - 'bold', 'italic', 'link', 'bulletedList', 'numberedList', - 'blockQuote', + "default": { + "toolbar": [ + "heading", "|", + "bold", "italic", "underline", "strikethrough", + "link", "|", + "bulletedList", "numberedList", "|", + "removeFormat", "|", + "undo", "redo", ], - }, + # Remove upload/media/table plugins to mirror the prior CKEditor 4 config + # that blocked image/file uploads and kept the toolbar minimal. + "removePlugins": [ + "CKBox", "CKFinderUploadAdapter", "EasyImage", + "Image", "ImageCaption", "ImageStyle", "ImageToolbar", + "ImageUpload", "MediaEmbed", "Table", "TableToolbar", + ], + } } # Set environment @@ -597,10 +607,9 @@ TESTING = 'test' in sys.argv if not DEBUG or (AWS_S3_ENDPOINT_URL and not TESTING): - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' STORAGES = { "default": { - "BACKEND": DEFAULT_FILE_STORAGE, + "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", }, "staticfiles": { "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", diff --git a/src/django/oar/urls.py b/src/django/oar/urls.py index 28a04b7a0..6f614e411 100644 --- a/src/django/oar/urls.py +++ b/src/django/oar/urls.py @@ -143,6 +143,7 @@ path('api/docs/', schema_view.with_ui('swagger'), name='schema-swagger-ui'), path('admin/', admin_site.urls), + path('ckeditor5/', include('django_ckeditor_5.urls')), path('health-check/', include('watchman.urls')), path('api-auth/', include('rest_framework.urls')), path('rest-auth/', include('dj_rest_auth.urls')), From fabcfa149c4add58af1467d1963abd218957b38d Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 23 Jan 2026 12:21:18 +0400 Subject: [PATCH 43/56] Upgrade dj-rest-auth, drf-yasg, django-ckeditor-5 --- src/django/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/django/requirements.txt b/src/django/requirements.txt index c5c53c68a..52bf65798 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -12,14 +12,14 @@ django-ecsmanage==2.0.1 django-elasticache==1.0.3 django-extensions==3.2.3 django-jsonview==2.0.0 -dj-rest-auth[with_social]==7.0.1 +dj-rest-auth[with_social]==7.0.2 django-simple-history==3.4.0 django-spa==0.3.5 django-storages==1.14.4 django-waffle==4.1.0 django-watchman==1.3.0 -drf-yasg==1.21.8 -django-ckeditor-5==0.2.13 +drf-yasg==1.21.14 +django-ckeditor-5==0.2.19 djangorestframework-gis==1.1.0 djangorestframework==3.15.2 flake8==7.1.0 From f87facba7524eb86209a893e161aba64785f8bf6 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 23 Jan 2026 12:22:54 +0400 Subject: [PATCH 44/56] Update static files backend settings --- src/django/oar/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 61f786a64..27667a000 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -612,7 +612,7 @@ "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", }, "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + "BACKEND": "spa.storage.SPAStaticFilesStorage", }, } From 35bb08f026f5b6ad93017675203aeb90a7034514 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 23 Jan 2026 14:44:19 +0400 Subject: [PATCH 45/56] Update release notes --- doc/release/RELEASE-NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index 1f671223f..912922567 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -18,7 +18,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html ### What's new * [OSDEV-814](https://opensupplyhub.atlassian.net/browse/OSDEV-814) - Major upgrade of Django application backend services: * Upgraded Python from `3.8` to `3.11`. - * Upgraded Django from `3.2.17` to `5.1.3`. + * Upgraded Django from `3.2.17` to `5.2.10`. * Upgraded Python and Django packages to maintain compatibility. From beb1d4bbf7132fed1208243876010b53484838b8 Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Tue, 27 Jan 2026 12:36:00 +0200 Subject: [PATCH 46/56] Set DEFAULT_FILE_STORAGE outside debug mode --- src/django/oar/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 27667a000..82505f565 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -606,10 +606,11 @@ TESTING = 'test' in sys.argv +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' if not DEBUG or (AWS_S3_ENDPOINT_URL and not TESTING): STORAGES = { "default": { - "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", + "BACKEND": DEFAULT_FILE_STORAGE, }, "staticfiles": { "BACKEND": "spa.storage.SPAStaticFilesStorage", From 7ba0861d77d31dc64db4bb2569068ec8dc7b19f1 Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Tue, 27 Jan 2026 12:57:39 +0200 Subject: [PATCH 47/56] Minor refactoring --- src/django/oar/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 82505f565..84988696a 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -606,8 +606,8 @@ TESTING = 'test' in sys.argv -DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' if not DEBUG or (AWS_S3_ENDPOINT_URL and not TESTING): + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' STORAGES = { "default": { "BACKEND": DEFAULT_FILE_STORAGE, @@ -618,6 +618,7 @@ } AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') + if AWS_STORAGE_BUCKET_NAME is None and not DEBUG: raise ImproperlyConfigured( 'Invalid AWS_STORAGE_BUCKET_NAME provided, must be set in the environment' From 4eef757f2f282e4ee073e74f72119d29cc968e99 Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Tue, 27 Jan 2026 13:38:56 +0200 Subject: [PATCH 48/56] Refactor storage settings --- src/django/oar/settings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 84988696a..f93ce5aaf 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -607,13 +607,12 @@ TESTING = 'test' in sys.argv if not DEBUG or (AWS_S3_ENDPOINT_URL and not TESTING): - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' STORAGES = { "default": { - "BACKEND": DEFAULT_FILE_STORAGE, + "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", }, "staticfiles": { - "BACKEND": "spa.storage.SPAStaticFilesStorage", + "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage", }, } From 9a0772e42fb9b61f55cd126cba809fd467e24b1e Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Tue, 27 Jan 2026 14:10:28 +0200 Subject: [PATCH 49/56] Fix storage condition for static files --- src/django/oar/settings.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index f93ce5aaf..3aaeb01ee 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -606,7 +606,13 @@ TESTING = 'test' in sys.argv -if not DEBUG or (AWS_S3_ENDPOINT_URL and not TESTING): +if DEBUG: + STORAGES = { + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, + } +else: STORAGES = { "default": { "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", From 166a51fedc07b4d248ceeda911c57359fa944fe4 Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Tue, 27 Jan 2026 14:43:07 +0200 Subject: [PATCH 50/56] Fix admin static manifest by running collectstatic with production storage in Docker build --- src/django/Dockerfile | 7 ++++++- src/django/oar/settings.py | 2 -- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/django/Dockerfile b/src/django/Dockerfile index dbc992e24..1eadfef3d 100644 --- a/src/django/Dockerfile +++ b/src/django/Dockerfile @@ -26,7 +26,12 @@ RUN set -ex \ COPY . /usr/local/src -RUN GOOGLE_SERVER_SIDE_API_KEY="" \ +RUN DJANGO_ENV=Production \ + BATCH_MODE=build \ + BATCH_JOB_QUEUE_NAME=build \ + BATCH_JOB_DEF_NAME=build \ + AWS_STORAGE_BUCKET_NAME=build \ + GOOGLE_SERVER_SIDE_API_KEY="" \ OAR_CLIENT_KEY="" \ HUBSPOT_API_KEY="" \ HUBSPOT_SUBSCRIPTION_ID="" \ diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 3aaeb01ee..fa07c21d3 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -406,8 +406,6 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'static') STATICFILES_DIRS = ((os.path.join(STATIC_ROOT, "static")),) -STATICFILES_STORAGE = 'spa.storage.SPAStaticFilesStorage' - # Watchman # https://github.com/mwarkentin/django-watchman From 825847114ac0d7768db52407033e0285f43a62b8 Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Tue, 27 Jan 2026 16:20:26 +0200 Subject: [PATCH 51/56] Bring back TESTING condition --- src/django/oar/settings.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index fa07c21d3..5c945ff0d 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -604,8 +604,16 @@ TESTING = 'test' in sys.argv -if DEBUG: +# Use filesystem storage when DEBUG or when running tests, so tests never use +# S3/MinIO (matches legacy DEFAULT_FILE_STORAGE behavior for CI/integration tests). +if DEBUG or TESTING: STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + "OPTIONS": { + "location": os.path.join(BASE_DIR, "media"), + }, + }, "staticfiles": { "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", }, From 4cf03ed6d280ad0bdcc8b0584c025c0dce7b748c Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Tue, 27 Jan 2026 16:31:19 +0200 Subject: [PATCH 52/56] Fix storage lookup for Debug mode while CI --- src/django/oar/settings.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 5c945ff0d..0cd412a31 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -604,9 +604,10 @@ TESTING = 'test' in sys.argv -# Use filesystem storage when DEBUG or when running tests, so tests never use -# S3/MinIO (matches legacy DEFAULT_FILE_STORAGE behavior for CI/integration tests). -if DEBUG or TESTING: +# Use filesystem for "default" when running tests (so tests never use S3/MinIO) or +# when DEBUG and no S3 endpoint (local without MinIO). Use S3 when not DEBUG or +# when DEBUG but AWS_S3_ENDPOINT_URL is set (e.g. reset_database with MinIO). +if (DEBUG and not AWS_S3_ENDPOINT_URL) or TESTING: STORAGES = { "default": { "BACKEND": "django.core.files.storage.FileSystemStorage", From 5d396d4753a12588a7ea677922cda552e66a0455 Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Tue, 27 Jan 2026 17:44:41 +0200 Subject: [PATCH 53/56] Keep only the valid identifiers and removing CKBox, EasyImage, and ImageUpload --- src/django/oar/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 0cd412a31..4f59a32d6 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -46,10 +46,10 @@ ], # Remove upload/media/table plugins to mirror the prior CKEditor 4 config # that blocked image/file uploads and kept the toolbar minimal. + # Only supported identifiers for django-ckeditor-5 0.2.19 are listed. "removePlugins": [ - "CKBox", "CKFinderUploadAdapter", "EasyImage", - "Image", "ImageCaption", "ImageStyle", "ImageToolbar", - "ImageUpload", "MediaEmbed", "Table", "TableToolbar", + "CKFinderUploadAdapter", "Image", "ImageCaption", "ImageStyle", + "ImageToolbar", "MediaEmbed", "Table", "TableToolbar", ], } } From 333c93bd92f0285a41dba5ec1de945b1bd13243a Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 30 Jan 2026 13:09:18 +0400 Subject: [PATCH 54/56] Bring back SPA storage for static files --- src/django/oar/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index 4f59a32d6..bc318db95 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -164,6 +164,7 @@ 'django_bleach', 'django_ckeditor_5', 'jsoneditor', + 'spa', ] # For allauth @@ -405,6 +406,7 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, 'static') STATICFILES_DIRS = ((os.path.join(STATIC_ROOT, "static")),) +STATICFILES_STORAGE = "spa.storage.SPAStaticFilesStorage" # Watchman # https://github.com/mwarkentin/django-watchman @@ -616,7 +618,7 @@ }, }, "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + "BACKEND": "spa.storage.SPAStaticFilesStorage", }, } else: @@ -625,7 +627,7 @@ "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", }, "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage", + "BACKEND": "spa.storage.SPAStaticFilesStorage", }, } From 53d3bf50b830878a32fdcd64024ce047ec626d23 Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 30 Jan 2026 14:14:44 +0400 Subject: [PATCH 55/56] [Test]: remove production flag in Django Dockerfile --- src/django/Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/django/Dockerfile b/src/django/Dockerfile index 1eadfef3d..dbc992e24 100644 --- a/src/django/Dockerfile +++ b/src/django/Dockerfile @@ -26,12 +26,7 @@ RUN set -ex \ COPY . /usr/local/src -RUN DJANGO_ENV=Production \ - BATCH_MODE=build \ - BATCH_JOB_QUEUE_NAME=build \ - BATCH_JOB_DEF_NAME=build \ - AWS_STORAGE_BUCKET_NAME=build \ - GOOGLE_SERVER_SIDE_API_KEY="" \ +RUN GOOGLE_SERVER_SIDE_API_KEY="" \ OAR_CLIENT_KEY="" \ HUBSPOT_API_KEY="" \ HUBSPOT_SUBSCRIPTION_ID="" \ From f64adaa885b64c1cff426dcdb9f63927d729323d Mon Sep 17 00:00:00 2001 From: VadimKovalenkoSNF Date: Fri, 30 Jan 2026 17:27:02 +0400 Subject: [PATCH 56/56] Remove redundant django-elasticache --- src/django/oar/settings.py | 3 --- src/django/requirements.txt | 1 - 2 files changed, 4 deletions(-) diff --git a/src/django/oar/settings.py b/src/django/oar/settings.py index bc318db95..1558139f2 100644 --- a/src/django/oar/settings.py +++ b/src/django/oar/settings.py @@ -317,9 +317,6 @@ # https://docs.djangoproject.com/en/3.2/topics/cache/ MEMCACHED_LOCATION = f"{os.getenv('CACHE_HOST')}:{os.getenv('CACHE_PORT')}" -# Use PyLibMCCache everywhere; django_elasticache is incompatible with Django 5 -# because it still imports smart_text. This keeps throttling cache working in -# jobs/containers that upgrade Django. CACHE_BACKEND = 'django.core.cache.backends.memcached.PyLibMCCache' CACHES = { diff --git a/src/django/requirements.txt b/src/django/requirements.txt index 52bf65798..468e26b1c 100644 --- a/src/django/requirements.txt +++ b/src/django/requirements.txt @@ -9,7 +9,6 @@ defusedxml==0.7.1 django-amazon-ses==4.0.1 django-cors-headers==4.4.0 django-ecsmanage==2.0.1 -django-elasticache==1.0.3 django-extensions==3.2.3 django-jsonview==2.0.0 dj-rest-auth[with_social]==7.0.2