[OSEDV-814] Upgrade Django and Python

doc/release/RELEASE-NOTES.md

@@ -3,6 +3,24 @@ All notable changes to this project will be documented in this file.
 
 This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). The format is based on the `RELEASE-NOTES-TEMPLATE.md` file.
 
+## Release 2.19.0
+
+## Introduction
+* Product name: Open Supply Hub
+* Release date: *Provide release date*
+
+### Database changes
+
+#### Migrations
+* 0196_switch_partner_field_source_by_editor.py - Migrates `PartnerField.source_by` to CKEditor5 so the rich-text content works after replacing `django-ckeditor` with `django-ckeditor-5` (required for Django 5); keeps existing source descriptions editable with formatting and links.
+* 0197_add_event_index.py - Adds an explicit index on `Event(content_type, object_id)` to replace the legacy `index_together` removed in Django 5, keeping the existing schema intact without editing old migrations.
+
+### What's new
+* [OSDEV-814](https://opensupplyhub.atlassian.net/browse/OSDEV-814) - Major upgrade of Django application backend services:
+    * Upgraded Python from `3.8` to `3.11`.
+    * Upgraded Django from `3.2.17` to `5.1.3`.
+    * Upgraded Python and Django packages to maintain compatibility.
+
 
 ## Release 2.18.1
 

src/django/Dockerfile

@@ -1,13 +1,10 @@
-FROM --platform=linux/amd64 python:3.8-slim-bookworm
+FROM --platform=linux/amd64 python:3.11-slim-bookworm
 
 RUN mkdir -p /usr/local/src/static/static
 WORKDIR /usr/local/src
 
 COPY requirements.txt /usr/local/src/
 
-# [OSDEV-1785][https://github.com/aio-libs/aiokafka/issues/1093] - Remove aiokafka in requirements.txt to avoid conflicts.
-RUN sed -i '/aiokafka/d' requirements.txt
-
 RUN set -ex \
     && buildDeps=" \
     build-essential \
@@ -24,8 +21,6 @@ RUN set -ex \
     " \
     && apt-get update && apt-get install -y $buildDeps $deps --no-install-recommends \
     && pip install --no-cache-dir -r requirements.txt \
-    # [OSDEV-1785][https://github.com/aio-libs/aiokafka/issues/1093] - Install aiokafka from the forked repo.
-    && pip install --no-cache-dir git+https://github.com/opensupplyhub/aiokafka.git@kafka-python-2.0.3 \
     && apt-get purge -y --auto-remove $buildDeps \
     && rm -rf /usr/local/src/requirements.txt /var/lib/apt/lists/*
 

src/django/api/admin.py

@@ -22,7 +22,7 @@
 from simple_history.admin import SimpleHistoryAdmin
 from waffle.models import Flag, Sample, Switch
 from waffle.admin import FlagAdmin, SampleAdmin, SwitchAdmin
-from ckeditor.widgets import CKEditorWidget
+from django_ckeditor_5.widgets import CKEditor5Widget
 from jsoneditor.forms import JSONEditor
 
 from api import models
@@ -256,7 +256,7 @@ def get_ordering(self, request):
 class PartnerFieldAdminForm(forms.ModelForm):
     source_by = forms.CharField(
         required=False,
-        widget=CKEditorWidget()
+        widget=CKEditor5Widget(config_name='default')
     )
     json_schema = forms.JSONField(
         required=False,

src/django/api/close_list.py

@@ -10,7 +10,7 @@
 def close_list(list_id, user_id):
     user = User.objects.get(id=user_id)
     contributor = Contributor.objects.get(admin=user)
-    now = datetime.now(tz=timezone.utc)
+    now = datetime.now(tz=timezone.get_default_timezone())
     reason = "Closed via bulk list closure"
     facilities = Facility.objects.filter(
         facilitylistitem__source__facility_list_id=list_id)

src/django/api/limitation/date/monthly_date_limitation.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timezone as dt_timezone
 from dateutil.relativedelta import relativedelta
 from django.utils import timezone
 
@@ -10,12 +10,19 @@
 class MonthlyDateLimitation(DateLimitation):
 
     def execute(self, period_start_date: datetime):
-        utc = timezone.utc
-        self.start_date = period_start_date
-
-        one_month_in_past = datetime.now(tz=utc) - relativedelta(months=1)
-        while (self.start_date < one_month_in_past):
-            self.start_date = self.start_date + relativedelta(months=1)
+        default_tz = timezone.get_default_timezone()
+        start_date = period_start_date
+        if timezone.is_naive(start_date):
+            start_date = timezone.make_aware(start_date, default_tz)
+        start_date_utc = start_date.astimezone(dt_timezone.utc)
+
+        one_month_ago_utc = (
+            datetime.now(tz=dt_timezone.utc) - relativedelta(months=1)
+        )
+        while start_date_utc < one_month_ago_utc:
+            start_date_utc = start_date_utc + relativedelta(months=1)
+
+        self.start_date = start_date_utc
 
         return self
 

src/django/api/limitation/date/yearly_date_limitation.py

@@ -10,10 +10,10 @@
 class YearlyDateLimitation(DateLimitation):
 
     def execute(self, period_start_date: datetime):
-        utc = timezone.utc
+        default_tz = timezone.get_default_timezone()
         self.start_date = period_start_date
 
-        one_year_in_past = datetime.now(tz=utc) - relativedelta(years=1)
+        one_year_in_past = datetime.now(tz=default_tz) - relativedelta(years=1)
         while (self.start_date < one_year_in_past):
             self.start_date = self.start_date + relativedelta(years=1)
 

src/django/api/middleware.py

@@ -78,7 +78,7 @@ def has_active_block(request):
 
         contributor = token.user.contributor
         apiBlock = get_api_block(contributor)
-        at_datetime = datetime.datetime.now(tz=timezone.utc)
+        at_datetime = datetime.datetime.now(tz=timezone.get_default_timezone())
         return (apiBlock is not None and
                 apiBlock.until > at_datetime and apiBlock.active)
     except ObjectDoesNotExist:

src/django/api/migrations/0196_switch_partner_field_source_by_editor.py

@@ -0,0 +1,22 @@
+from django.db import migrations
+import django_ckeditor_5.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0195_add_mit_livingwage_partner_field'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='partnerfield',
+            name='source_by',
+            field=django_ckeditor_5.fields.CKEditor5Field(
+                blank=True,
+                help_text='Rich text field describing the source of this partner field.',
+                null=True,
+            ),
+        ),
+    ]
+

src/django/api/migrations/0197_add_event_index.py

@@ -0,0 +1,24 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0196_switch_partner_field_source_by_editor"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            sql=[
+                "DROP INDEX IF EXISTS api_event_content_type_id_object_id_idx",
+                "DROP INDEX IF EXISTS api_event_content_type_id_object_id_b046420a_idx",
+            ],
+            reverse_sql=migrations.RunSQL.noop,
+        ),
+        migrations.AddIndex(
+            model_name="event",
+            index=models.Index(
+                fields=["content_type", "object_id"],
+                name="api_event_content_object_idx",
+            ),
+        ),
+    ]

src/django/api/models/event.py

@@ -7,7 +7,12 @@
 
 class Event(models.Model):
     class Meta:
-        index_together = (('content_type', 'object_id'),)
+        indexes = [
+            models.Index(
+                fields=['content_type', 'object_id'],
+                name='api_event_content_object_idx',
+            ),
+        ]
 
     EVENT_TYPE_CHOICES = (
         (FacilityHistoryActions.CREATE, FacilityHistoryActions.CREATE),

src/django/api/models/partner_field.py

@@ -3,7 +3,7 @@
 
 from django.db import models
 from django.core.cache import cache
-from ckeditor.fields import RichTextField
+from django_ckeditor_5.fields import CKEditor5Field
 
 from api.constants import PARTNER_FIELD_LIST_KEY, PARTNER_FIELD_NAMES_KEY
 from api.models.partner_field_manager import PartnerFieldManager
@@ -68,13 +68,11 @@ class Meta:
         max_length=500,
         blank=True,
     )
-    source_by = RichTextField(
+    source_by = CKEditor5Field(
         blank=True,
         null=True,
-        config_name="default",
         help_text=(
-            "Rich text field describing the "
-            "source of this partner field."
+            "Rich text field describing the source of this partner field."
         ),
     )
     json_schema = models.JSONField(

src/django/api/processing.py

@@ -78,7 +78,7 @@ def parse_production_location_list(location_list: FacilityList):
 
 def geocode_facility_list_item(item):
     started = str(timezone.now())
-    if type(item) != FacilityListItem:
+    if not isinstance(item, FacilityListItem):
         raise ValueError('Argument must be a FacilityListItem')
     if item.status == FacilityListItem.ITEM_REMOVED:
         raise ItemRemovedException()

src/django/api/reports.py

@@ -50,7 +50,10 @@ def monthly_promoted_name_and_address():
 def geocoding_time_without_queue(date_format):
     temp_data = dict()
 
-    start_date = datetime.now(tz=timezone.utc) - relativedelta(months=4)
+    start_date = (
+        datetime.now(tz=timezone.get_default_timezone())
+        - relativedelta(months=4)
+    )
     processing_results = FacilityListItem.objects.filter(
         created_at__gte=start_date,
         processing_results__contains=[{"action": "geocode"}]
@@ -89,7 +92,10 @@ def weekly_geocoding_time_without_queue():
 
 def geocoding_time_with_queue(date_format):
     temp_data = dict()
-    start_date = datetime.now(tz=timezone.utc) - relativedelta(months=4)
+    start_date = (
+        datetime.now(tz=timezone.get_default_timezone())
+        - relativedelta(months=4)
+    )
     listitems = FacilityListItem.objects.filter(
         created_at__gte=start_date,
         processing_results__contains=[{"action": "geocode"}]

src/django/api/serializers/facility/utils.py

@@ -1,13 +1,12 @@
 from typing import (Union)
 from itertools import groupby
+from datetime import timezone as dt_timezone
 
 from api.constants import (
     FacilityClaimStatuses,
     FacilitiesQueryParams
 )
 from dateutil import parser
-from django.utils import timezone
-
 from ...helpers.helpers import (
     cleanup_data,
     replace_invalid_data,
@@ -265,7 +264,7 @@ def format_date(date: str) -> str:
     original_datetime = parser.isoparse(date)
 
     # Convert the datetime object to UTC.
-    utc_datetime = original_datetime.astimezone(timezone.utc)
+    utc_datetime = original_datetime.astimezone(dt_timezone.utc)
 
     # Format the UTC datetime with the "Z" notation.
     formatted_datetime_str = utc_datetime.strftime('%Y-%m-%dT%H:%M:%S.%fZ')

src/django/api/serializers/user/user_password_reset_confirm_serializer.py

@@ -1,4 +1,4 @@
-from rest_auth.serializers import PasswordResetConfirmSerializer
+from dj_rest_auth.serializers import PasswordResetConfirmSerializer
 from django.db import transaction
 
 

src/django/api/serializers/user/user_password_reset_serializer.py

@@ -1,4 +1,4 @@
-from rest_auth.serializers import PasswordResetSerializer
+from dj_rest_auth.serializers import PasswordResetSerializer
 from rest_framework.exceptions import ValidationError
 from rest_framework.serializers import EmailField
 from django.conf import settings

src/django/api/throttles.py

@@ -11,7 +11,7 @@
 
 
 class UserCustomRateThrottle(UserRateThrottle):
-    """Allow custom per-user throttle rates defined on custom Django user model.
+    """Allow per-user throttle rates defined on the custom user model.
 
     `model_rate_field`: Specify the throttle rate field. Required.
     """

src/django/api/views/auth/login_to_oar_client.py

@@ -1,7 +1,7 @@
 from allauth.account.models import EmailAddress
 from rest_framework.exceptions import AuthenticationFailed
 from rest_framework.response import Response
-from rest_auth.views import LoginView
+from dj_rest_auth.views import LoginView
 from django.contrib.auth import authenticate, login
 
 from ...serializers.user.user_serializer import UserSerializer

src/django/api/views/auth/logout_to_oar_client.py

@@ -1,6 +1,6 @@
 from rest_framework import status
 from rest_framework.response import Response
-from rest_auth.views import LogoutView
+from dj_rest_auth.views import LogoutView
 from django.contrib.auth import logout
 
 from ...serializers.user.user_serializer import UserSerializer

src/django/ckeditor/__init__.py

@@ -0,0 +1 @@
+"""Compatibility shim for legacy migrations that import `ckeditor`."""

src/django/ckeditor/fields.py

@@ -0,0 +1,10 @@
+from django.db import models
+
+
+class RichTextField(models.TextField):
+    """Minimal stub to satisfy legacy migrations."""
+
+    def __init__(self, *args, **kwargs):
+        # Legacy migrations pass config_name; ignore it for compatibility.
+        kwargs.pop("config_name", None)
+        super().__init__(*args, **kwargs)

src/django/ckeditor/widgets.py

@@ -0,0 +1,10 @@
+from django.forms import Textarea
+
+
+class CKEditorWidget(Textarea):
+    """Minimal stub to satisfy any legacy imports."""
+
+    def __init__(self, *args, **kwargs):
+        # Ignore legacy config_name argument if provided.
+        kwargs.pop("config_name", None)
+        super().__init__(*args, **kwargs)