[OSDEV-1515] Database. Upgrade the PostgreSQL version to 16.

deployment/environments/terraform-development.tfvars

@@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "128"
-rds_engine_version = "13"
-rds_parameter_group_family = "postgres13"
+rds_engine_version = "16"
+rds_parameter_group_family = "postgres16"
 rds_instance_type = "db.t3.micro"
 rds_database_identifier = "opensupplyhub-enc-dev"
 rds_database_name = "opensupplyhub"

deployment/environments/terraform-preprod.tfvars

@@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "256"
-rds_engine_version = "13"
-rds_parameter_group_family = "postgres13"
+rds_engine_version = "16"
+rds_parameter_group_family = "postgres16"
 rds_instance_type = "db.m6in.4xlarge"
 rds_database_identifier = "opensupplyhub-enc-pp"
 rds_database_name = "opensupplyhub"

deployment/environments/terraform-production.tfvars

@@ -13,8 +13,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "256"
-rds_engine_version = "13"
-rds_parameter_group_family = "postgres13"
+rds_engine_version = "16"
+rds_parameter_group_family = "postgres16"
 rds_instance_type = "db.m6in.4xlarge"
 rds_database_identifier = "opensupplyhub-enc-prd"
 rds_database_name = "opensupplyhub"

deployment/environments/terraform-staging.tfvars

@@ -12,8 +12,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "128"
-rds_engine_version = "13"
-rds_parameter_group_family = "postgres13"
+rds_engine_version = "16"
+rds_parameter_group_family = "postgres16"
 rds_instance_type = "db.t3.large"
 rds_database_identifier = "opensupplyhub-enc-stg"
 rds_database_name = "opensupplyhub"

deployment/environments/terraform-test.tfvars

@@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "256"
-rds_engine_version = "13"
-rds_parameter_group_family = "postgres13"
+rds_engine_version = "16"
+rds_parameter_group_family = "postgres16"
 rds_instance_type = "db.t3.xlarge"
 rds_database_identifier = "opensupplyhub-enc-tst"
 rds_database_name = "opensupplyhub"

deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgis/postgis:13-3.4-alpine
+FROM postgis/postgis:16-3.4-alpine
 
 WORKDIR /opt/
 
@@ -9,4 +9,3 @@ RUN set -ex && \
 
 COPY ./dump.sh ./dump.sh
 CMD ["bash", "dump.sh"]
-

deployment/terraform/database.tf

@@ -76,7 +76,7 @@ resource "aws_db_parameter_group" "default" {
 }
 
 module "database_enc" {
-  source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.1.0"
+  source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.2.0"
 
   vpc_id                      = module.vpc.id
   allocated_storage           = var.rds_allocated_storage

deployment/terraform/variables.tf

@@ -70,11 +70,11 @@ variable "rds_allocated_storage" {
 }
 
 variable "rds_engine_version" {
-  default = "13"
+  default = "16"
 }
 
 variable "rds_parameter_group_family" {
-  default = "postgres13"
+  default = "postgres16"
 }
 
 variable "rds_instance_type" {

doc/release/RELEASE-NOTES.md

@@ -10,10 +10,11 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
 * Release date: February 8, 2025
 
 ### Database changes
-* *Describe high-level database changes.*
+* [OSDEV-1515](https://opensupplyhub.atlassian.net/browse/OSDEV-1515) - Upgraded the PostgreSQL version from 13 to 16 for the database used in local development, DB anonymization, DB restore setup, and environments in the AWS cloud. Additionally, the pg_trgm extension has been upgraded to version 1.6 based on the available extension version for PostgreSQL 16.3 in AWS RDS. For more information, see [Extensions supported for RDS for PostgreSQL 16](https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-extensions.html#postgresql-extensions-16x).
 
 #### Migrations:
-* *Describe migrations here.*
+* 0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py - Updated the SQL script within the migration that upgrades the DB extension versions to handle previously failure cases when a higher version is available for upgrade or when the extension is not installed. This is primarily useful for local development or DB resets in the Development environment, where migrations are applied from scratch, one by one. This fix will not negatively affect other environments, as the migration has already been applied and will not be reapplied. Additionally, the changes are backward compatible.
+* 0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.py - This migration refreshes the `pg_statistic` table after the upgrade to PostgreSQL 16 and upgrades the pg_trgm extension to version 1.6. The SQL script within the migration that upgrades the DB extension versions handles previously failure cases where a higher version is available for upgrade or where the extension is not installed.
 
 #### Schema changes
 * *Describe schema changes here.*
@@ -37,6 +38,10 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
 * Ensure that the following commands are included in the `post_deployment` command:
     * `migrate`
     * `reindex_database`
+* This release will upgrade PostgreSQL from version 13 to version 16.
+    * The upgrade will be performed automatically by Terrafrom and AWS, but some steps need to be completed **before** and **after** the upgrade. Please refer to [the Confluence article](https://opensupplyhub.atlassian.net/wiki/spaces/SD/pages/640155649/PostgreSQL+database+upgrade+from+version+13+to+version+16) for detailed instructions.
+    * Steps to be completed before the upgrade are marked with the statement: "**This should be done before deploying the upgraded database.**". Post-upgrade tasks can be found under the [After the PostgreSQL major version upgrade](https://opensupplyhub.atlassian.net/wiki/spaces/SD/pages/640155649/PostgreSQL+database+upgrade+from+version+13+to+version+16#After-the-PostgreSQL-major-version-upgrade) section.
+    * In case of an unsuccessful release along with the database upgrade, follow the instructions under the [Guide for rolling back the PostgreSQL major version upgrade](https://opensupplyhub.atlassian.net/wiki/spaces/SD/pages/640155649/PostgreSQL+database+upgrade+from+version+13+to+version+16#Guide-for-rolling-back-the-PostgreSQL-major-version-upgrade) section.
 
 
 ## Release 1.28.1

src/anon-tools/Dockerfile.dump

@@ -1,4 +1,4 @@
-FROM postgis/postgis:13-3.4-alpine
+FROM postgis/postgis:16-3.4-alpine
 
 WORKDIR /opt/
 
@@ -35,4 +35,3 @@ COPY ./do_dump.sh ./do_dump.sh
 VOLUME /keys
 
 CMD ["sh", "do_dump.sh"]
-

src/anon-tools/Dockerfile.restore

@@ -1,4 +1,4 @@
-FROM postgis/postgis:13-3.4-alpine
+FROM postgis/postgis:16-3.4-alpine
 
 WORKDIR /opt/
 
@@ -29,4 +29,3 @@ COPY ./do_restore.sh ./
 VOLUME /keys
 
 CMD ["sh", "do_restore.sh"]
-

src/database/Dockerfile.local

@@ -1,19 +1,20 @@
-# The latest supported version of PostgreSQL in AWS RDS is 13.15.
-# However, the official PostgreSQL Docker registry provides only
-# the latest minor version for PostgreSQL 13, and it is not
-# possible to specify a specific minor version like 13.15. Therefore,
-# it was decided to provide only the major version and let Docker
-# determine which minor version to install. This should not cause
-# incompatibilities, even if PostgreSQL 13.15 is used in AWS.
-FROM postgres:13
+# The latest supported version of PostgreSQL in AWS RDS is 16.3
+# for PostgreSQL 16. However, the official PostgreSQL Docker
+# registry provides only the latest minor version for PostgreSQL 16,
+# and it is not possible to specify a specific minor version like
+# 16.3. Therefore, it was decided to provide only the major version
+# and let Docker determine which minor version to install. This
+# should not cause incompatibilities, even if PostgreSQL 16.3 is
+# currently used in AWS.
+FROM postgres:16
 
 ENV POSTGRES_USER=opensupplyhub \
     POSTGRES_PASSWORD=opensupplyhub \
     POSTGRES_DB=opensupplyhub
 
 
 # Build and install the PostGIS package from scratch to ensure it matches the
-# LLVM version used by the Docker container based on postgres:13 image.
+# LLVM version used by the Docker container based on postgres:16 image.
 # See PostGIS installation details at https://postgis.net/docs/manual-3.4/postgis_installation.html
 
 # Install the requirements for building and using PostGIS.
@@ -28,7 +29,7 @@ RUN apt-get update && apt-get install -y \
     libprotobuf-c-dev \
     protobuf-c-compiler \
     pkg-config \
-    postgresql-server-dev-13
+    postgresql-server-dev-16
 
 # Download the PostGIS source archive, build and install the PostGIS package.
 RUN wget https://download.osgeo.org/postgis/source/postgis-3.4.2.tar.gz && \

src/django/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 python:3.7-slim-bullseye
+FROM --platform=linux/amd64 python:3.7-slim-bookworm
 
 RUN mkdir -p /usr/local/src/static/static
 WORKDIR /usr/local/src
@@ -12,7 +12,7 @@ RUN set -ex \
     && deps=" \
     gdal-bin \
     gettext \
-    postgresql-client-13 \
+    postgresql-client-15 \
     " \
     && apt-get update && apt-get install -y $buildDeps $deps --no-install-recommends \
     && pip install --no-cache-dir -r requirements.txt \

src/django/api/migrations/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py

@@ -17,7 +17,7 @@ def refresh_pg_statistic_and_perform_upgrading_pg_extensions(
 class Migration(Migration):
     '''
     This migration refreshes the pg_statistic table and upgrades the
-    PostgreSQL database extensions.
+    PostgreSQL database extensions in light of the upgrade to Postgres 13.
 
     Since this migration will be executed after the PostgreSQL major version
     upgrade, the ANALYZE operation should be run to refresh the pg_statistic

src/django/api/migrations/0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.py

@@ -0,0 +1,51 @@
+# Generated by Django 3.2.17 on 2025-01-24 15:21
+
+from django.db.migrations import Migration, RunPython
+from django.db import connection
+from api.migrations._migration_helper import MigrationHelper
+
+helper = MigrationHelper(connection)
+
+
+def refresh_pg_statistic_and_perform_upgrading_pg_extensions(
+        apps, schema_editor):
+    helper.run_sql_files([
+        ('0164_refresh_pg_statistic_and_upgrade_postgres_extensions'
+         '_after_db_upgrade_to_postgres_16.sql')
+    ])
+
+
+class Migration(Migration):
+    '''
+    This migration refreshes the pg_statistic table and upgrades the
+    PostgreSQL database extensions in light of the upgrade to Postgres 16.
+
+    Since this migration will be executed after the PostgreSQL major version
+    upgrade, the ANALYZE operation should be run to refresh the pg_statistic
+    table. Optimizer statistics aren't transferred during a major version
+    upgrade, so it is necessary to regenerate all statistics to avoid
+    performance issues.
+
+    Currently, the database uses the following extensions:
+    1. postgis
+    2. unaccent
+    3. pg_trgm
+    4. plpgsql
+    5. btree_gin
+    6. pgcrypto
+
+    Based on the available extension versions for PostgreSQL 16.3 in AWS RDS,
+    which will be used across all AWS environments after the database upgrade
+    to Postgres 16, it was found that the `pg_trgm` extensions can be upgraded
+    to version 1.6, respectively, in Production, Staging, Test, and
+    Development environments. If the specified versions are already installed
+    in the database, there will be no issues.
+    '''
+
+    dependencies = [
+        ('api', '0163_refresh_pg_statistic_and_upgrade_postgres_extensions'),
+    ]
+
+    operations = [
+        RunPython(refresh_pg_statistic_and_perform_upgrading_pg_extensions)
+    ]

src/django/sqls/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql

@@ -1,4 +1,54 @@
 ANALYZE VERBOSE;
 
-ALTER EXTENSION postgis UPDATE TO '3.4.2';
-ALTER EXTENSION pg_trgm UPDATE TO '1.5';
+DO $$
+DECLARE
+-- Variables for postgis.
+current_version_postgis TEXT;
+target_version_postgis TEXT := '3.4.2';
+
+-- Variables for pg_trgm.
+current_version_pg_trgm TEXT;
+target_version_pg_trgm TEXT := '1.5';
+
+BEGIN
+-- Extension 1: Check and upgrade "postgis" to its target version.
+SELECT
+	extversion
+INTO
+	current_version_postgis
+FROM
+	pg_extension
+WHERE
+	extname = 'postgis';
+
+IF current_version_postgis IS NULL THEN
+	RAISE EXCEPTION 'Extension "postgis" is not installed.';
+ELSIF current_version_postgis < target_version_postgis
+THEN
+	EXECUTE FORMAT('ALTER EXTENSION postgis UPDATE TO %L', target_version_postgis);
+	RAISE NOTICE 'Extension "postgis" has been upgraded from % to %.', current_version_postgis, target_version_postgis;
+ELSE
+	RAISE NOTICE 'Extension "postgis" is already at the target version or higher.';
+END IF;
+
+-- Extension 2: Check and upgrade "pg_trgm" to its target version.
+SELECT
+	extversion
+INTO
+	current_version_pg_trgm
+FROM
+	pg_extension
+WHERE
+	extname = 'pg_trgm';
+
+IF current_version_pg_trgm IS NULL
+THEN
+	RAISE EXCEPTION 'Extension "pg_trgm" is not installed.';
+ELSIF current_version_pg_trgm < target_version_pg_trgm
+THEN
+	EXECUTE FORMAT('ALTER EXTENSION pg_trgm UPDATE TO %L', target_version_pg_trgm);
+	RAISE NOTICE 'Extension "pg_trgm" has been upgraded from % to %.', current_version_pg_trgm, target_version_pg_trgm;
+ELSE
+	RAISE NOTICE 'Extension "pg_trgm" is already at the target version or higher.';
+END IF;
+END $$;

src/django/sqls/0164_refresh_pg_statistic_and_upgrade_postgres_extensions_after_db_upgrade_to_postgres_16.sql

@@ -0,0 +1,30 @@
+ANALYZE VERBOSE;
+
+DO $$
+DECLARE
+-- Variables for pg_trgm.
+current_version_pg_trgm TEXT;
+target_version_pg_trgm TEXT := '1.6';
+
+BEGIN
+-- Extension 1: Check and upgrade "pg_trgm" to its target version.
+SELECT
+	extversion
+INTO
+	current_version_pg_trgm
+FROM
+	pg_extension
+WHERE
+	extname = 'pg_trgm';
+
+IF current_version_pg_trgm IS NULL
+THEN
+	RAISE EXCEPTION 'Extension "pg_trgm" is not installed.';
+ELSIF current_version_pg_trgm < target_version_pg_trgm
+THEN
+	EXECUTE FORMAT('ALTER EXTENSION pg_trgm UPDATE TO %L', target_version_pg_trgm);
+	RAISE NOTICE 'Extension "pg_trgm" has been upgraded from % to %.', current_version_pg_trgm, target_version_pg_trgm;
+ELSE
+	RAISE NOTICE 'Extension "pg_trgm" is already at the target version or higher.';
+END IF;
+END $$;