testing

.github/workflows/django-ci.yml

@@ -7,8 +7,10 @@ on:
     branches: [main]
 
 jobs:
-  build:
+  run_tests:
     runs-on: ubuntu-latest
+    env:
+      DEPLOY_ENVIRONMENT: test
     steps:
       - uses: actions/checkout@v4
       - name: build containers and run Django tests

.github/workflows/e2e.yml

@@ -9,6 +9,8 @@ on:
 jobs:
   e2e:
     runs-on: ubuntu-latest
+    env:
+      DEPLOY_ENVIRONMENT: test
     steps:
       - uses: actions/checkout@v4
 

Makefile

@@ -1,6 +1,3 @@
-# set DEPLOY_ENVIRONMENT in config.mk
-DEPLOY_ENVIRONMENT := dev
-
 DOCKER_SHARED_DIR=docker/shared
 # shared directory subdirectories for postgres (data), frontend (vite), logs, static assets to be delivered by nginx
 # `data` directory is used for direct postgres db server-side outputs, e.g., postgres COPY commands issued in
@@ -24,6 +21,7 @@ BORG_REPO_URL := https://example.com/repo.tar.xz
 BORG_REPO_PATH=${BUILD_DIR}/repo.tar.xz
 REPO_BACKUPS_PATH=${DOCKER_SHARED_DIR}/backups
 
+# DEPLOY_ENVIRONMENT must be set in config.mk
 include config.mk
 include .env
 
@@ -34,13 +32,13 @@ include .env
 
 .PHONY: build
 build: docker-compose.yml secrets $(DOCKER_SHARED_DIR)
-	docker compose build --pull
+	docker compose build --pull -q
 
 $(BORG_REPO_PATH):
 	wget -c ${BORG_REPO_URL} -P ${BUILD_DIR}
 
 config.mk:
-	DEPLOY_ENVIRONMENT=${DEPLOY_ENVIRONMENT} envsubst < ${DEPLOY_CONF_DIR}/config.mk.template > config.mk
+	envsubst < ${DEPLOY_CONF_DIR}/config.mk.template > config.mk
 
 .PHONY: $(DOCKER_SHARED_DIR)
 $(DOCKER_SHARED_DIR):
@@ -82,9 +80,9 @@ release-version: .env
 	$(ENVREPLACE) TEST_BASIC_AUTH_PASSWORD $$(openssl rand -base64 42) .env
 
 .PHONY: docker-compose.yml
-docker-compose.yml: base.yml dev.yml staging.yml prod.yml config.mk $(PGPASS_PATH) release-version .env
+docker-compose.yml: base.yml dev.yml staging.yml test.yml prod.yml config.mk $(PGPASS_PATH) release-version .env
 	case "$(DEPLOY_ENVIRONMENT)" in \
-	  dev|staging) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
+	  dev|staging|test) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
 	  prod) docker compose -f base.yml -f staging.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
 	  *) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \
 	esac
@@ -148,12 +146,15 @@ E2E_REPO_PATH=${E2E_BACKUPS_PATH}/repo
 
 $(E2E_REPO_PATH):
 	mkdir -p $(E2E_BACKUPS_PATH)
-	wget -c ${BORG_REPO_URL} -P $(E2E_BACKUPS_PATH)
+	wget -c --no-check-certificate ${BORG_REPO_URL} -P $(E2E_BACKUPS_PATH)
 	tar -Jxf $(E2E_BACKUPS_PATH)/repo.tar.xz -C $(E2E_BACKUPS_PATH)
 
 .PHONY: e2e
 e2e: docker-compose.yml secrets $(DOCKER_SHARED_DIR) $(E2E_REPO_PATH)
-	docker compose -f docker-compose.yml -f e2e.yml up -d --build
+	docker compose -f docker-compose.yml -f e2e.yml build -q
+	docker compose -f docker-compose.yml -f e2e.yml up -d
+	sleep 42
 	docker compose -f docker-compose.yml -f e2e.yml exec server bash -c "\
 		inv borg.restore --force && \
+		inv db.init && \
 		inv prepare"

django/core/settings/__init__.py

@@ -0,0 +1 @@
+from .defaults import *

django/core/settings/defaults.py

@@ -10,8 +10,10 @@
 """
 
 import os
+import sys
 import warnings
 from elasticsearch.exceptions import ElasticsearchWarning
+from collections import namedtuple
 from enum import Enum
 from pathlib import Path
 
@@ -26,15 +28,19 @@ def read_secret(file, fallback=""):
         return fallback
 
 
+EnvConfig = namedtuple("EnvConfig", ["base_url", "label"])
+
+
 class Environment(Enum):
-    DEVELOPMENT = "http://localhost:8000"
-    STAGING = "https://staging.comses.net"
-    PRODUCTION = "https://www.comses.net"
-    TEST = "http://localhost:8000"
+    DEVELOPMENT = EnvConfig(base_url="http://localhost:8000", label="DEVELOPMENT")
+    STAGING = EnvConfig(base_url="https://staging.comses.net", label="STAGING")
+    PRODUCTION = EnvConfig(base_url="https://www.comses.net", label="PRODUCTION")
+    # TEST is used for local and github testing, not a real environment
+    TEST = EnvConfig(base_url="http://localhost:8000", label="TEST")
 
     @property
     def base_url(self):
-        return self.value
+        return self.value.base_url
 
     @property
     def is_staging_or_production(self):
@@ -57,7 +63,14 @@ def is_test(self):
         return self == Environment.TEST
 
 
-DEPLOY_ENVIRONMENT = Environment.DEVELOPMENT
+def set_environment(env: Environment):
+    global DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL
+    DEPLOY_ENVIRONMENT = env
+    # Base URL to use when referring to full URLs within the Wagtail admin backend -
+    # e.g. in notification emails. Don't include '/admin' or a trailing slash
+    WAGTAILADMIN_BASE_URL = BASE_URL = env.base_url
+    return DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL
+
 
 # go two levels up for root project directory
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -72,13 +85,11 @@ def is_test(self):
 
 DJANGO_VITE_DEV_MODE = True
 
-# Base URL to use when referring to full URLs within the Wagtail admin backend -
-# e.g. in notification emails. Don't include '/admin' or a trailing slash
-# FIXME: needs to be overridden in staging and prod after updating DEPLOY_ENVIRONMENT which is less than ideal
 
-WAGTAILADMIN_BASE_URL = BASE_URL = DEPLOY_ENVIRONMENT.base_url
+TESTING = "test" in sys.argv or "PYTEST_VERSION" in os.environ
+
 # set up robots + sitemaps inclusion https://django-robots.readthedocs.io/en/latest/
-ROBOTS_SITEMAP_URLS = [f"{BASE_URL}/sitemap.xml"]
+# ROBOTS_SITEMAP_URLS = [f"{BASE_URL}/sitemap.xml"]
 
 # wagtail config: https://docs.wagtail.io/en/v2.10.1/getting_started/integrating_into_django.html
 WAGTAIL_APPS = [
@@ -191,6 +202,7 @@ def is_test(self):
     "cdn.jsdelivr.net",  # codemirror spell checker
     "*.comses.net",  # sentry.comses.net / forum.comses.net
     "www.google-analytics.com",  # google analytics
+    "export.highcharts.com",  # highcharts metrics export
 )
 CSP_FONT_SRC = ("'self'", "fonts.googleapis.com", "fonts.gstatic.com", "localhost:*")
 CSP_STYLE_SRC = (
@@ -433,7 +445,7 @@ def is_test(self):
 DJANGO_VITE_ASSETS_PATH = os.path.join(SHARE_DIR, "vite")
 DJANGO_VITE_STATIC_URL_PREFIX = "bundles"
 DJANGO_VITE_DEV_SERVER_PORT = 5173
-DJANG_VITE_MANIFEST_PATH = os.path.join(
+DJANGO_VITE_MANIFEST_PATH = os.path.join(
     DJANGO_VITE_ASSETS_PATH, DJANGO_VITE_STATIC_URL_PREFIX, "manifest.json"
 )
 

django/core/settings/dev.py

@@ -3,14 +3,19 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
+DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment(
+    Environment.DEVELOPMENT
+)
+
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
 
-MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"]
 
-INSTALLED_APPS += [
-    "debug_toolbar",
-    "fixture_magic",
-]
+if not TESTING:
+    INSTALLED_APPS += [
+        "debug_toolbar",
+        "fixture_magic",
+    ]
+    MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"]
 
 ALLOWED_HOSTS = ["localhost", "127.0.0.1", "server"]
 

django/core/settings/e2e.py

@@ -1,6 +1,10 @@
 from .test import *
 
 DEBUG = True
+DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment(
+    Environment.DEVELOPMENT
+)
+TESTING = True
 
 DJANGO_VITE_DEV_MODE = False
 

django/core/settings/production.py

@@ -10,7 +10,10 @@
 
 DEBUG = False
 DJANGO_VITE_DEV_MODE = False
-DEPLOY_ENVIRONMENT = Environment.PRODUCTION
+DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment(
+    Environment.PRODUCTION
+)
+
 EMAIL_SUBJECT_PREFIX = os.getenv("EMAIL_SUBJECT_PREFIX", "[comses.net]")
 # See http://django-allauth.readthedocs.io/en/latest/providers.html#orcid for more context.
 #
@@ -64,9 +67,6 @@
 )
 CSP_INCLUDE_NONCE_IN = ["script-src"]
 
-# Base URL to use when referring to full URLs within the Wagtail admin backend -
-# e.g. in notification emails. Don't include '/admin' or a trailing slash
-WAGTAILADMIN_BASE_URL = BASE_URL = DEPLOY_ENVIRONMENT.base_url
 # set up robots + sitemaps inclusion https://django-robots.readthedocs.io/en/latest/
 ROBOTS_SITEMAP_URLS = [f"{BASE_URL}/sitemap.xml"]
 

django/core/settings/staging.py

@@ -5,7 +5,9 @@
 
 DEBUG = False
 DJANGO_VITE_DEV_MODE = False
-DEPLOY_ENVIRONMENT = Environment.STAGING
+DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment(
+    Environment.STAGING
+)
 
 # datacite sandbox configuration inherited from defaults should suffice
 # DATACITE_PREFIX = "10.82853"

django/core/settings/test.py

@@ -2,8 +2,8 @@
 
 from .defaults import *
 
-
-DEPLOY_ENVIRONMENT = Environment.TEST
+DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment(Environment.TEST)
+TESTING = True
 
 ALLOWED_HOSTS = ["localhost", "127.0.0.1", "server"]
 

django/core/urls.py

@@ -77,30 +77,27 @@ def get_core_urls():
     path("", include((get_core_urls(), "core"), namespace="core")),
     path("accounts/", include("allauth.urls")),
     path("django/admin/", admin.site.urls),
-    # Replace the default wagtail admin home page
-    # path('wagtail/admin/', view=wagtail_hooks.DashboardView.as_view(), name='wagtailadmin_home'),
     path("wagtail/admin/", include(wagtailadmin_urls)),
     path("api/schema/", schema_view),
     path("api-auth/", include("rest_framework.urls")),
     # configure sitemaps and robots.txt, see https://django-robots.readthedocs.io/en/latest/
-    # https://docs.wagtail.io/en/v2.9.2/reference/contrib/sitemaps.html
+    # https://docs.wagtail.org/en/stable/reference/contrib/sitemaps.html
     path("sitemap.xml", sitemap),
     path("robots.txt", include("robots.urls")),
 ]
 
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 
-if not settings.DEPLOY_ENVIRONMENT.is_production:
-    import debug_toolbar
-
+if settings.DEPLOY_ENVIRONMENT.is_development:
     urlpatterns += [
         path("argh/", handler500, name="error"),
         path("make-error/", views.make_error),
-        path("__debug__/", include(debug_toolbar.urls)),
-    ]
+    ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+    if not settings.TESTING:
+        from debug_toolbar.toolbar import debug_toolbar_urls
+
+        urlpatterns += debug_toolbar_urls()
 
-if settings.DEPLOY_ENVIRONMENT.is_development:
-    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 
 # NB: wagtail_urls are the catchall, must be last
 urlpatterns.append(path("", include(wagtail_urls)))

django/curator/invoke_tasks/utils.py

@@ -29,9 +29,9 @@ def dj(ctx, command, **kwargs):
     """
     Run a Django manage.py command on the server.
     """
+    django_settings_module = os.environ.get("DJANGO_SETTINGS_MODULE")
+    invocation = f"python3 manage.py {command} --settings {django_settings_module}"
+    print("Invoking command: ", invocation)
     ctx.run(
-        "{python} manage.py {dj_command} --settings {project_conf}".format(
-            dj_command=command, **env
-        ),
-        **kwargs,
+        invocation, env={"DJANGO_SETTINGS_MODULE": django_settings_module}, **kwargs
     )

django/deploy/dev.sh

@@ -16,4 +16,5 @@ initdb() {
     fi
 }
 initdb
-exec /code/manage.py runserver 0.0.0.0:8000
+echo "Running dev server with ${DJANGO_SETTINGS_MODULE}"
+exec env DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} /code/manage.py runserver 0.0.0.0:8000

django/deploy/test.sh

@@ -1,13 +1,11 @@
 #!/bin/sh
 
-export DJANGO_SETTINGS_MODULE="core.settings.test"
-
 chmod a+x /code/deploy/*.sh;
 
 initdb() {
     cd /code;
     echo "Destroying and initializing database from scratch"
-    invoke db.init
+    exec env DJANGO_SETTINGS_MODULE="core.settings.test" invoke db.init
 }
 initdb
-exec invoke prepare test --tests="$@" --coverage
+exec env DJANGO_SETTINGS_MODULE="core.settings.test" invoke prepare test --tests="$@" --coverage

django/home/feeds.py

@@ -204,7 +204,7 @@ class FeedItem:
 class AbstractFeed(ABC):
     max_number_of_items = DEFAULT_HOMEPAGE_FEED_MAX_ITEMS
     _cache_key = None  # subclasses can define a custom cache key if needed
-    rate_limited = False # set to True if the feed is rate limited to cache in dev mode
+    rate_limited = False  # set to True if the feed is rate limited to cache in dev mode
     cache_timeout = DEFAULT_CACHE_TIMEOUT
 
     def __init__(self, max_items=None):
@@ -239,7 +239,11 @@ def cache_key(self):
 
     def get_feed_items(self):
         # check for cached data first (skip in dev mode)
-        source_feed_data = None if (settings.DEBUG and not self.rate_limited) else cache.get(self.cache_key)
+        source_feed_data = (
+            None
+            if (settings.DEBUG and not self.rate_limited)
+            else cache.get(self.cache_key)
+        )
         if not source_feed_data:
             source_feed_data = self._get_feed_source_data()
             if not source_feed_data:
@@ -262,7 +266,7 @@ def to_feed_item(self, release):
         short_author_string = release.citation_authors
         if short_author_string and "," in short_author_string:
             short_author_string = f"{short_author_string.split(",")[0].strip()} et al."
-        
+
         feed_item = FeedItem(
             title=release.title,
             summary=release.codebase.summary or release.codebase.description.raw,
@@ -315,10 +319,12 @@ def to_feed_item(self, post):
 
 class ForumCategoryFeed(ForumFeed):
     mock = False
-    cache_timeout = 60 * 60 * 24 * 30 # 30 days
+    cache_timeout = 60 * 60 * 24 * 30  # 30 days
 
     def _get_feed_source_data(self):
-        return get_categories(number_of_categories=self.max_number_of_items, mock=self.mock)
+        return get_categories(
+            number_of_categories=self.max_number_of_items, mock=self.mock
+        )
 
     def to_feed_item(self, category):
         return FeedItem(
@@ -392,7 +398,7 @@ def get(self, request, *args, **kwargs):
                 max_items = min(100, limit_value) if limit_value > 0 else None
             except ValueError:
                 pass
-        
+
         feed_data = self.feed_class(max_items=max_items).get_feed_data()
         if feed_data is None:
             return JsonResponse({"error": "Feed not available"}, status=503)
@@ -440,7 +446,11 @@ def urlpatterns():
         ),
         path("api/feeds/events/", EventFeedView.as_view(), name="event-feed"),
         path("api/feeds/forum/", ForumFeedView.as_view(), name="forum-feed"),
-        path("api/feeds/forum-categories/", ForumCategoryFeedView.as_view(), name="forum-categories-feed"),
+        path(
+            "api/feeds/forum-categories/",
+            ForumCategoryFeedView.as_view(),
+            name="forum-categories-feed",
+        ),
         path("api/feeds/jobs/", JobFeedView.as_view(), name="job-feed"),
         path("api/feeds/yt/", YouTubeFeedView.as_view(), name="youtube-feed"),
     ]