diff --git a/.github/workflows/django-ci.yml b/.github/workflows/django-ci.yml index e0f5ad4bd..09d6a1f2d 100644 --- a/.github/workflows/django-ci.yml +++ b/.github/workflows/django-ci.yml @@ -7,8 +7,10 @@ on: branches: [main] jobs: - build: + run_tests: runs-on: ubuntu-latest + env: + DEPLOY_ENVIRONMENT: test steps: - uses: actions/checkout@v4 - name: build containers and run Django tests diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 842f9d496..d7f572116 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -9,6 +9,8 @@ on: jobs: e2e: runs-on: ubuntu-latest + env: + DEPLOY_ENVIRONMENT: test steps: - uses: actions/checkout@v4 diff --git a/Makefile b/Makefile index 35038b9c6..f255f7485 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,3 @@ -# set DEPLOY_ENVIRONMENT in config.mk -DEPLOY_ENVIRONMENT := dev - DOCKER_SHARED_DIR=docker/shared # shared directory subdirectories for postgres (data), frontend (vite), logs, static assets to be delivered by nginx # `data` directory is used for direct postgres db server-side outputs, e.g., postgres COPY commands issued in @@ -24,6 +21,7 @@ BORG_REPO_URL := https://example.com/repo.tar.xz BORG_REPO_PATH=${BUILD_DIR}/repo.tar.xz REPO_BACKUPS_PATH=${DOCKER_SHARED_DIR}/backups +# DEPLOY_ENVIRONMENT must be set in config.mk include config.mk include .env @@ -34,13 +32,13 @@ include .env .PHONY: build build: docker-compose.yml secrets $(DOCKER_SHARED_DIR) - docker compose build --pull + docker compose build --pull -q $(BORG_REPO_PATH): wget -c ${BORG_REPO_URL} -P ${BUILD_DIR} config.mk: - DEPLOY_ENVIRONMENT=${DEPLOY_ENVIRONMENT} envsubst < ${DEPLOY_CONF_DIR}/config.mk.template > config.mk + envsubst < ${DEPLOY_CONF_DIR}/config.mk.template > config.mk .PHONY: $(DOCKER_SHARED_DIR) $(DOCKER_SHARED_DIR): @@ -82,9 +80,9 @@ release-version: .env $(ENVREPLACE) TEST_BASIC_AUTH_PASSWORD $$(openssl rand -base64 42) .env .PHONY: docker-compose.yml -docker-compose.yml: base.yml dev.yml staging.yml prod.yml config.mk $(PGPASS_PATH) release-version .env +docker-compose.yml: base.yml dev.yml staging.yml test.yml prod.yml config.mk $(PGPASS_PATH) release-version .env case "$(DEPLOY_ENVIRONMENT)" in \ - dev|staging) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \ + dev|staging|test) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \ prod) docker compose -f base.yml -f staging.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \ *) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \ esac @@ -148,12 +146,15 @@ E2E_REPO_PATH=${E2E_BACKUPS_PATH}/repo $(E2E_REPO_PATH): mkdir -p $(E2E_BACKUPS_PATH) - wget -c ${BORG_REPO_URL} -P $(E2E_BACKUPS_PATH) + wget -c --no-check-certificate ${BORG_REPO_URL} -P $(E2E_BACKUPS_PATH) tar -Jxf $(E2E_BACKUPS_PATH)/repo.tar.xz -C $(E2E_BACKUPS_PATH) .PHONY: e2e e2e: docker-compose.yml secrets $(DOCKER_SHARED_DIR) $(E2E_REPO_PATH) - docker compose -f docker-compose.yml -f e2e.yml up -d --build + docker compose -f docker-compose.yml -f e2e.yml build -q + docker compose -f docker-compose.yml -f e2e.yml up -d + sleep 42 docker compose -f docker-compose.yml -f e2e.yml exec server bash -c "\ inv borg.restore --force && \ + inv db.init && \ inv prepare" diff --git a/django/core/settings/__init__.py b/django/core/settings/__init__.py index e69de29bb..9f1ca55f8 100644 --- a/django/core/settings/__init__.py +++ b/django/core/settings/__init__.py @@ -0,0 +1 @@ +from .defaults import * diff --git a/django/core/settings/defaults.py b/django/core/settings/defaults.py index b2fd554ba..f6863c770 100644 --- a/django/core/settings/defaults.py +++ b/django/core/settings/defaults.py @@ -10,8 +10,10 @@ """ import os +import sys import warnings from elasticsearch.exceptions import ElasticsearchWarning +from collections import namedtuple from enum import Enum from pathlib import Path @@ -26,15 +28,19 @@ def read_secret(file, fallback=""): return fallback +EnvConfig = namedtuple("EnvConfig", ["base_url", "label"]) + + class Environment(Enum): - DEVELOPMENT = "http://localhost:8000" - STAGING = "https://staging.comses.net" - PRODUCTION = "https://www.comses.net" - TEST = "http://localhost:8000" + DEVELOPMENT = EnvConfig(base_url="http://localhost:8000", label="DEVELOPMENT") + STAGING = EnvConfig(base_url="https://staging.comses.net", label="STAGING") + PRODUCTION = EnvConfig(base_url="https://www.comses.net", label="PRODUCTION") + # TEST is used for local and github testing, not a real environment + TEST = EnvConfig(base_url="http://localhost:8000", label="TEST") @property def base_url(self): - return self.value + return self.value.base_url @property def is_staging_or_production(self): @@ -57,7 +63,14 @@ def is_test(self): return self == Environment.TEST -DEPLOY_ENVIRONMENT = Environment.DEVELOPMENT +def set_environment(env: Environment): + global DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL + DEPLOY_ENVIRONMENT = env + # Base URL to use when referring to full URLs within the Wagtail admin backend - + # e.g. in notification emails. Don't include '/admin' or a trailing slash + WAGTAILADMIN_BASE_URL = BASE_URL = env.base_url + return DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL + # go two levels up for root project directory PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -72,13 +85,11 @@ def is_test(self): DJANGO_VITE_DEV_MODE = True -# Base URL to use when referring to full URLs within the Wagtail admin backend - -# e.g. in notification emails. Don't include '/admin' or a trailing slash -# FIXME: needs to be overridden in staging and prod after updating DEPLOY_ENVIRONMENT which is less than ideal -WAGTAILADMIN_BASE_URL = BASE_URL = DEPLOY_ENVIRONMENT.base_url +TESTING = "test" in sys.argv or "PYTEST_VERSION" in os.environ + # set up robots + sitemaps inclusion https://django-robots.readthedocs.io/en/latest/ -ROBOTS_SITEMAP_URLS = [f"{BASE_URL}/sitemap.xml"] +# ROBOTS_SITEMAP_URLS = [f"{BASE_URL}/sitemap.xml"] # wagtail config: https://docs.wagtail.io/en/v2.10.1/getting_started/integrating_into_django.html WAGTAIL_APPS = [ @@ -191,6 +202,7 @@ def is_test(self): "cdn.jsdelivr.net", # codemirror spell checker "*.comses.net", # sentry.comses.net / forum.comses.net "www.google-analytics.com", # google analytics + "export.highcharts.com", # highcharts metrics export ) CSP_FONT_SRC = ("'self'", "fonts.googleapis.com", "fonts.gstatic.com", "localhost:*") CSP_STYLE_SRC = ( @@ -433,7 +445,7 @@ def is_test(self): DJANGO_VITE_ASSETS_PATH = os.path.join(SHARE_DIR, "vite") DJANGO_VITE_STATIC_URL_PREFIX = "bundles" DJANGO_VITE_DEV_SERVER_PORT = 5173 -DJANG_VITE_MANIFEST_PATH = os.path.join( +DJANGO_VITE_MANIFEST_PATH = os.path.join( DJANGO_VITE_ASSETS_PATH, DJANGO_VITE_STATIC_URL_PREFIX, "manifest.json" ) diff --git a/django/core/settings/dev.py b/django/core/settings/dev.py index 1e35ce01a..3f9cc0182 100644 --- a/django/core/settings/dev.py +++ b/django/core/settings/dev.py @@ -3,14 +3,19 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment( + Environment.DEVELOPMENT +) + EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" -MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] -INSTALLED_APPS += [ - "debug_toolbar", - "fixture_magic", -] +if not TESTING: + INSTALLED_APPS += [ + "debug_toolbar", + "fixture_magic", + ] + MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] ALLOWED_HOSTS = ["localhost", "127.0.0.1", "server"] diff --git a/django/core/settings/e2e.py b/django/core/settings/e2e.py index 7c9e72a67..7bef8ee3e 100644 --- a/django/core/settings/e2e.py +++ b/django/core/settings/e2e.py @@ -1,6 +1,10 @@ from .test import * DEBUG = True +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment( + Environment.DEVELOPMENT +) +TESTING = True DJANGO_VITE_DEV_MODE = False diff --git a/django/core/settings/production.py b/django/core/settings/production.py index fc4e92592..04cc24123 100644 --- a/django/core/settings/production.py +++ b/django/core/settings/production.py @@ -10,7 +10,10 @@ DEBUG = False DJANGO_VITE_DEV_MODE = False -DEPLOY_ENVIRONMENT = Environment.PRODUCTION +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment( + Environment.PRODUCTION +) + EMAIL_SUBJECT_PREFIX = os.getenv("EMAIL_SUBJECT_PREFIX", "[comses.net]") # See http://django-allauth.readthedocs.io/en/latest/providers.html#orcid for more context. # @@ -64,9 +67,6 @@ ) CSP_INCLUDE_NONCE_IN = ["script-src"] -# Base URL to use when referring to full URLs within the Wagtail admin backend - -# e.g. in notification emails. Don't include '/admin' or a trailing slash -WAGTAILADMIN_BASE_URL = BASE_URL = DEPLOY_ENVIRONMENT.base_url # set up robots + sitemaps inclusion https://django-robots.readthedocs.io/en/latest/ ROBOTS_SITEMAP_URLS = [f"{BASE_URL}/sitemap.xml"] diff --git a/django/core/settings/staging.py b/django/core/settings/staging.py index 74332965c..268921ad9 100644 --- a/django/core/settings/staging.py +++ b/django/core/settings/staging.py @@ -5,7 +5,9 @@ DEBUG = False DJANGO_VITE_DEV_MODE = False -DEPLOY_ENVIRONMENT = Environment.STAGING +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment( + Environment.STAGING +) # datacite sandbox configuration inherited from defaults should suffice # DATACITE_PREFIX = "10.82853" diff --git a/django/core/settings/test.py b/django/core/settings/test.py index f699cc030..004f1d63b 100644 --- a/django/core/settings/test.py +++ b/django/core/settings/test.py @@ -2,8 +2,8 @@ from .defaults import * - -DEPLOY_ENVIRONMENT = Environment.TEST +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment(Environment.TEST) +TESTING = True ALLOWED_HOSTS = ["localhost", "127.0.0.1", "server"] diff --git a/django/core/urls.py b/django/core/urls.py index 04e767ea4..8641d2df7 100644 --- a/django/core/urls.py +++ b/django/core/urls.py @@ -77,30 +77,27 @@ def get_core_urls(): path("", include((get_core_urls(), "core"), namespace="core")), path("accounts/", include("allauth.urls")), path("django/admin/", admin.site.urls), - # Replace the default wagtail admin home page - # path('wagtail/admin/', view=wagtail_hooks.DashboardView.as_view(), name='wagtailadmin_home'), path("wagtail/admin/", include(wagtailadmin_urls)), path("api/schema/", schema_view), path("api-auth/", include("rest_framework.urls")), # configure sitemaps and robots.txt, see https://django-robots.readthedocs.io/en/latest/ - # https://docs.wagtail.io/en/v2.9.2/reference/contrib/sitemaps.html + # https://docs.wagtail.org/en/stable/reference/contrib/sitemaps.html path("sitemap.xml", sitemap), path("robots.txt", include("robots.urls")), ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) -if not settings.DEPLOY_ENVIRONMENT.is_production: - import debug_toolbar - +if settings.DEPLOY_ENVIRONMENT.is_development: urlpatterns += [ path("argh/", handler500, name="error"), path("make-error/", views.make_error), - path("__debug__/", include(debug_toolbar.urls)), - ] + ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + if not settings.TESTING: + from debug_toolbar.toolbar import debug_toolbar_urls + + urlpatterns += debug_toolbar_urls() -if settings.DEPLOY_ENVIRONMENT.is_development: - urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) # NB: wagtail_urls are the catchall, must be last urlpatterns.append(path("", include(wagtail_urls))) diff --git a/django/curator/invoke_tasks/utils.py b/django/curator/invoke_tasks/utils.py index 3a2dff4ef..9bc9f13a0 100644 --- a/django/curator/invoke_tasks/utils.py +++ b/django/curator/invoke_tasks/utils.py @@ -29,9 +29,9 @@ def dj(ctx, command, **kwargs): """ Run a Django manage.py command on the server. """ + django_settings_module = os.environ.get("DJANGO_SETTINGS_MODULE") + invocation = f"python3 manage.py {command} --settings {django_settings_module}" + print("Invoking command: ", invocation) ctx.run( - "{python} manage.py {dj_command} --settings {project_conf}".format( - dj_command=command, **env - ), - **kwargs, + invocation, env={"DJANGO_SETTINGS_MODULE": django_settings_module}, **kwargs ) diff --git a/django/deploy/dev.sh b/django/deploy/dev.sh index 25dbabd44..577ef0b13 100755 --- a/django/deploy/dev.sh +++ b/django/deploy/dev.sh @@ -16,4 +16,5 @@ initdb() { fi } initdb -exec /code/manage.py runserver 0.0.0.0:8000 +echo "Running dev server with ${DJANGO_SETTINGS_MODULE}" +exec env DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} /code/manage.py runserver 0.0.0.0:8000 diff --git a/django/deploy/test.sh b/django/deploy/test.sh index bab46edf3..d3d78fda7 100755 --- a/django/deploy/test.sh +++ b/django/deploy/test.sh @@ -1,13 +1,11 @@ #!/bin/sh -export DJANGO_SETTINGS_MODULE="core.settings.test" - chmod a+x /code/deploy/*.sh; initdb() { cd /code; echo "Destroying and initializing database from scratch" - invoke db.init + exec env DJANGO_SETTINGS_MODULE="core.settings.test" invoke db.init } initdb -exec invoke prepare test --tests="$@" --coverage +exec env DJANGO_SETTINGS_MODULE="core.settings.test" invoke prepare test --tests="$@" --coverage diff --git a/django/home/feeds.py b/django/home/feeds.py index 1e57c8d29..07ca458f4 100644 --- a/django/home/feeds.py +++ b/django/home/feeds.py @@ -204,7 +204,7 @@ class FeedItem: class AbstractFeed(ABC): max_number_of_items = DEFAULT_HOMEPAGE_FEED_MAX_ITEMS _cache_key = None # subclasses can define a custom cache key if needed - rate_limited = False # set to True if the feed is rate limited to cache in dev mode + rate_limited = False # set to True if the feed is rate limited to cache in dev mode cache_timeout = DEFAULT_CACHE_TIMEOUT def __init__(self, max_items=None): @@ -239,7 +239,11 @@ def cache_key(self): def get_feed_items(self): # check for cached data first (skip in dev mode) - source_feed_data = None if (settings.DEBUG and not self.rate_limited) else cache.get(self.cache_key) + source_feed_data = ( + None + if (settings.DEBUG and not self.rate_limited) + else cache.get(self.cache_key) + ) if not source_feed_data: source_feed_data = self._get_feed_source_data() if not source_feed_data: @@ -262,7 +266,7 @@ def to_feed_item(self, release): short_author_string = release.citation_authors if short_author_string and "," in short_author_string: short_author_string = f"{short_author_string.split(",")[0].strip()} et al." - + feed_item = FeedItem( title=release.title, summary=release.codebase.summary or release.codebase.description.raw, @@ -315,10 +319,12 @@ def to_feed_item(self, post): class ForumCategoryFeed(ForumFeed): mock = False - cache_timeout = 60 * 60 * 24 * 30 # 30 days + cache_timeout = 60 * 60 * 24 * 30 # 30 days def _get_feed_source_data(self): - return get_categories(number_of_categories=self.max_number_of_items, mock=self.mock) + return get_categories( + number_of_categories=self.max_number_of_items, mock=self.mock + ) def to_feed_item(self, category): return FeedItem( @@ -392,7 +398,7 @@ def get(self, request, *args, **kwargs): max_items = min(100, limit_value) if limit_value > 0 else None except ValueError: pass - + feed_data = self.feed_class(max_items=max_items).get_feed_data() if feed_data is None: return JsonResponse({"error": "Feed not available"}, status=503) @@ -440,7 +446,11 @@ def urlpatterns(): ), path("api/feeds/events/", EventFeedView.as_view(), name="event-feed"), path("api/feeds/forum/", ForumFeedView.as_view(), name="forum-feed"), - path("api/feeds/forum-categories/", ForumCategoryFeedView.as_view(), name="forum-categories-feed"), + path( + "api/feeds/forum-categories/", + ForumCategoryFeedView.as_view(), + name="forum-categories-feed", + ), path("api/feeds/jobs/", JobFeedView.as_view(), name="job-feed"), path("api/feeds/yt/", YouTubeFeedView.as_view(), name="youtube-feed"), ] diff --git a/django/home/management/commands/update_digest_archive.py b/django/home/management/commands/update_digest_archive.py index f5e5b46b7..53a9244e1 100644 --- a/django/home/management/commands/update_digest_archive.py +++ b/django/home/management/commands/update_digest_archive.py @@ -22,7 +22,9 @@ def handle(self, *args, **options): ComsesDigest.objects.all().delete() err_msg = "" - response = requests.get(ZENODO_COMMUNITY_API_URL, params={"size": 1000, "sort": "publication-desc"}) + response = requests.get( + ZENODO_COMMUNITY_API_URL, params={"size": 1000, "sort": "publication-desc"} + ) if response.status_code != 200: logger.error("Failed to fetch Zenodo records for CoMSES Digest.") return @@ -35,7 +37,9 @@ def handle(self, *args, **options): title=record["metadata"]["title"], volume=int(record["metadata"]["journal"]["volume"]), issue_number=int(record["metadata"]["journal"]["issue"]), - publication_date=datetime.strptime(record["metadata"]["publication_date"], "%Y-%m-%d").date(), + publication_date=datetime.strptime( + record["metadata"]["publication_date"], "%Y-%m-%d" + ).date(), url=record["links"]["latest_html"], ) except Exception as e: @@ -49,4 +53,6 @@ def handle(self, *args, **options): ) logger.error(err_msg) else: - logger.info(f"\n\nSuccessfully indexed all digest records from Zenodo community.") + logger.info( + f"\n\nSuccessfully indexed all digest records from Zenodo community." + ) diff --git a/django/home/models.py b/django/home/models.py index 942df5014..b4ea723f2 100644 --- a/django/home/models.py +++ b/django/home/models.py @@ -147,7 +147,9 @@ class LandingPage(Page): mission_statement = models.CharField(max_length=512) library_title = models.CharField(max_length=255, default="The CoMSES Model Library") library_blurb = MarkdownField(max_length=1024, blank=True) - community_title = models.CharField(max_length=255, default="A community of researchers") + community_title = models.CharField( + max_length=255, default="A community of researchers" + ) community_blurb = MarkdownField(max_length=1024, blank=True) resources_title = models.CharField(max_length=255, default="Resources for modeling") resources_blurb = MarkdownField(max_length=1024, blank=True) diff --git a/django/home/tests/test_feeds.py b/django/home/tests/test_feeds.py index c422010a0..bc874fa10 100644 --- a/django/home/tests/test_feeds.py +++ b/django/home/tests/test_feeds.py @@ -82,7 +82,9 @@ def test_forum_feed(self): self._verify_feed_structure(ForumFeed(mock=True)) def test_reviewed_model_feed(self): - self._verify_feed_structure(ReviewedModelFeed(), CodebaseRelease.objects.reviewed().count()) + self._verify_feed_structure( + ReviewedModelFeed(), CodebaseRelease.objects.reviewed().count() + ) def test_event_feed(self): self._verify_feed_structure(EventFeed(), Event.objects.count()) diff --git a/django/library/doi.py b/django/library/doi.py index 060fa3389..713c61da4 100644 --- a/django/library/doi.py +++ b/django/library/doi.py @@ -6,6 +6,7 @@ from collections import defaultdict from django.conf import settings +from huey.contrib.djhuey import on_commit_task from .models import ( Codebase, @@ -101,7 +102,7 @@ def print_console_message(dry_run: bool, interactive: bool): def is_valid_doi(doi: str) -> bool: # checks if DOI is formatted like this "00.12345/q2xt-rj46" if doi: - return re.match(DOI_PATTERN, doi) + return bool(re.match(DOI_PATTERN, doi, re.IGNORECASE)) return False @@ -195,6 +196,7 @@ def mint_public_doi(self, codebase_or_release: Codebase | CodebaseRelease): """ Mint a public DOI for the given codebase or release. + Args: codebase_or_release (Codebase | CodebaseRelease): The codebase or release for which to mint a DOI. @@ -202,7 +204,11 @@ def mint_public_doi(self, codebase_or_release: Codebase | CodebaseRelease): tuple: A tuple containing the DOI and a boolean indicating if the minting was successful. """ if self.dry_run: - return "XX.DRYXX/XXXX-XRUN", True + return ( + DataCiteRegistrationLog.objects.mock(codebase_or_release), + True, + ) + # clear cached datacite property to ensure it is fresh if hasattr(codebase_or_release, "datacite"): del codebase_or_release.datacite diff --git a/django/library/jinja2/library/codebases/releases/retrieve.jinja b/django/library/jinja2/library/codebases/releases/retrieve.jinja index 7340faa89..7a7339402 100644 --- a/django/library/jinja2/library/codebases/releases/retrieve.jinja +++ b/django/library/jinja2/library/codebases/releases/retrieve.jinja @@ -94,8 +94,8 @@ {% elif release.is_under_review %}