diff --git a/.github/workflows/django-ci.yml b/.github/workflows/django-ci.yml index e0f5ad4bd..09d6a1f2d 100644 --- a/.github/workflows/django-ci.yml +++ b/.github/workflows/django-ci.yml @@ -7,8 +7,10 @@ on: branches: [main] jobs: - build: + run_tests: runs-on: ubuntu-latest + env: + DEPLOY_ENVIRONMENT: test steps: - uses: actions/checkout@v4 - name: build containers and run Django tests diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 842f9d496..d7f572116 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -9,6 +9,8 @@ on: jobs: e2e: runs-on: ubuntu-latest + env: + DEPLOY_ENVIRONMENT: test steps: - uses: actions/checkout@v4 diff --git a/Makefile b/Makefile index 35038b9c6..f255f7485 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,3 @@ -# set DEPLOY_ENVIRONMENT in config.mk -DEPLOY_ENVIRONMENT := dev - DOCKER_SHARED_DIR=docker/shared # shared directory subdirectories for postgres (data), frontend (vite), logs, static assets to be delivered by nginx # `data` directory is used for direct postgres db server-side outputs, e.g., postgres COPY commands issued in @@ -24,6 +21,7 @@ BORG_REPO_URL := https://example.com/repo.tar.xz BORG_REPO_PATH=${BUILD_DIR}/repo.tar.xz REPO_BACKUPS_PATH=${DOCKER_SHARED_DIR}/backups +# DEPLOY_ENVIRONMENT must be set in config.mk include config.mk include .env @@ -34,13 +32,13 @@ include .env .PHONY: build build: docker-compose.yml secrets $(DOCKER_SHARED_DIR) - docker compose build --pull + docker compose build --pull -q $(BORG_REPO_PATH): wget -c ${BORG_REPO_URL} -P ${BUILD_DIR} config.mk: - DEPLOY_ENVIRONMENT=${DEPLOY_ENVIRONMENT} envsubst < ${DEPLOY_CONF_DIR}/config.mk.template > config.mk + envsubst < ${DEPLOY_CONF_DIR}/config.mk.template > config.mk .PHONY: $(DOCKER_SHARED_DIR) $(DOCKER_SHARED_DIR): @@ -82,9 +80,9 @@ release-version: .env $(ENVREPLACE) TEST_BASIC_AUTH_PASSWORD $$(openssl rand -base64 42) .env .PHONY: docker-compose.yml -docker-compose.yml: base.yml dev.yml staging.yml prod.yml config.mk $(PGPASS_PATH) release-version .env +docker-compose.yml: base.yml dev.yml staging.yml test.yml prod.yml config.mk $(PGPASS_PATH) release-version .env case "$(DEPLOY_ENVIRONMENT)" in \ - dev|staging) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \ + dev|staging|test) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \ prod) docker compose -f base.yml -f staging.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \ *) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \ esac @@ -148,12 +146,15 @@ E2E_REPO_PATH=${E2E_BACKUPS_PATH}/repo $(E2E_REPO_PATH): mkdir -p $(E2E_BACKUPS_PATH) - wget -c ${BORG_REPO_URL} -P $(E2E_BACKUPS_PATH) + wget -c --no-check-certificate ${BORG_REPO_URL} -P $(E2E_BACKUPS_PATH) tar -Jxf $(E2E_BACKUPS_PATH)/repo.tar.xz -C $(E2E_BACKUPS_PATH) .PHONY: e2e e2e: docker-compose.yml secrets $(DOCKER_SHARED_DIR) $(E2E_REPO_PATH) - docker compose -f docker-compose.yml -f e2e.yml up -d --build + docker compose -f docker-compose.yml -f e2e.yml build -q + docker compose -f docker-compose.yml -f e2e.yml up -d + sleep 42 docker compose -f docker-compose.yml -f e2e.yml exec server bash -c "\ inv borg.restore --force && \ + inv db.init && \ inv prepare" diff --git a/django/core/settings/__init__.py b/django/core/settings/__init__.py index e69de29bb..9f1ca55f8 100644 --- a/django/core/settings/__init__.py +++ b/django/core/settings/__init__.py @@ -0,0 +1 @@ +from .defaults import * diff --git a/django/core/settings/defaults.py b/django/core/settings/defaults.py index b2fd554ba..f6863c770 100644 --- a/django/core/settings/defaults.py +++ b/django/core/settings/defaults.py @@ -10,8 +10,10 @@ """ import os +import sys import warnings from elasticsearch.exceptions import ElasticsearchWarning +from collections import namedtuple from enum import Enum from pathlib import Path @@ -26,15 +28,19 @@ def read_secret(file, fallback=""): return fallback +EnvConfig = namedtuple("EnvConfig", ["base_url", "label"]) + + class Environment(Enum): - DEVELOPMENT = "http://localhost:8000" - STAGING = "https://staging.comses.net" - PRODUCTION = "https://www.comses.net" - TEST = "http://localhost:8000" + DEVELOPMENT = EnvConfig(base_url="http://localhost:8000", label="DEVELOPMENT") + STAGING = EnvConfig(base_url="https://staging.comses.net", label="STAGING") + PRODUCTION = EnvConfig(base_url="https://www.comses.net", label="PRODUCTION") + # TEST is used for local and github testing, not a real environment + TEST = EnvConfig(base_url="http://localhost:8000", label="TEST") @property def base_url(self): - return self.value + return self.value.base_url @property def is_staging_or_production(self): @@ -57,7 +63,14 @@ def is_test(self): return self == Environment.TEST -DEPLOY_ENVIRONMENT = Environment.DEVELOPMENT +def set_environment(env: Environment): + global DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL + DEPLOY_ENVIRONMENT = env + # Base URL to use when referring to full URLs within the Wagtail admin backend - + # e.g. in notification emails. Don't include '/admin' or a trailing slash + WAGTAILADMIN_BASE_URL = BASE_URL = env.base_url + return DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL + # go two levels up for root project directory PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -72,13 +85,11 @@ def is_test(self): DJANGO_VITE_DEV_MODE = True -# Base URL to use when referring to full URLs within the Wagtail admin backend - -# e.g. in notification emails. Don't include '/admin' or a trailing slash -# FIXME: needs to be overridden in staging and prod after updating DEPLOY_ENVIRONMENT which is less than ideal -WAGTAILADMIN_BASE_URL = BASE_URL = DEPLOY_ENVIRONMENT.base_url +TESTING = "test" in sys.argv or "PYTEST_VERSION" in os.environ + # set up robots + sitemaps inclusion https://django-robots.readthedocs.io/en/latest/ -ROBOTS_SITEMAP_URLS = [f"{BASE_URL}/sitemap.xml"] +# ROBOTS_SITEMAP_URLS = [f"{BASE_URL}/sitemap.xml"] # wagtail config: https://docs.wagtail.io/en/v2.10.1/getting_started/integrating_into_django.html WAGTAIL_APPS = [ @@ -191,6 +202,7 @@ def is_test(self): "cdn.jsdelivr.net", # codemirror spell checker "*.comses.net", # sentry.comses.net / forum.comses.net "www.google-analytics.com", # google analytics + "export.highcharts.com", # highcharts metrics export ) CSP_FONT_SRC = ("'self'", "fonts.googleapis.com", "fonts.gstatic.com", "localhost:*") CSP_STYLE_SRC = ( @@ -433,7 +445,7 @@ def is_test(self): DJANGO_VITE_ASSETS_PATH = os.path.join(SHARE_DIR, "vite") DJANGO_VITE_STATIC_URL_PREFIX = "bundles" DJANGO_VITE_DEV_SERVER_PORT = 5173 -DJANG_VITE_MANIFEST_PATH = os.path.join( +DJANGO_VITE_MANIFEST_PATH = os.path.join( DJANGO_VITE_ASSETS_PATH, DJANGO_VITE_STATIC_URL_PREFIX, "manifest.json" ) diff --git a/django/core/settings/dev.py b/django/core/settings/dev.py index 1e35ce01a..3f9cc0182 100644 --- a/django/core/settings/dev.py +++ b/django/core/settings/dev.py @@ -3,14 +3,19 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment( + Environment.DEVELOPMENT +) + EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" -MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] -INSTALLED_APPS += [ - "debug_toolbar", - "fixture_magic", -] +if not TESTING: + INSTALLED_APPS += [ + "debug_toolbar", + "fixture_magic", + ] + MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] ALLOWED_HOSTS = ["localhost", "127.0.0.1", "server"] diff --git a/django/core/settings/e2e.py b/django/core/settings/e2e.py index 7c9e72a67..7bef8ee3e 100644 --- a/django/core/settings/e2e.py +++ b/django/core/settings/e2e.py @@ -1,6 +1,10 @@ from .test import * DEBUG = True +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment( + Environment.DEVELOPMENT +) +TESTING = True DJANGO_VITE_DEV_MODE = False diff --git a/django/core/settings/production.py b/django/core/settings/production.py index fc4e92592..04cc24123 100644 --- a/django/core/settings/production.py +++ b/django/core/settings/production.py @@ -10,7 +10,10 @@ DEBUG = False DJANGO_VITE_DEV_MODE = False -DEPLOY_ENVIRONMENT = Environment.PRODUCTION +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment( + Environment.PRODUCTION +) + EMAIL_SUBJECT_PREFIX = os.getenv("EMAIL_SUBJECT_PREFIX", "[comses.net]") # See http://django-allauth.readthedocs.io/en/latest/providers.html#orcid for more context. # @@ -64,9 +67,6 @@ ) CSP_INCLUDE_NONCE_IN = ["script-src"] -# Base URL to use when referring to full URLs within the Wagtail admin backend - -# e.g. in notification emails. Don't include '/admin' or a trailing slash -WAGTAILADMIN_BASE_URL = BASE_URL = DEPLOY_ENVIRONMENT.base_url # set up robots + sitemaps inclusion https://django-robots.readthedocs.io/en/latest/ ROBOTS_SITEMAP_URLS = [f"{BASE_URL}/sitemap.xml"] diff --git a/django/core/settings/staging.py b/django/core/settings/staging.py index 74332965c..268921ad9 100644 --- a/django/core/settings/staging.py +++ b/django/core/settings/staging.py @@ -5,7 +5,9 @@ DEBUG = False DJANGO_VITE_DEV_MODE = False -DEPLOY_ENVIRONMENT = Environment.STAGING +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment( + Environment.STAGING +) # datacite sandbox configuration inherited from defaults should suffice # DATACITE_PREFIX = "10.82853" diff --git a/django/core/settings/test.py b/django/core/settings/test.py index f699cc030..004f1d63b 100644 --- a/django/core/settings/test.py +++ b/django/core/settings/test.py @@ -2,8 +2,8 @@ from .defaults import * - -DEPLOY_ENVIRONMENT = Environment.TEST +DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment(Environment.TEST) +TESTING = True ALLOWED_HOSTS = ["localhost", "127.0.0.1", "server"] diff --git a/django/core/urls.py b/django/core/urls.py index 04e767ea4..8641d2df7 100644 --- a/django/core/urls.py +++ b/django/core/urls.py @@ -77,30 +77,27 @@ def get_core_urls(): path("", include((get_core_urls(), "core"), namespace="core")), path("accounts/", include("allauth.urls")), path("django/admin/", admin.site.urls), - # Replace the default wagtail admin home page - # path('wagtail/admin/', view=wagtail_hooks.DashboardView.as_view(), name='wagtailadmin_home'), path("wagtail/admin/", include(wagtailadmin_urls)), path("api/schema/", schema_view), path("api-auth/", include("rest_framework.urls")), # configure sitemaps and robots.txt, see https://django-robots.readthedocs.io/en/latest/ - # https://docs.wagtail.io/en/v2.9.2/reference/contrib/sitemaps.html + # https://docs.wagtail.org/en/stable/reference/contrib/sitemaps.html path("sitemap.xml", sitemap), path("robots.txt", include("robots.urls")), ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) -if not settings.DEPLOY_ENVIRONMENT.is_production: - import debug_toolbar - +if settings.DEPLOY_ENVIRONMENT.is_development: urlpatterns += [ path("argh/", handler500, name="error"), path("make-error/", views.make_error), - path("__debug__/", include(debug_toolbar.urls)), - ] + ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + if not settings.TESTING: + from debug_toolbar.toolbar import debug_toolbar_urls + + urlpatterns += debug_toolbar_urls() -if settings.DEPLOY_ENVIRONMENT.is_development: - urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) # NB: wagtail_urls are the catchall, must be last urlpatterns.append(path("", include(wagtail_urls))) diff --git a/django/curator/invoke_tasks/utils.py b/django/curator/invoke_tasks/utils.py index 3a2dff4ef..9bc9f13a0 100644 --- a/django/curator/invoke_tasks/utils.py +++ b/django/curator/invoke_tasks/utils.py @@ -29,9 +29,9 @@ def dj(ctx, command, **kwargs): """ Run a Django manage.py command on the server. """ + django_settings_module = os.environ.get("DJANGO_SETTINGS_MODULE") + invocation = f"python3 manage.py {command} --settings {django_settings_module}" + print("Invoking command: ", invocation) ctx.run( - "{python} manage.py {dj_command} --settings {project_conf}".format( - dj_command=command, **env - ), - **kwargs, + invocation, env={"DJANGO_SETTINGS_MODULE": django_settings_module}, **kwargs ) diff --git a/django/deploy/dev.sh b/django/deploy/dev.sh index 25dbabd44..577ef0b13 100755 --- a/django/deploy/dev.sh +++ b/django/deploy/dev.sh @@ -16,4 +16,5 @@ initdb() { fi } initdb -exec /code/manage.py runserver 0.0.0.0:8000 +echo "Running dev server with ${DJANGO_SETTINGS_MODULE}" +exec env DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} /code/manage.py runserver 0.0.0.0:8000 diff --git a/django/deploy/test.sh b/django/deploy/test.sh index bab46edf3..d3d78fda7 100755 --- a/django/deploy/test.sh +++ b/django/deploy/test.sh @@ -1,13 +1,11 @@ #!/bin/sh -export DJANGO_SETTINGS_MODULE="core.settings.test" - chmod a+x /code/deploy/*.sh; initdb() { cd /code; echo "Destroying and initializing database from scratch" - invoke db.init + exec env DJANGO_SETTINGS_MODULE="core.settings.test" invoke db.init } initdb -exec invoke prepare test --tests="$@" --coverage +exec env DJANGO_SETTINGS_MODULE="core.settings.test" invoke prepare test --tests="$@" --coverage diff --git a/django/home/feeds.py b/django/home/feeds.py index 1e57c8d29..07ca458f4 100644 --- a/django/home/feeds.py +++ b/django/home/feeds.py @@ -204,7 +204,7 @@ class FeedItem: class AbstractFeed(ABC): max_number_of_items = DEFAULT_HOMEPAGE_FEED_MAX_ITEMS _cache_key = None # subclasses can define a custom cache key if needed - rate_limited = False # set to True if the feed is rate limited to cache in dev mode + rate_limited = False # set to True if the feed is rate limited to cache in dev mode cache_timeout = DEFAULT_CACHE_TIMEOUT def __init__(self, max_items=None): @@ -239,7 +239,11 @@ def cache_key(self): def get_feed_items(self): # check for cached data first (skip in dev mode) - source_feed_data = None if (settings.DEBUG and not self.rate_limited) else cache.get(self.cache_key) + source_feed_data = ( + None + if (settings.DEBUG and not self.rate_limited) + else cache.get(self.cache_key) + ) if not source_feed_data: source_feed_data = self._get_feed_source_data() if not source_feed_data: @@ -262,7 +266,7 @@ def to_feed_item(self, release): short_author_string = release.citation_authors if short_author_string and "," in short_author_string: short_author_string = f"{short_author_string.split(",")[0].strip()} et al." - + feed_item = FeedItem( title=release.title, summary=release.codebase.summary or release.codebase.description.raw, @@ -315,10 +319,12 @@ def to_feed_item(self, post): class ForumCategoryFeed(ForumFeed): mock = False - cache_timeout = 60 * 60 * 24 * 30 # 30 days + cache_timeout = 60 * 60 * 24 * 30 # 30 days def _get_feed_source_data(self): - return get_categories(number_of_categories=self.max_number_of_items, mock=self.mock) + return get_categories( + number_of_categories=self.max_number_of_items, mock=self.mock + ) def to_feed_item(self, category): return FeedItem( @@ -392,7 +398,7 @@ def get(self, request, *args, **kwargs): max_items = min(100, limit_value) if limit_value > 0 else None except ValueError: pass - + feed_data = self.feed_class(max_items=max_items).get_feed_data() if feed_data is None: return JsonResponse({"error": "Feed not available"}, status=503) @@ -440,7 +446,11 @@ def urlpatterns(): ), path("api/feeds/events/", EventFeedView.as_view(), name="event-feed"), path("api/feeds/forum/", ForumFeedView.as_view(), name="forum-feed"), - path("api/feeds/forum-categories/", ForumCategoryFeedView.as_view(), name="forum-categories-feed"), + path( + "api/feeds/forum-categories/", + ForumCategoryFeedView.as_view(), + name="forum-categories-feed", + ), path("api/feeds/jobs/", JobFeedView.as_view(), name="job-feed"), path("api/feeds/yt/", YouTubeFeedView.as_view(), name="youtube-feed"), ] diff --git a/django/home/management/commands/update_digest_archive.py b/django/home/management/commands/update_digest_archive.py index f5e5b46b7..53a9244e1 100644 --- a/django/home/management/commands/update_digest_archive.py +++ b/django/home/management/commands/update_digest_archive.py @@ -22,7 +22,9 @@ def handle(self, *args, **options): ComsesDigest.objects.all().delete() err_msg = "" - response = requests.get(ZENODO_COMMUNITY_API_URL, params={"size": 1000, "sort": "publication-desc"}) + response = requests.get( + ZENODO_COMMUNITY_API_URL, params={"size": 1000, "sort": "publication-desc"} + ) if response.status_code != 200: logger.error("Failed to fetch Zenodo records for CoMSES Digest.") return @@ -35,7 +37,9 @@ def handle(self, *args, **options): title=record["metadata"]["title"], volume=int(record["metadata"]["journal"]["volume"]), issue_number=int(record["metadata"]["journal"]["issue"]), - publication_date=datetime.strptime(record["metadata"]["publication_date"], "%Y-%m-%d").date(), + publication_date=datetime.strptime( + record["metadata"]["publication_date"], "%Y-%m-%d" + ).date(), url=record["links"]["latest_html"], ) except Exception as e: @@ -49,4 +53,6 @@ def handle(self, *args, **options): ) logger.error(err_msg) else: - logger.info(f"\n\nSuccessfully indexed all digest records from Zenodo community.") + logger.info( + f"\n\nSuccessfully indexed all digest records from Zenodo community." + ) diff --git a/django/home/models.py b/django/home/models.py index 942df5014..b4ea723f2 100644 --- a/django/home/models.py +++ b/django/home/models.py @@ -147,7 +147,9 @@ class LandingPage(Page): mission_statement = models.CharField(max_length=512) library_title = models.CharField(max_length=255, default="The CoMSES Model Library") library_blurb = MarkdownField(max_length=1024, blank=True) - community_title = models.CharField(max_length=255, default="A community of researchers") + community_title = models.CharField( + max_length=255, default="A community of researchers" + ) community_blurb = MarkdownField(max_length=1024, blank=True) resources_title = models.CharField(max_length=255, default="Resources for modeling") resources_blurb = MarkdownField(max_length=1024, blank=True) diff --git a/django/home/tests/test_feeds.py b/django/home/tests/test_feeds.py index c422010a0..bc874fa10 100644 --- a/django/home/tests/test_feeds.py +++ b/django/home/tests/test_feeds.py @@ -82,7 +82,9 @@ def test_forum_feed(self): self._verify_feed_structure(ForumFeed(mock=True)) def test_reviewed_model_feed(self): - self._verify_feed_structure(ReviewedModelFeed(), CodebaseRelease.objects.reviewed().count()) + self._verify_feed_structure( + ReviewedModelFeed(), CodebaseRelease.objects.reviewed().count() + ) def test_event_feed(self): self._verify_feed_structure(EventFeed(), Event.objects.count()) diff --git a/django/library/doi.py b/django/library/doi.py index 060fa3389..713c61da4 100644 --- a/django/library/doi.py +++ b/django/library/doi.py @@ -6,6 +6,7 @@ from collections import defaultdict from django.conf import settings +from huey.contrib.djhuey import on_commit_task from .models import ( Codebase, @@ -101,7 +102,7 @@ def print_console_message(dry_run: bool, interactive: bool): def is_valid_doi(doi: str) -> bool: # checks if DOI is formatted like this "00.12345/q2xt-rj46" if doi: - return re.match(DOI_PATTERN, doi) + return bool(re.match(DOI_PATTERN, doi, re.IGNORECASE)) return False @@ -195,6 +196,7 @@ def mint_public_doi(self, codebase_or_release: Codebase | CodebaseRelease): """ Mint a public DOI for the given codebase or release. + Args: codebase_or_release (Codebase | CodebaseRelease): The codebase or release for which to mint a DOI. @@ -202,7 +204,11 @@ def mint_public_doi(self, codebase_or_release: Codebase | CodebaseRelease): tuple: A tuple containing the DOI and a boolean indicating if the minting was successful. """ if self.dry_run: - return "XX.DRYXX/XXXX-XRUN", True + return ( + DataCiteRegistrationLog.objects.mock(codebase_or_release), + True, + ) + # clear cached datacite property to ensure it is fresh if hasattr(codebase_or_release, "datacite"): del codebase_or_release.datacite diff --git a/django/library/jinja2/library/codebases/releases/retrieve.jinja b/django/library/jinja2/library/codebases/releases/retrieve.jinja index 7340faa89..7a7339402 100644 --- a/django/library/jinja2/library/codebases/releases/retrieve.jinja +++ b/django/library/jinja2/library/codebases/releases/retrieve.jinja @@ -94,8 +94,8 @@ {% elif release.is_under_review %}
This release is currently undergoing peer review and must remain unpublished until complete.
{% elif release.is_review_complete %} -
This release has undergone peer review and is currently unpublished. You can - publish it here +
This release has passed peer review but is currently unpublished. + Click here to publish it and receive a DOI.
{% elif release.is_draft %}
The release you are viewing is currently a draft.
diff --git a/django/library/models.py b/django/library/models.py index 9360f475a..ca82a1aea 100644 --- a/django/library/models.py +++ b/django/library/models.py @@ -7,17 +7,14 @@ from string import Template import uuid import semver -import uuid -from collections import OrderedDict from datetime import timedelta from packaging.version import Version from abc import ABC -from collections import OrderedDict, defaultdict +from collections import defaultdict from datetime import date, timedelta from django.conf import settings from django.contrib.postgres.fields import ArrayField -from django.core.cache import cache from django.core.files.images import ImageFile from django.core.files.storage import FileSystemStorage from django.db import models, transaction @@ -53,13 +50,13 @@ from core.queryset import get_viewable_objects_for_user from core.utils import send_markdown_email from core.view_helpers import get_search_queryset -from .metadata import CodeMetaConverter, DataCiteConverter, CitationFileFormatConverter from .fs import ( CodebaseReleaseFsApi, StagingDirectories, FileCategoryDirectories, MessageLevels, ) +from .metadata import CodeMetaConverter, DataCiteConverter, CitationFileFormatConverter logger = logging.getLogger(__name__) @@ -1194,14 +1191,15 @@ def most_recently_reviewed(self, number=10, published_only=True): qs = ( self.reviewed() .select_related("codebase", "submitter__member_profile", "review") - .filter(review__event_set__action='RELEASE_CERTIFIED') + .filter(review__event_set__action="RELEASE_CERTIFIED") ) if published_only: qs = qs.public() - + # order by the certification event date return qs.order_by("-review__event_set__date_created").distinct()[:number] + @add_to_comses_permission_whitelist class CodebaseRelease(index.Indexed, ClusterableModel): """ @@ -1820,6 +1818,13 @@ def add_contributor( def publish(self): self.validate_publishable() self._publish() + if self.peer_reviewed: + # if this release is peer reviewed schedule a DOI minting + from .tasks import schedule_mint_public_doi + + schedule_mint_public_doi( + self, dry_run=settings.DEPLOY_ENVIRONMENT.is_development + ) def _publish(self): if not self.live: @@ -3169,6 +3174,27 @@ class PeerReviewEventLog(models.Model): class DataCiteRegistrationLogQuerySet(models.QuerySet): + def mock(self, codebase_or_release, **kwargs): + """ + Returns a mock DataCiteRegistrationLog entry for testing + """ + return DataCiteRegistrationLog( + codebase=( + codebase_or_release + if isinstance(codebase_or_release, Codebase) + else None + ), + release=( + codebase_or_release + if isinstance(codebase_or_release, CodebaseRelease) + else None + ), + doi="10.1234/XYZZY.DRY.RUN", + metadata_hash="dry-run-metadata-hash", + http_status=200, + message="Mock log entry for testing", + ) + def latest_entry(self, codebase_or_release, **kwargs): """ Returns the latest "successful" (200 status code from DataCite) diff --git a/django/library/tasks.py b/django/library/tasks.py index 239d79460..a055e7097 100644 --- a/django/library/tasks.py +++ b/django/library/tasks.py @@ -1,6 +1,7 @@ -from huey.contrib.djhuey import db_task +from huey.contrib.djhuey import db_task, on_commit_task -from .models import Codebase, CodebaseRelease +from .doi import DataCiteApi +from .models import CodebaseRelease import logging @@ -10,6 +11,21 @@ @db_task(retries=1, retry_delay=30) def update_fs_release_metadata(release_id: int): release = CodebaseRelease.objects.get(id=release_id) - codebase = release.codebase fs_api = release.get_fs_api() fs_api.rebuild(metadata_only=True) + + +@on_commit_task() +def schedule_mint_public_doi(release: CodebaseRelease, dry_run: bool = False): + """ + Mint a DOI for the given release. + + Args: + release (CodebaseRelease): The release for which to mint a DOI. + dry_run (bool, optional): Flag indicating whether the operation should be performed in dry run mode. + Defaults to False. + + Returns: + A tuple of DataCiteRegistrationLog or None and a boolean indicating whether the operation was successful + """ + return DataCiteApi(dry_run=dry_run).mint_public_doi(release) diff --git a/django/library/urls.py b/django/library/urls.py index 06379e7f5..ef3524210 100644 --- a/django/library/urls.py +++ b/django/library/urls.py @@ -38,7 +38,7 @@ basename="codebaserelease-share", ) -if settings.DEPLOY_ENVIRONMENT == Environment.DEVELOPMENT: +if settings.DEPLOY_ENVIRONMENT.is_development: router.register( r"test_codebases", views.DevelopmentCodebaseDeleteView, diff --git a/django/manage.py b/django/manage.py index 4ae463618..33e31407d 100755 --- a/django/manage.py +++ b/django/manage.py @@ -7,4 +7,5 @@ from django.core.management import execute_from_command_line + print("executing with: ", os.environ.get("DJANGO_SETTINGS_MODULE")) execute_from_command_line(sys.argv) diff --git a/django/requirements-test.txt b/django/requirements-test.txt new file mode 100644 index 000000000..ffab80caa --- /dev/null +++ b/django/requirements-test.txt @@ -0,0 +1,8 @@ +-r requirements.txt +# test requirements +coverage +coveralls +# https://hypothesis.readthedocs.io/en/latest/django.html +hypothesis[django] +sphinx +sphinx_rtd_theme diff --git a/django/requirements.txt b/django/requirements.txt index 315e5879c..1d61eefcc 100644 --- a/django/requirements.txt +++ b/django/requirements.txt @@ -20,7 +20,7 @@ django-rest-swagger==2.2.0 django-reversion==5.1.0 django-robots==6.1 django-timezone-field==7.0.0 -django-vite==3.0.5 # latest is 3.0.5 +django-vite==3.0.5 django-waffle==4.2.0 djangorestframework==3.15.2 djangorestframework-camel-case==1.4.2 @@ -28,7 +28,7 @@ Django==4.2.22 elasticsearch-dsl>=7.0.0,<8.0.0 elasticsearch>=7.0.0,<8.0.0 html2text>=2016.9.19 -huey==2.5.1 +huey==2.5.3 jinja2==3.1.6 jsonschema==4.23.0 markdown==3.7 @@ -56,5 +56,5 @@ wagtail-modeladmin==2.1.0 invoke ipython pgcli -# replace with pypi install eventually (and transfer to OMF?) +# replace with pypi scicodes install eventually git+https://github.com/comses/codemeticulous.git diff --git a/e2e.yml b/e2e.yml index df77242fb..e5560cdd1 100644 --- a/e2e.yml +++ b/e2e.yml @@ -4,6 +4,9 @@ services: command: tail -f /dev/null healthcheck: disable: true + elasticsearch: + environment: + - discovery.type=single-node e2edb: image: postgis/postgis:16-3.5 secrets: @@ -21,9 +24,16 @@ services: POSTGRES_PASSWORD_FILE: /run/secrets/db_password vite: command: ["npm", "run", "build"] + ports: + - "127.0.0.1:5173:5173" environment: NODE_ENV: "e2e" + volumes: + - ./frontend:/code server: + build: + args: + RUN_SCRIPT: ./deploy/dev.sh depends_on: db: condition: service_started @@ -40,5 +50,7 @@ services: interval: 30s timeout: 10s retries: 5 + ports: + - "127.0.0.1:8000:8000" environment: DJANGO_SETTINGS_MODULE: "core.settings.e2e" diff --git a/test.yml b/test.yml new file mode 100644 index 000000000..71f8fe892 --- /dev/null +++ b/test.yml @@ -0,0 +1,15 @@ +services: + server: + build: + args: + UBUNTU_MIRROR: "${UBUNTU_MIRROR}" + REQUIREMENTS_FILE: "requirements-test.txt" + RUN_SCRIPT: "./deploy/test.sh" + image: comses/server:dev + volumes: + - ./django:/code + - ./docs:/docs + environment: + DJANGO_SETTINGS_MODULE: "core.settings.test" + ports: + - "127.0.0.1:8000:8000"