From 31b9bf289b2213ef0b6e748b0987f9487c7b09ff Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 16:59:23 +0600 Subject: [PATCH 001/171] Version Type Added --- readthedocs/builds/constants.py | 2 ++ .../0008_added_pull_request_version_type.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 readthedocs/builds/migrations/0008_added_pull_request_version_type.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index b0dc6cfba94..e2e4fc44ac3 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -33,11 +33,13 @@ BRANCH = 'branch' TAG = 'tag' +PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), + (PULL_REQUEST, _('Pull Request')), (UNKNOWN, _('Unknown')), ) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py new file mode 100644 index 00000000000..14d8968866d --- /dev/null +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-05-30 10:02 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('builds', '0007_add-automation-rules'), + ] + + operations = [ + migrations.AlterField( + model_name='version', + name='type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + ), + migrations.AlterField( + model_name='versionautomationrule', + name='version_type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + ), + ] From bd119944f3010a83713a3a7a80f3d49d9098e385 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 23:52:36 +0600 Subject: [PATCH 002/171] Version Model Methods Updated --- readthedocs/builds/models.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 1039b815326..ca44b39f252 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -40,6 +40,7 @@ BUILD_TYPES, LATEST, NON_REPOSITORY_VERSIONS, + PULL_REQUEST, STABLE, TAG, VERSION_TYPES, @@ -142,7 +143,8 @@ def vcs_url(self): """ Generate VCS (github, gitlab, bitbucket) URL for this version. - Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -152,12 +154,24 @@ def vcs_url(self): else: slug_url = self.slug - if ('github' in self.project.repo) or ('gitlab' in self.project.repo): - url = f'/tree/{slug_url}/' + if self.type == PULL_REQUEST: + if 'github' in self.project.repo: + url = f'/pull/{slug_url}/' - if 'bitbucket' in self.project.repo: - slug_url = self.identifier - url = f'/src/{slug_url}' + if 'gitlab' in self.project.repo: + slug_url = self.identifier + url = f'/merge_requests/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/pull-requests/{slug_url}' + else: + if ('github' in self.project.repo) or ('gitlab' in self.project.repo): + url = f'/tree/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/src/{slug_url}' # TODO: improve this replacing return self.project.repo.replace('git://', 'https://').replace('.git', '') + url @@ -220,7 +234,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - # If we came that far it's not a special version nor a branch or tag. + if self.type == PULL_REQUEST: + # If this version is a Pull Request, the identifier will + # contain the actual commit hash. which we can use to + # generate url for a given file name + return self.identifier + + # If we came that far it's not a special version + # nor a branch, tag or Pull Request. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', From 8f3e64b70712766aa1b85dcfef5b5a75d2f10aff Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 15:40:32 +0600 Subject: [PATCH 003/171] Internal and External Version Manager added --- readthedocs/builds/managers.py | 33 +++++++++++++++++++++++++++++++++ readthedocs/builds/models.py | 10 +++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 68cdec6efe9..991293beae2 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,6 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, + PULL_REQUEST, ) from .querysets import VersionQuerySet @@ -85,6 +86,38 @@ def get_object_or_log(self, **kwargs): log.warning('Version not found for given kwargs. %s' % kwargs) +class InternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes internal version. + + It will exclude PULL_REQUEST type from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + def get_queryset(self): + return super().get_queryset().exclude(type=PULL_REQUEST) + + +class ExternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes external version. + + It will only include PULL_REQUEST type Versions in the queries. + """ + def get_queryset(self): + return super().get_queryset().filter(type=PULL_REQUEST) + + class VersionManager(SettingsOverrideObject): _default_class = VersionManagerBase _override_setting = 'VERSION_MANAGER' + + +class InternalVersionManager(SettingsOverrideObject): + _default_class = InternalVersionManagerBase + _override_setting = 'INTERNAL_VERSION_MANAGER' + + +class ExternalVersionManager(SettingsOverrideObject): + _default_class = ExternalVersionManagerBase + _override_setting = 'EXTERNAL_VERSION_MANAGER' + diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index ca44b39f252..bca375570c9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -45,7 +45,11 @@ TAG, VERSION_TYPES, ) -from .managers import VersionManager +from .managers import ( + VersionManager, + InternalVersionManager, + ExternalVersionManager +) from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet from .utils import ( get_bitbucket_username_repo, @@ -112,6 +116,10 @@ class Version(models.Model): machine = models.BooleanField(_('Machine Created'), default=False) objects = VersionManager.from_queryset(VersionQuerySet)() + # Only include BRANCH, TAG, UNKONWN type Versions. + internal = InternalVersionManager.from_queryset(VersionQuerySet)() + # Only include PULL_REQUEST type Versions. + external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: unique_together = [('project', 'slug')] From 66200a0b1405f9ae301c8c581324d5420b204250 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 16:20:19 +0600 Subject: [PATCH 004/171] lint fix --- readthedocs/builds/managers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 991293beae2..2a8bb4840d9 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -87,22 +87,26 @@ def get_object_or_log(self, **kwargs): class InternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes internal version. It will exclude PULL_REQUEST type from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ + def get_queryset(self): return super().get_queryset().exclude(type=PULL_REQUEST) class ExternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes external version. It will only include PULL_REQUEST type Versions in the queries. """ + def get_queryset(self): return super().get_queryset().filter(type=PULL_REQUEST) @@ -120,4 +124,3 @@ class InternalVersionManager(SettingsOverrideObject): class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase _override_setting = 'EXTERNAL_VERSION_MANAGER' - From 34e4134d17622291605d50118c5967f64eaf0177 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 5 Jun 2019 01:18:20 +0600 Subject: [PATCH 005/171] All Version Querysets Updated with InternalVersionManager --- readthedocs/api/v2/views/model_views.py | 2 +- readthedocs/api/v3/views.py | 2 +- readthedocs/builds/models.py | 4 +- readthedocs/builds/views.py | 2 +- .../core/management/commands/update_repos.py | 42 +++++++++++++++++++ .../management/commands/update_versions.py | 2 +- readthedocs/core/views/serve.py | 7 ++-- readthedocs/projects/admin.py | 4 +- readthedocs/projects/forms.py | 2 +- readthedocs/projects/models.py | 8 ++-- readthedocs/projects/views/public.py | 6 +-- 11 files changed, 63 insertions(+), 18 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index af41f28ab84..5f594a032b8 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions.filter(active=True) + versions = project.versions(manager='internal').filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 0e8b2b55a0e..57b86360e96 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -230,7 +230,7 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, lookup_value_regex = r'[^/]+' filterset_class = VersionFilter - queryset = Version.objects.all() + queryset = Version.internal.all() permit_list_expands = [ 'last_build', 'last_build.config', diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bca375570c9..bcfc5dfff5c 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -142,7 +142,9 @@ def __str__(self): @property def ref(self): if self.slug == STABLE: - stable = determine_stable_version(self.project.versions.all()) + stable = determine_stable_version( + self.project.versions(manager='internal').all() + ) if stable: return stable.slug diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index e8e3d458e2c..3d6e2cc0ca1 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context['project'] = self.project context['active_builds'] = active_builds - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=self.project, ) diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 5852592d360..0e14f9ce339 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -76,6 +76,48 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) + elif version == 'internal': + log.info('Updating all internal versions for %s', slug) + for version in Version.internal.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) + elif version == 'external': + log.info('Updating all external versions for %s', slug) + for version in Version.external.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) else: p = Project.all_objects.get(slug=slug) log.info('Building %s', p) diff --git a/readthedocs/core/management/commands/update_versions.py b/readthedocs/core/management/commands/update_versions.py index b0aa1f877cb..6456accc430 100644 --- a/readthedocs/core/management/commands/update_versions.py +++ b/readthedocs/core/management/commands/update_versions.py @@ -13,7 +13,7 @@ class Command(BaseCommand): help = __doc__ def handle(self, *args, **options): - for version in Version.objects.filter(active=True, built=False): + for version in Version.internal.filter(active=True, built=False): # pylint: disable=no-value-for-parameter update_docs_task( version.pk, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 390063f256a..61ecc9f504a 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -383,7 +383,7 @@ def changefreqs_generator(): raise Http404 sorted_versions = sort_version_aware( - Version.objects.public( + Version.internal.public( project=project, only_active=True, ), @@ -409,8 +409,9 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): - translation_versions = translation.versions.public( - ).values_list('slug', flat=True) + translation_versions = translation.versions( + manager='internal' + ).public().values_list('slug', flat=True) if version.slug in translation_versions: href = project.get_docs_url( version_slug=version.slug, diff --git a/readthedocs/projects/admin.py b/readthedocs/projects/admin.py index d74e8d99075..fb4aef4772d 100644 --- a/readthedocs/projects/admin.py +++ b/readthedocs/projects/admin.py @@ -239,7 +239,7 @@ def reindex_active_versions(self, request, queryset): """Reindex all active versions of the selected projects to ES.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) active_versions = version_qs.filter(active=True) if not active_versions.exists(): @@ -271,7 +271,7 @@ def wipe_all_versions(self, request, queryset): """Wipe indexes of all versions of selected projects.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) if not version_qs.exists(): self.message_user( request, diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index a701a49c020..65c589c8b18 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -240,7 +240,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions.filter( + versions_choices = self.instance.versions(manager='internal').filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 19f58f7fab0..97f9b90c8d4 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -908,7 +908,7 @@ def api_versions(self): def active_versions(self): from readthedocs.builds.models import Version - versions = Version.objects.public(project=self, only_active=True) + versions = Version.internal.public(project=self, only_active=True) return ( versions.filter(built=True, active=True) | versions.filter(active=True, uploaded=True) @@ -922,7 +922,7 @@ def ordered_active_versions(self, user=None): } if user: kwargs['user'] = user - versions = Version.objects.public(**kwargs).select_related( + versions = Version.internal.public(**kwargs).select_related( 'project', 'project__main_language_project', ).prefetch_related( @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions.filter(active=True) + return self.versions(manager='internal').filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions.all() + versions = self.versions(manager='internal').all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index 64d82272738..ea15bfb168b 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) project = self.get_object() - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=project, ) @@ -179,7 +179,7 @@ def project_downloads(request, project_slug): Project.objects.protected(request.user), slug=project_slug, ) - versions = Version.objects.public(user=request.user, project=project) + versions = Version.internal.public(user=request.user, project=project) versions = sort_version_aware(versions) version_data = OrderedDict() for version in versions: @@ -268,7 +268,7 @@ def project_versions(request, project_slug): slug=project_slug, ) - versions = Version.objects.public( + versions = Version.internal.public( user=request.user, project=project, only_active=False, From 3a3cfdf4f6fed6736453d7368e9961c5cb9da774 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 02:15:34 +0600 Subject: [PATCH 006/171] Manager names moved to Constants --- readthedocs/api/v2/views/model_views.py | 4 ++-- readthedocs/builds/constants.py | 7 +++++++ readthedocs/builds/models.py | 5 +++-- readthedocs/core/management/commands/update_repos.py | 5 +++-- readthedocs/core/views/serve.py | 3 ++- readthedocs/projects/forms.py | 3 ++- readthedocs/projects/models.py | 6 +++--- 7 files changed, 22 insertions(+), 11 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index 5f594a032b8..fca91b32626 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG +from readthedocs.builds.constants import BRANCH, TAG, INTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions(manager='internal').filter(active=True) + versions = project.versions(manager=INTERNAL).filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e2e4fc44ac3..e200fc5fe25 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -55,3 +55,10 @@ LATEST, STABLE, ) + +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding PULL_REQUEST Type. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only PULL_REQUEST Type Versions and Builds. +EXTERNAL = 'external' diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bcfc5dfff5c..d0c5f6a854e 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -32,12 +32,13 @@ from readthedocs.projects.models import APIProject, Project from readthedocs.projects.version_handling import determine_stable_version -from .constants import ( +from readthedocs.builds.constants import ( BRANCH, BUILD_STATE, BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, BUILD_TYPES, + INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, PULL_REQUEST, @@ -143,7 +144,7 @@ def __str__(self): def ref(self): if self.slug == STABLE: stable = determine_stable_version( - self.project.versions(manager='internal').all() + self.project.versions(manager=INTERNAL).all() ) if stable: return stable.slug diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 0e14f9ce339..b15ea8c3c9c 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -10,6 +10,7 @@ from django.core.management.base import BaseCommand +from readthedocs.builds.constants import EXTERNAL, INTERNAL from readthedocs.builds.models import Build, Version from readthedocs.core.utils import trigger_build from readthedocs.projects import tasks @@ -76,7 +77,7 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) - elif version == 'internal': + elif version == INTERNAL: log.info('Updating all internal versions for %s', slug) for version in Version.internal.filter( project__slug=slug, @@ -97,7 +98,7 @@ def handle(self, *args, **options): build_pk=build.pk, version_pk=version.pk, ) - elif version == 'external': + elif version == EXTERNAL: log.info('Updating all external versions for %s', slug) for version in Version.external.filter( project__slug=slug, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 61ecc9f504a..c6374d76cd6 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,6 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve +from readthedocs.builds.constants import INTERNAL from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path @@ -410,7 +411,7 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): translation_versions = translation.versions( - manager='internal' + manager=INTERNAL ).public().values_list('slug', flat=True) if version.slug in translation_versions: href = project.get_docs_url( diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index 65c589c8b18..3c86cc39e12 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -14,6 +14,7 @@ from guardian.shortcuts import assign from textclassifier.validators import ClassifierValidator +from readthedocs.builds.constants import INTERNAL from readthedocs.core.utils import slugify, trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.integrations.models import Integration @@ -240,7 +241,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions(manager='internal').filter( + versions_choices = self.instance.versions(manager=INTERNAL).filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 97f9b90c8d4..e5d6c6c442d 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -20,7 +20,7 @@ from taggit.managers import TaggableManager from readthedocs.api.v2.client import api -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, INTERNAL from readthedocs.core.resolver import resolve, resolve_domain from readthedocs.core.utils import broadcast, slugify from readthedocs.projects import constants @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions(manager='internal').filter(active=True) + return self.versions(manager=INTERNAL).filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions(manager='internal').all() + versions = self.versions(manager=INTERNAL).all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() From 3d3de29bd0b26295215ac3c0d17b8454f2dc02b5 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:25:03 +0600 Subject: [PATCH 007/171] Tests added --- readthedocs/builds/constants.py | 2 +- .../rtd_tests/tests/test_doc_serving.py | 23 +++- readthedocs/rtd_tests/tests/test_managers.py | 100 ++++++++++++++++++ readthedocs/rtd_tests/tests/test_project.py | 28 ++++- .../rtd_tests/tests/test_project_forms.py | 25 ++++- .../rtd_tests/tests/test_project_views.py | 30 +++++- readthedocs/rtd_tests/tests/test_version.py | 68 ++++++++++++ 7 files changed, 269 insertions(+), 7 deletions(-) create mode 100644 readthedocs/rtd_tests/tests/test_managers.py create mode 100644 readthedocs/rtd_tests/tests/test_version.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e200fc5fe25..37474eb5365 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -58,7 +58,7 @@ # Manager name for Internal Versions or Builds. # ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' +INTERNAL = 'internal' # Manager name for External Versions or Builds. # ie: Only PULL_REQUEST Type Versions and Builds. EXTERNAL = 'external' diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 134b191235c..ad7a56753fe 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -11,6 +11,7 @@ from django.urls import reverse from mock import mock_open, patch +from readthedocs.builds.constants import PULL_REQUEST, INTERNAL from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -240,6 +241,16 @@ def test_sitemap_xml(self): project=self.public, active=True ) + # This is a Pull Request Version + pr_version = fixture.get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.public, + active=True, + type=PULL_REQUEST + ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( Project, @@ -259,7 +270,7 @@ def test_sitemap_xml(self): ) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/xml') - for version in self.public.versions.filter(privacy_level=constants.PUBLIC): + for version in self.public.versions(manager=INTERNAL).filter(privacy_level=constants.PUBLIC): self.assertContains( response, self.public.get_docs_url( @@ -293,3 +304,13 @@ def test_sitemap_xml(self): # hreflang should use hyphen instead of underscore # in language and country value. (zh_CN should be zh-CN) self.assertContains(response, 'zh-CN') + + # PR Versions should not be in the sitemap_xml. + self.assertNotContains( + response, + self.public.get_docs_url( + version_slug=pr_version.slug, + lang_slug=self.public.language, + private=True, + ), + ) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py new file mode 100644 index 00000000000..8e440e18b85 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -0,0 +1,100 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED +from readthedocs.projects.models import Project + + +class TestVersionManagerBase(TestCase): + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.public_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC + ) + self.private_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PRIVATE + ) + self.protected_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PROTECTED + ) + self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + + +class TestInternalVersionManager(TestVersionManagerBase): + + """ + Queries using Internal Manager should only include Internal Versions. + + It will exclude PULL_REQUEST type Versions from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + + def test_internal_version_manager_with_all(self): + self.assertNotIn(self.public_pr_version, Version.internal.all()) + + def test_internal_version_manager_with_public(self): + self.assertNotIn(self.public_pr_version, Version.internal.public()) + + def test_internal_version_manager_with_protected(self): + self.assertNotIn(self.protected_pr_version, Version.internal.protected()) + + def test_internal_version_manager_with_private(self): + self.assertNotIn(self.private_pr_version, Version.internal.private()) + + def test_internal_version_manager_with_api(self): + self.assertNotIn(self.public_pr_version, Version.internal.api()) + + def test_internal_version_manager_with_for_project(self): + self.assertNotIn(self.public_pr_version, Version.internal.for_project(self.pip)) + + +class TestExternalVersionManager(TestVersionManagerBase): + + """ + Queries using External Manager should only include External Versions. + + It will only include PULL_REQUEST type Versions in the queries. + """ + + def test_external_version_manager_with_all(self): + self.assertNotIn(self.internal_versions, Version.external.all()) + self.assertIn(self.public_pr_version, Version.external.all()) + + def test_external_version_manager_with_public(self): + self.assertNotIn(self.internal_versions, Version.external.public()) + self.assertIn(self.public_pr_version, Version.external.public()) + + def test_external_version_manager_with_protected(self): + self.assertNotIn(self.internal_versions, Version.external.protected()) + self.assertIn(self.protected_pr_version, Version.external.protected()) + + def test_external_version_manager_with_private(self): + self.assertNotIn(self.internal_versions, Version.external.private()) + self.assertIn(self.private_pr_version, Version.external.private()) + + def test_external_version_manager_with_api(self): + self.assertNotIn(self.internal_versions, Version.external.api()) + self.assertIn(self.public_pr_version, Version.external.api()) + + def test_external_version_manager_with_for_project(self): + self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) + self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 40f86350ed4..c2db080f201 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,8 +14,9 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, + PULL_REQUEST, ) -from readthedocs.builds.models import Build +from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project from readthedocs.projects.tasks import finish_inactive_builds @@ -29,6 +30,16 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) class TestProject(ProjectMixin, TestCase): @@ -134,6 +145,21 @@ def test_get_storage_path(self): 'htmlzip/pip/latest/pip.zip', ) + def test_ordered_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.ordered_active_versions()) + + def test_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.active_versions()) + + def test_all_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.all_active_versions()) + + def test_update_stable_version_excludes_pr_versions(self): + # Delete all versions excluding PR Versions. + self.pip.versions.exclude(type=PULL_REQUEST).delete() + # Test that PR Version is not considered for stable. + self.assertEqual(self.pip.update_stable_version(), None) + class TestProjectTranslations(ProjectMixin, TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index 50d9815d051..b2e8687208a 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -314,7 +314,7 @@ def setUp(self): verbose_name='protected', ) - def test_list_only_non_auto_generated_versions_on_default_branch(self): + def test_list_only_non_auto_generated_versions_in_default_branch_choices(self): form = ProjectAdvancedForm(instance=self.project) # This version is created automatically by the project on save latest = self.project.versions.filter(slug=LATEST) @@ -336,7 +336,7 @@ def test_list_only_non_auto_generated_versions_on_default_branch(self): 'default_branch'].widget.choices], ) - def test_list_user_created_latest_and_stable_versions_on_default_branch(self): + def test_list_user_created_latest_and_stable_versions_in_default_branch_choices(self): self.project.versions.filter(slug=LATEST).first().delete() user_created_latest_version = get( Version, @@ -383,6 +383,25 @@ def test_commit_name_not_in_default_branch_choices(self): 'default_branch'].widget.choices], ) + def test_pr_version_not_in_default_branch_choices(self): + pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.project, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC, + ) + form = ProjectAdvancedForm(instance=self.project) + + self.assertNotIn( + pr_version.verbose_name, + [identifier for identifier, _ in form.fields[ + 'default_branch'].widget.choices], + ) + class TestTranslationForms(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 7c5b286bdd4..90578c8dceb 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST +from readthedocs.builds.constants import LATEST, PULL_REQUEST from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -370,12 +370,40 @@ def test_import_demo_imported_duplicate(self): class TestPublicViews(MockBuildTestCase): def setUp(self): self.pip = get(Project, slug='pip') + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) def test_project_download_media(self): url = reverse('project_download_media', args=[self.pip.slug, 'pdf', LATEST]) response = self.client.get(url) self.assertEqual(response.status_code, 302) + def test_project_detail_view_only_shows_internal_versons(self): + url = reverse('projects_detail', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_downloads_only_shows_internal_versons(self): + url = reverse('project_downloads', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_versions_only_shows_internal_versons(self): + url = reverse('project_version_list', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['active_versions']) + self.assertNotIn(self.pr_version, response.context['inactive_versions']) + class TestPrivateViews(MockBuildTestCase): def setUp(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py new file mode 100644 index 00000000000..8a6f5bf3db8 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -0,0 +1,68 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.models import Project + + +class VersionMixin: + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='9F86D081884C7D659A2FEAA0C55AD015A', + verbose_name='pr-version', + slug='9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) + self.branch_version = get( + Version, + identifier='origin/stable', + verbose_name='stable', + slug='stable', + project=self.pip, + active=True, + type=BRANCH + ) + self.tag_version = get( + Version, + identifier='origin/master', + verbose_name='latest', + slug='latest', + project=self.pip, + active=True, + type=TAG + ) + + +class TestVersionModel(VersionMixin, TestCase): + + def test_vcs_url_for_pr_version(self): + expected_url = f'https://github.com/pypa/pip/pull/{self.pr_version.slug}/' + self.assertEqual(self.pr_version.vcs_url, expected_url) + + def test_vcs_url_for_latest_version(self): + slug = self.pip.default_branch or self.pip.vcs_repo().fallback_branch + expected_url = f'https://github.com/pypa/pip/tree/{slug}/' + self.assertEqual(self.tag_version.vcs_url, expected_url) + + def test_vcs_url_for_stable_version(self): + expected_url = f'https://github.com/pypa/pip/tree/{self.branch_version.ref}/' + self.assertEqual(self.branch_version.vcs_url, expected_url) + + def test_commit_name_for_stable_version(self): + self.assertEqual(self.branch_version.commit_name, 'stable') + + def test_commit_name_for_latest_version(self): + self.assertEqual(self.tag_version.commit_name, 'master') + + def test_commit_name_for_pr_version(self): + self.assertEqual(self.pr_version.commit_name, self.pr_version.identifier) From b55983f3467061e8ef09eca46170ccc35dcac5b2 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:52:31 +0600 Subject: [PATCH 008/171] Version Update, Delete Html and wipe updated to exclude PR Versions --- readthedocs/core/views/__init__.py | 2 +- readthedocs/projects/views/private.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 6b438866ae5..de519a13759 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -73,7 +73,7 @@ def random_page(request, project_slug=None): # pylint: disable=unused-argument def wipe_version(request, project_slug, version_slug): version = get_object_or_404( - Version, + Version.internal.all(), project__slug=project_slug, slug=version_slug, ) diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index 46e83d96188..044a83641e6 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -157,7 +157,7 @@ def project_version_detail(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, @@ -682,7 +682,7 @@ def project_version_delete_html(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, From 61872721a793861235aef8dc283cb676b959a737 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 23:03:09 +0600 Subject: [PATCH 009/171] Updated migrations --- .../builds/migrations/0008_added_pull_request_version_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 14d8968866d..75a126d37b6 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-05-30 10:02 +# Generated by Django 1.11.20 on 2019-06-06 16:51 from __future__ import unicode_literals from django.db import migrations, models From 386ca07ba20f134d9eef9ef969a4c91796c7ae42 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sun, 9 Jun 2019 02:17:05 +0600 Subject: [PATCH 010/171] footer API updated --- readthedocs/api/v2/views/footer_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/api/v2/views/footer_views.py b/readthedocs/api/v2/views/footer_views.py index 09709ac1bb1..ece5e09bd89 100644 --- a/readthedocs/api/v2/views/footer_views.py +++ b/readthedocs/api/v2/views/footer_views.py @@ -9,7 +9,7 @@ from rest_framework_jsonp.renderers import JSONPRenderer from readthedocs.api.v2.signals import footer_response -from readthedocs.builds.constants import LATEST, TAG +from readthedocs.builds.constants import LATEST, TAG, INTERNAL from readthedocs.builds.models import Version from readthedocs.projects.models import Project from readthedocs.projects.version_handling import ( @@ -25,7 +25,7 @@ def get_version_compare_data(project, base_version=None): :param base_version: We assert whether or not the base_version is also the highest version in the resulting "is_highest" value. """ - versions_qs = project.versions.public().filter(active=True) + versions_qs = project.versions(manager=INTERNAL).public().filter(active=True) # Take preferences over tags only if the project has at least one tag if versions_qs.filter(type=TAG).exists(): From 29348ef08c14d472aaeace74005e1e45c6f7d6d1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 00:45:00 +0600 Subject: [PATCH 011/171] manager tests updated --- readthedocs/rtd_tests/tests/test_managers.py | 34 +++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 8e440e18b85..e69abdc4dd8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -1,3 +1,4 @@ +from django.contrib.auth import get_user_model from django.test import TestCase from django_dynamic_fixture import get @@ -7,12 +8,16 @@ from readthedocs.projects.models import Project +User = get_user_model() + + class TestVersionManagerBase(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ['test_data'] def setUp(self): - self.client.login(username='eric', password='test') + self.user = User.objects.create(username='test_user', password='test') + self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') # Create a External Version. ie: PULL_REQUEST type Version. self.public_pr_version = get( @@ -54,6 +59,12 @@ def test_internal_version_manager_with_all(self): def test_internal_version_manager_with_public(self): self.assertNotIn(self.public_pr_version, Version.internal.public()) + def test_internal_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.public_pr_version, + Version.internal.public(self.user, self.pip) + ) + def test_internal_version_manager_with_protected(self): self.assertNotIn(self.protected_pr_version, Version.internal.protected()) @@ -83,6 +94,17 @@ def test_external_version_manager_with_public(self): self.assertNotIn(self.internal_versions, Version.external.public()) self.assertIn(self.public_pr_version, Version.external.public()) + def test_external_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.internal_versions, + Version.external.public(self.user, self.pip) + ) + self.assertIn( + self.public_pr_version, + Version.external.public(self.user, self.pip) + ) + + def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) @@ -96,5 +118,9 @@ def test_external_version_manager_with_api(self): self.assertIn(self.public_pr_version, Version.external.api()) def test_external_version_manager_with_for_project(self): - self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) - self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) + self.assertNotIn( + self.internal_versions, Version.external.for_project(self.pip) + ) + self.assertIn( + self.public_pr_version, Version.external.for_project(self.pip) + ) From 11a0583abf4b11a394fa169fc7a14f669bdea0aa Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 18:56:06 +0600 Subject: [PATCH 012/171] _override_setting removed from new managers --- readthedocs/builds/managers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 2a8bb4840d9..0bc30f02807 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -118,9 +118,7 @@ class VersionManager(SettingsOverrideObject): class InternalVersionManager(SettingsOverrideObject): _default_class = InternalVersionManagerBase - _override_setting = 'INTERNAL_VERSION_MANAGER' class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase - _override_setting = 'EXTERNAL_VERSION_MANAGER' From c100c81beba9a4f9d596a216b0a36b849bfe1fad Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:11:51 +0600 Subject: [PATCH 013/171] BuildTriggerMixin updated --- readthedocs/builds/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index 3d6e2cc0ca1..3333495f6c8 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -57,7 +57,7 @@ def post(self, request, project_slug): version_slug = request.POST.get('version_slug') version = get_object_or_404( - Version, + Version.internal.all(), project=project, slug=version_slug, ) From c1c03bb951d8cd0baf4b19e6acb47e599a183da7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:25:28 +0600 Subject: [PATCH 014/171] More Update --- readthedocs/search/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 5f04f6736b7..cab67ba5ff8 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,7 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.objects.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) if version.exists(): project_list.append(version.first().project) return project_list From 5ea5f6755c4860298a9018bebf9ed857fdd6cb54 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:55:32 +0600 Subject: [PATCH 015/171] lint fix --- readthedocs/rtd_tests/tests/test_managers.py | 1 - readthedocs/search/utils.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index e69abdc4dd8..f830c9af549 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -104,7 +104,6 @@ def test_external_version_manager_with_public_with_user_and_project(self): Version.external.public(self.user, self.pip) ) - def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index cab67ba5ff8..3433195bb55 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,9 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter( + project__slug=project.slug, slug=version_slug + ) if version.exists(): project_list.append(version.first().project) return project_list From 56493fa016f39224ba4ef3b5a2322b853aeb0261 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 17:35:34 +0600 Subject: [PATCH 016/171] removed unused import --- readthedocs/core/views/serve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index ae60011e54e..3daebf8c721 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,7 +38,6 @@ from django.views.decorators.cache import cache_page from django.views.static import serve -from readthedocs.builds.constants import INTERNAL from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path From 84a69753606cb04185e4c9aabed29559e9061254 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:30:26 +0600 Subject: [PATCH 017/171] External version name added everywhere --- readthedocs/builds/constants.py | 17 ++++++++--------- readthedocs/builds/managers.py | 10 +++++----- .../0008_added_pull_request_version_type.py | 6 +++--- readthedocs/builds/models.py | 14 +++++++------- readthedocs/rtd_tests/tests/test_doc_serving.py | 6 +++--- readthedocs/rtd_tests/tests/test_managers.py | 16 ++++++++-------- readthedocs/rtd_tests/tests/test_project.py | 8 ++++---- .../rtd_tests/tests/test_project_forms.py | 4 ++-- .../rtd_tests/tests/test_project_views.py | 4 ++-- readthedocs/rtd_tests/tests/test_version.py | 6 +++--- 10 files changed, 45 insertions(+), 46 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 37474eb5365..bdc9a70fe79 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -31,15 +31,21 @@ ('dash', _('Dash')), ) +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding pull request/merge request Versions and Builds. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only pull request/merge request Versions and Builds. +EXTERNAL = 'external' + BRANCH = 'branch' TAG = 'tag' -PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), - (PULL_REQUEST, _('Pull Request')), + (EXTERNAL, _('External')), (UNKNOWN, _('Unknown')), ) @@ -55,10 +61,3 @@ LATEST, STABLE, ) - -# Manager name for Internal Versions or Builds. -# ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' -# Manager name for External Versions or Builds. -# ie: Only PULL_REQUEST Type Versions and Builds. -EXTERNAL = 'external' diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 0bc30f02807..06262ed3e24 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,7 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, - PULL_REQUEST, + EXTERNAL, ) from .querysets import VersionQuerySet @@ -91,12 +91,12 @@ class InternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes internal version. - It will exclude PULL_REQUEST type from the queries + It will exclude pull request/merge request versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ def get_queryset(self): - return super().get_queryset().exclude(type=PULL_REQUEST) + return super().get_queryset().exclude(type=EXTERNAL) class ExternalVersionManagerBase(VersionManagerBase): @@ -104,11 +104,11 @@ class ExternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes external version. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull request/merge request Versions in the queries. """ def get_queryset(self): - return super().get_queryset().filter(type=PULL_REQUEST) + return super().get_queryset().filter(type=EXTERNAL) class VersionManager(SettingsOverrideObject): diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 75a126d37b6..2bf9dc62d66 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-06-06 16:51 +# Generated by Django 1.11.21 on 2019-06-17 19:26 from __future__ import unicode_literals from django.db import migrations, models @@ -15,11 +15,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='version', name='type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), ), migrations.AlterField( model_name='versionautomationrule', name='version_type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), ), ] diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index d0c5f6a854e..34a3fd372a9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -41,7 +41,7 @@ INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, - PULL_REQUEST, + EXTERNAL, STABLE, TAG, VERSION_TYPES, @@ -119,7 +119,7 @@ class Version(models.Model): objects = VersionManager.from_queryset(VersionQuerySet)() # Only include BRANCH, TAG, UNKONWN type Versions. internal = InternalVersionManager.from_queryset(VersionQuerySet)() - # Only include PULL_REQUEST type Versions. + # Only include EXTERNAL type Versions. external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: @@ -155,7 +155,7 @@ def vcs_url(self): Generate VCS (github, gitlab, bitbucket) URL for this version. Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. - Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. + Pull/merge Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -165,7 +165,7 @@ def vcs_url(self): else: slug_url = self.slug - if self.type == PULL_REQUEST: + if self.type == EXTERNAL: if 'github' in self.project.repo: url = f'/pull/{slug_url}/' @@ -245,14 +245,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - if self.type == PULL_REQUEST: - # If this version is a Pull Request, the identifier will + if self.type == EXTERNAL: + # If this version is a EXTERNAL version, the identifier will # contain the actual commit hash. which we can use to # generate url for a given file name return self.identifier # If we came that far it's not a special version - # nor a branch, tag or Pull Request. + # nor a branch, tag or EXTERNAL version. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 8a5a6609b6b..9de5afccf73 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -10,7 +10,7 @@ from django.urls import reverse from mock import mock_open, patch -from readthedocs.builds.constants import LATEST, PULL_REQUEST, INTERNAL +from readthedocs.builds.constants import LATEST, EXTERNAL, INTERNAL from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -240,7 +240,7 @@ def test_sitemap_xml(self): project=self.public, active=True ) - # This is a Pull Request Version + # This is a EXTERNAL Version pr_version = fixture.get( Version, identifier='pr-version', @@ -248,7 +248,7 @@ def test_sitemap_xml(self): slug='pr-9999', project=self.public, active=True, - type=PULL_REQUEST + type=EXTERNAL ) stable_version = fixture.get( Version, diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index f830c9af549..924544a8be8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -2,7 +2,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED from readthedocs.projects.models import Project @@ -19,29 +19,29 @@ def setUp(self): self.user = User.objects.create(username='test_user', password='test') self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.public_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC ) self.private_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PRIVATE ) self.protected_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PROTECTED ) - self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + self.internal_versions = Version.objects.exclude(type=EXTERNAL) class TestInternalVersionManager(TestVersionManagerBase): @@ -49,7 +49,7 @@ class TestInternalVersionManager(TestVersionManagerBase): """ Queries using Internal Manager should only include Internal Versions. - It will exclude PULL_REQUEST type Versions from the queries + It will exclude EXTERNAL type Versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ @@ -83,7 +83,7 @@ class TestExternalVersionManager(TestVersionManagerBase): """ Queries using External Manager should only include External Versions. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull/merge request Version in the queries. """ def test_external_version_manager_with_all(self): diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index c2db080f201..5d5194d04d2 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,7 +14,7 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, - PULL_REQUEST, + EXTERNAL, ) from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError @@ -30,7 +30,7 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='pr-version', @@ -38,7 +38,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) @@ -156,7 +156,7 @@ def test_all_active_versions_excludes_pr_versions(self): def test_update_stable_version_excludes_pr_versions(self): # Delete all versions excluding PR Versions. - self.pip.versions.exclude(type=PULL_REQUEST).delete() + self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index b2e8687208a..ba5b3c88b9d 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST +from readthedocs.builds.constants import LATEST, STABLE, EXTERNAL from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -391,7 +391,7 @@ def test_pr_version_not_in_default_branch_choices(self): slug='pr-9999', project=self.project, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC, ) form = ProjectAdvancedForm(instance=self.project) diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 90578c8dceb..3bc948aff9b 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST, PULL_REQUEST +from readthedocs.builds.constants import LATEST, EXTERNAL from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -377,7 +377,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) def test_project_download_media(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py index 8a6f5bf3db8..e414ba79cc6 100644 --- a/readthedocs/rtd_tests/tests/test_version.py +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -1,7 +1,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.models import Project @@ -13,7 +13,7 @@ class VersionMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='9F86D081884C7D659A2FEAA0C55AD015A', @@ -21,7 +21,7 @@ def setUp(self): slug='9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) self.branch_version = get( Version, From 05951cfc99ddee655969e8482d5d666b690b5cdf Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:43:48 +0600 Subject: [PATCH 018/171] Migration name changed --- ...uest_version_type.py => 0008_added_external_version_type.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename readthedocs/builds/migrations/{0008_added_pull_request_version_type.py => 0008_added_external_version_type.py} (94%) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_external_version_type.py similarity index 94% rename from readthedocs/builds/migrations/0008_added_pull_request_version_type.py rename to readthedocs/builds/migrations/0008_added_external_version_type.py index 2bf9dc62d66..8de9c4f504f 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_external_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.21 on 2019-06-17 19:26 +# Generated by Django 1.11.21 on 2019-06-17 19:43 from __future__ import unicode_literals from django.db import migrations, models From e192300e11da4b930ceeb4df10ae8d2a1682f5d7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 23 May 2019 18:15:08 +0600 Subject: [PATCH 019/171] Sitemap sort order priorities updated --- readthedocs/core/views/serve.py | 4 +-- readthedocs/projects/version_handling.py | 10 +++--- .../rtd_tests/tests/test_doc_serving.py | 33 ++++++++++++++++++- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index cd5ab8fdb64..b7687d0a132 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -373,14 +373,14 @@ def changefreqs_generator(): """ Generator returning ``changefreq`` needed by sitemap.xml. - It returns ``daily`` on first iteration, then ``weekly`` and then it + It returns ``weekly`` on first iteration, then ``daily`` and then it will return always ``monthly``. We are using ``monthly`` as last value because ``never`` is too aggressive. If the tag is removed and a branch is created with the same name, we will want bots to revisit this. """ - changefreqs = ['daily', 'weekly'] + changefreqs = ['weekly', 'daily'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) if project.privacy_level == constants.PRIVATE: diff --git a/readthedocs/projects/version_handling.py b/readthedocs/projects/version_handling.py index 7a730e61fd0..093bb705c68 100644 --- a/readthedocs/projects/version_handling.py +++ b/readthedocs/projects/version_handling.py @@ -43,23 +43,23 @@ def comparable_version(version_string): """ Can be used as ``key`` argument to ``sorted``. - The ``LATEST`` version shall always beat other versions in comparison. - ``STABLE`` should be listed second. If we cannot figure out the version + The ``STABLE`` version shall always beat other versions in comparison. + ``LATEST`` should be listed second. If we cannot figure out the version number then we sort it to the bottom of the list. :param version_string: version as string object (e.g. '3.10.1' or 'latest') :type version_string: str or unicode - :returns: a comparable version object (e.g. 'latest' -> Version('99999.0')) + :returns: a comparable version object (e.g. 'latest' -> Version('9999.0')) :rtype: packaging.version.Version """ comparable = parse_version_failsafe(version_string) if not comparable: if version_string == LATEST_VERBOSE_NAME: - comparable = Version('99999.0') - elif version_string == STABLE_VERBOSE_NAME: comparable = Version('9999.0') + elif version_string == STABLE_VERBOSE_NAME: + comparable = Version('99999.0') else: comparable = Version('0.01') return comparable diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 4f6cb93990c..fed119bbab5 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -240,6 +240,15 @@ def test_sitemap_xml(self): project=self.public, active=True ) + stable_version = fixture.get( + Version, + identifier='stable', + verbose_name='stable', + slug='stable', + privacy_level=constants.PUBLIC, + project=self.public, + active=True + ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( Project, @@ -269,7 +278,7 @@ def test_sitemap_xml(self): ), ) - # stable is marked as PRIVATE and should not appear here + # PRIVATE version should not appear here self.assertNotContains( response, self.public.get_docs_url( @@ -294,6 +303,28 @@ def test_sitemap_xml(self): # in language and country value. (zh_CN should be zh-CN) self.assertContains(response, 'zh-CN') + # Check if STABLE version has 'priority of 1 and changefreq of weekly. + self.assertEqual( + response.context['versions'][0]['loc'], + self.public.get_docs_url( + version_slug=stable_version.slug, + lang_slug=self.public.language, + private=False, + ),) + self.assertEqual(response.context['versions'][0]['priority'], 1) + self.assertEqual(response.context['versions'][0]['changefreq'], 'weekly') + + # Check if LATEST version has priority of 0.9 and changefreq of daily. + self.assertEqual( + response.context['versions'][1]['loc'], + self.public.get_docs_url( + version_slug='latest', + lang_slug=self.public.language, + private=False, + ),) + self.assertEqual(response.context['versions'][1]['priority'], 0.9) + self.assertEqual(response.context['versions'][1]['changefreq'], 'daily') + @override_settings( PYTHON_MEDIA=True, USE_SUBDOMAIN=False, From d5ec837aa3ea9147ab19a538d83ff219087975b1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 25 May 2019 17:33:14 +0600 Subject: [PATCH 020/171] ordering fix --- readthedocs/core/views/serve.py | 4 ++-- readthedocs/projects/version_handling.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index b7687d0a132..f7ab665bbc5 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -355,7 +355,7 @@ def priorities_generator(): It generates values from 1 to 0.1 by decreasing in 0.1 on each iteration. After 0.1 is reached, it will keep returning 0.1. """ - priorities = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] + priorities = [0.9, 1, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] yield from itertools.chain(priorities, itertools.repeat(0.1)) def hreflang_formatter(lang): @@ -380,7 +380,7 @@ def changefreqs_generator(): aggressive. If the tag is removed and a branch is created with the same name, we will want bots to revisit this. """ - changefreqs = ['weekly', 'daily'] + changefreqs = ['daily', 'weekly'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) if project.privacy_level == constants.PRIVATE: diff --git a/readthedocs/projects/version_handling.py b/readthedocs/projects/version_handling.py index 093bb705c68..7a730e61fd0 100644 --- a/readthedocs/projects/version_handling.py +++ b/readthedocs/projects/version_handling.py @@ -43,23 +43,23 @@ def comparable_version(version_string): """ Can be used as ``key`` argument to ``sorted``. - The ``STABLE`` version shall always beat other versions in comparison. - ``LATEST`` should be listed second. If we cannot figure out the version + The ``LATEST`` version shall always beat other versions in comparison. + ``STABLE`` should be listed second. If we cannot figure out the version number then we sort it to the bottom of the list. :param version_string: version as string object (e.g. '3.10.1' or 'latest') :type version_string: str or unicode - :returns: a comparable version object (e.g. 'latest' -> Version('9999.0')) + :returns: a comparable version object (e.g. 'latest' -> Version('99999.0')) :rtype: packaging.version.Version """ comparable = parse_version_failsafe(version_string) if not comparable: if version_string == LATEST_VERBOSE_NAME: - comparable = Version('9999.0') - elif version_string == STABLE_VERBOSE_NAME: comparable = Version('99999.0') + elif version_string == STABLE_VERBOSE_NAME: + comparable = Version('9999.0') else: comparable = Version('0.01') return comparable From 1f13be1861362fd4255412015b44f6629ad80001 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 25 May 2019 17:34:46 +0600 Subject: [PATCH 021/171] doc string fix --- readthedocs/core/views/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index f7ab665bbc5..49207c0b5bd 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -373,7 +373,7 @@ def changefreqs_generator(): """ Generator returning ``changefreq`` needed by sitemap.xml. - It returns ``weekly`` on first iteration, then ``daily`` and then it + It returns ``daily`` on first iteration, then ``weekly`` and then it will return always ``monthly``. We are using ``monthly`` as last value because ``never`` is too From e0232b3e4ea4ba37bade2fdeb294d846629f8ab1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 29 May 2019 17:04:04 +0600 Subject: [PATCH 022/171] sort_by_priority function added to sitemap --- readthedocs/core/views/serve.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 49207c0b5bd..80db933ab69 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -383,6 +383,14 @@ def changefreqs_generator(): changefreqs = ['daily', 'weekly'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) + def sort_by_priority(version_list): + """This will sort the versions by priority""" + return sorted( + version_list, + key=lambda version: version['priority'], + reverse=True + ) + if project.privacy_level == constants.PRIVATE: raise Http404 @@ -437,7 +445,7 @@ def changefreqs_generator(): versions.append(element) context = { - 'versions': versions, + 'versions': sort_by_priority(versions), } return render( request, From fdcfcf6b0517af063f39adf3b8fbf06929e3ed01 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 29 May 2019 17:14:19 +0600 Subject: [PATCH 023/171] lint fix --- readthedocs/core/views/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 80db933ab69..9a0dd3593cc 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -384,7 +384,7 @@ def changefreqs_generator(): yield from itertools.chain(changefreqs, itertools.repeat('monthly')) def sort_by_priority(version_list): - """This will sort the versions by priority""" + """Sorts the versions by priority. i.e: 1, 0.9, 0.8...""" return sorted( version_list, key=lambda version: version['priority'], From ef1321491e1ec767e53a81600ddb3c9145968d14 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 00:33:08 +0600 Subject: [PATCH 024/171] Sorting by swapping positions --- readthedocs/core/views/serve.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 9a0dd3593cc..3d1a971cfd4 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,6 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve +from readthedocs.builds.constants import LATEST, STABLE from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path @@ -355,7 +356,7 @@ def priorities_generator(): It generates values from 1 to 0.1 by decreasing in 0.1 on each iteration. After 0.1 is reached, it will keep returning 0.1. """ - priorities = [0.9, 1, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] + priorities = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] yield from itertools.chain(priorities, itertools.repeat(0.1)) def hreflang_formatter(lang): @@ -373,24 +374,16 @@ def changefreqs_generator(): """ Generator returning ``changefreq`` needed by sitemap.xml. - It returns ``daily`` on first iteration, then ``weekly`` and then it + It returns ``weekly`` on first iteration, then ``daily`` and then it will return always ``monthly``. We are using ``monthly`` as last value because ``never`` is too aggressive. If the tag is removed and a branch is created with the same name, we will want bots to revisit this. """ - changefreqs = ['daily', 'weekly'] + changefreqs = ['weekly', 'daily'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) - def sort_by_priority(version_list): - """Sorts the versions by priority. i.e: 1, 0.9, 0.8...""" - return sorted( - version_list, - key=lambda version: version['priority'], - reverse=True - ) - if project.privacy_level == constants.PRIVATE: raise Http404 @@ -400,6 +393,18 @@ def sort_by_priority(version_list): only_active=True, ), ) + + # This is a hack to swap the latest version with + # stable version to get the stable version first in the sitemap. + # We want stable with priority=1 and changefreq='weekly' and + # latest with priority=0.9 and changefreq='daily' + # More details on this: https://github.com/rtfd/readthedocs.org/issues/5447 + if ( + sorted_versions[0].slug == LATEST and + sorted_versions[1].slug == STABLE + ): + sorted_versions[0], sorted_versions[1] = sorted_versions[1], sorted_versions[0] + versions = [] for version, priority, changefreq in zip( sorted_versions, @@ -445,7 +450,7 @@ def sort_by_priority(version_list): versions.append(element) context = { - 'versions': sort_by_priority(versions), + 'versions': versions, } return render( request, From 736a3377128f3ce1f773a05aafa4081a1b27f657 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 17:30:50 +0600 Subject: [PATCH 025/171] index out of range issue fix --- readthedocs/core/views/serve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 3d1a971cfd4..237b843e0fb 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -400,6 +400,7 @@ def changefreqs_generator(): # latest with priority=0.9 and changefreq='daily' # More details on this: https://github.com/rtfd/readthedocs.org/issues/5447 if ( + len(sorted_versions) >= 2 and sorted_versions[0].slug == LATEST and sorted_versions[1].slug == STABLE ): From c18f3810a5f76ee7038e095778eb52e5b4b2d0f7 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 12 Jun 2019 21:55:35 +0200 Subject: [PATCH 026/171] Use a real SessionBase object on FooterNoSessionMiddleware This allows all Django internals method to keep working in the same way but still using an empty session. --- readthedocs/core/middleware.py | 3 ++- readthedocs/rtd_tests/tests/test_footer.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 60cf4a277d3..692e45a2f8d 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -1,6 +1,7 @@ import logging from django.conf import settings +from django.contrib.sessions.backends.base import SessionBase from django.contrib.sessions.middleware import SessionMiddleware from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.http import Http404, HttpResponseBadRequest @@ -205,7 +206,7 @@ def process_request(self, request): settings.SESSION_COOKIE_NAME not in request.COOKIES ): # Hack request.session otherwise the Authentication middleware complains. - request.session = {} + request.session = SessionBase() # create an empty session return super().process_request(request) diff --git a/readthedocs/rtd_tests/tests/test_footer.py b/readthedocs/rtd_tests/tests/test_footer.py index 34c1fec1f92..241bc434f59 100644 --- a/readthedocs/rtd_tests/tests/test_footer.py +++ b/readthedocs/rtd_tests/tests/test_footer.py @@ -1,4 +1,5 @@ import mock +from django.contrib.sessions.backends.base import SessionBase from django.test import TestCase from rest_framework.test import APIRequestFactory, APITestCase @@ -80,7 +81,8 @@ def test_no_session_logged_out(self): # Null session here request = self.factory.get('/api/v2/footer_html/') mid.process_request(request) - self.assertEqual(request.session, {}) + self.assertIsInstance(request.session, SessionBase) + self.assertEqual(list(request.session.keys()), []) # Proper session here home_request = self.factory.get('/') From 908860a2bc4f41943982ae17097f0cc88db25e2b Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 16:59:23 +0600 Subject: [PATCH 027/171] Version Type Added --- readthedocs/builds/constants.py | 2 ++ .../0008_added_pull_request_version_type.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 readthedocs/builds/migrations/0008_added_pull_request_version_type.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index b0dc6cfba94..e2e4fc44ac3 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -33,11 +33,13 @@ BRANCH = 'branch' TAG = 'tag' +PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), + (PULL_REQUEST, _('Pull Request')), (UNKNOWN, _('Unknown')), ) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py new file mode 100644 index 00000000000..14d8968866d --- /dev/null +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-05-30 10:02 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('builds', '0007_add-automation-rules'), + ] + + operations = [ + migrations.AlterField( + model_name='version', + name='type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + ), + migrations.AlterField( + model_name='versionautomationrule', + name='version_type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + ), + ] From f81bcf0718fccc8bad270da517b97dfb4df7e9c9 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 23:52:36 +0600 Subject: [PATCH 028/171] Version Model Methods Updated --- readthedocs/builds/models.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 1039b815326..ca44b39f252 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -40,6 +40,7 @@ BUILD_TYPES, LATEST, NON_REPOSITORY_VERSIONS, + PULL_REQUEST, STABLE, TAG, VERSION_TYPES, @@ -142,7 +143,8 @@ def vcs_url(self): """ Generate VCS (github, gitlab, bitbucket) URL for this version. - Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -152,12 +154,24 @@ def vcs_url(self): else: slug_url = self.slug - if ('github' in self.project.repo) or ('gitlab' in self.project.repo): - url = f'/tree/{slug_url}/' + if self.type == PULL_REQUEST: + if 'github' in self.project.repo: + url = f'/pull/{slug_url}/' - if 'bitbucket' in self.project.repo: - slug_url = self.identifier - url = f'/src/{slug_url}' + if 'gitlab' in self.project.repo: + slug_url = self.identifier + url = f'/merge_requests/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/pull-requests/{slug_url}' + else: + if ('github' in self.project.repo) or ('gitlab' in self.project.repo): + url = f'/tree/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/src/{slug_url}' # TODO: improve this replacing return self.project.repo.replace('git://', 'https://').replace('.git', '') + url @@ -220,7 +234,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - # If we came that far it's not a special version nor a branch or tag. + if self.type == PULL_REQUEST: + # If this version is a Pull Request, the identifier will + # contain the actual commit hash. which we can use to + # generate url for a given file name + return self.identifier + + # If we came that far it's not a special version + # nor a branch, tag or Pull Request. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', From 66377c2d75ad702f6ff3520bd2bdf43f7ab739da Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 15:40:32 +0600 Subject: [PATCH 029/171] Internal and External Version Manager added --- readthedocs/builds/managers.py | 33 +++++++++++++++++++++++++++++++++ readthedocs/builds/models.py | 10 +++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 68cdec6efe9..991293beae2 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,6 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, + PULL_REQUEST, ) from .querysets import VersionQuerySet @@ -85,6 +86,38 @@ def get_object_or_log(self, **kwargs): log.warning('Version not found for given kwargs. %s' % kwargs) +class InternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes internal version. + + It will exclude PULL_REQUEST type from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + def get_queryset(self): + return super().get_queryset().exclude(type=PULL_REQUEST) + + +class ExternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes external version. + + It will only include PULL_REQUEST type Versions in the queries. + """ + def get_queryset(self): + return super().get_queryset().filter(type=PULL_REQUEST) + + class VersionManager(SettingsOverrideObject): _default_class = VersionManagerBase _override_setting = 'VERSION_MANAGER' + + +class InternalVersionManager(SettingsOverrideObject): + _default_class = InternalVersionManagerBase + _override_setting = 'INTERNAL_VERSION_MANAGER' + + +class ExternalVersionManager(SettingsOverrideObject): + _default_class = ExternalVersionManagerBase + _override_setting = 'EXTERNAL_VERSION_MANAGER' + diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index ca44b39f252..bca375570c9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -45,7 +45,11 @@ TAG, VERSION_TYPES, ) -from .managers import VersionManager +from .managers import ( + VersionManager, + InternalVersionManager, + ExternalVersionManager +) from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet from .utils import ( get_bitbucket_username_repo, @@ -112,6 +116,10 @@ class Version(models.Model): machine = models.BooleanField(_('Machine Created'), default=False) objects = VersionManager.from_queryset(VersionQuerySet)() + # Only include BRANCH, TAG, UNKONWN type Versions. + internal = InternalVersionManager.from_queryset(VersionQuerySet)() + # Only include PULL_REQUEST type Versions. + external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: unique_together = [('project', 'slug')] From 8bdf75c69fcd72a86abb0ac5cd05d09b0b5cf97a Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 16:20:19 +0600 Subject: [PATCH 030/171] lint fix --- readthedocs/builds/managers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 991293beae2..2a8bb4840d9 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -87,22 +87,26 @@ def get_object_or_log(self, **kwargs): class InternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes internal version. It will exclude PULL_REQUEST type from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ + def get_queryset(self): return super().get_queryset().exclude(type=PULL_REQUEST) class ExternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes external version. It will only include PULL_REQUEST type Versions in the queries. """ + def get_queryset(self): return super().get_queryset().filter(type=PULL_REQUEST) @@ -120,4 +124,3 @@ class InternalVersionManager(SettingsOverrideObject): class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase _override_setting = 'EXTERNAL_VERSION_MANAGER' - From 21f96de12712042a7ad42ba89052b092bb3504bb Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 5 Jun 2019 01:18:20 +0600 Subject: [PATCH 031/171] All Version Querysets Updated with InternalVersionManager --- readthedocs/api/v2/views/model_views.py | 2 +- readthedocs/api/v3/views.py | 2 +- readthedocs/builds/models.py | 4 +- readthedocs/builds/views.py | 2 +- .../core/management/commands/update_repos.py | 42 +++++++++++++++++++ .../management/commands/update_versions.py | 2 +- readthedocs/core/views/serve.py | 4 +- readthedocs/projects/admin.py | 4 +- readthedocs/projects/forms.py | 2 +- readthedocs/projects/models.py | 8 ++-- readthedocs/projects/views/public.py | 6 +-- 11 files changed, 61 insertions(+), 17 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index af41f28ab84..5f594a032b8 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions.filter(active=True) + versions = project.versions(manager='internal').filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 0e8b2b55a0e..57b86360e96 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -230,7 +230,7 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, lookup_value_regex = r'[^/]+' filterset_class = VersionFilter - queryset = Version.objects.all() + queryset = Version.internal.all() permit_list_expands = [ 'last_build', 'last_build.config', diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bca375570c9..bcfc5dfff5c 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -142,7 +142,9 @@ def __str__(self): @property def ref(self): if self.slug == STABLE: - stable = determine_stable_version(self.project.versions.all()) + stable = determine_stable_version( + self.project.versions(manager='internal').all() + ) if stable: return stable.slug diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index e8e3d458e2c..3d6e2cc0ca1 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context['project'] = self.project context['active_builds'] = active_builds - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=self.project, ) diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 5852592d360..0e14f9ce339 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -76,6 +76,48 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) + elif version == 'internal': + log.info('Updating all internal versions for %s', slug) + for version in Version.internal.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) + elif version == 'external': + log.info('Updating all external versions for %s', slug) + for version in Version.external.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) else: p = Project.all_objects.get(slug=slug) log.info('Building %s', p) diff --git a/readthedocs/core/management/commands/update_versions.py b/readthedocs/core/management/commands/update_versions.py index b0aa1f877cb..6456accc430 100644 --- a/readthedocs/core/management/commands/update_versions.py +++ b/readthedocs/core/management/commands/update_versions.py @@ -13,7 +13,7 @@ class Command(BaseCommand): help = __doc__ def handle(self, *args, **options): - for version in Version.objects.filter(active=True, built=False): + for version in Version.internal.filter(active=True, built=False): # pylint: disable=no-value-for-parameter update_docs_task( version.pk, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 237b843e0fb..f03f958b172 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -388,7 +388,7 @@ def changefreqs_generator(): raise Http404 sorted_versions = sort_version_aware( - Version.objects.public( + Version.internal.public( project=project, only_active=True, ), @@ -428,7 +428,7 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): translation_versions = ( - Version.objects.public(project=translation) + Version.internal.public(project=translation) .values_list('slug', flat=True) ) if version.slug in translation_versions: diff --git a/readthedocs/projects/admin.py b/readthedocs/projects/admin.py index d74e8d99075..fb4aef4772d 100644 --- a/readthedocs/projects/admin.py +++ b/readthedocs/projects/admin.py @@ -239,7 +239,7 @@ def reindex_active_versions(self, request, queryset): """Reindex all active versions of the selected projects to ES.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) active_versions = version_qs.filter(active=True) if not active_versions.exists(): @@ -271,7 +271,7 @@ def wipe_all_versions(self, request, queryset): """Wipe indexes of all versions of selected projects.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) if not version_qs.exists(): self.message_user( request, diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index a701a49c020..65c589c8b18 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -240,7 +240,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions.filter( + versions_choices = self.instance.versions(manager='internal').filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index b636b348aa8..dff87e3885f 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -908,7 +908,7 @@ def api_versions(self): def active_versions(self): from readthedocs.builds.models import Version - versions = Version.objects.public(project=self, only_active=True) + versions = Version.internal.public(project=self, only_active=True) return ( versions.filter(built=True, active=True) | versions.filter(active=True, uploaded=True) @@ -922,7 +922,7 @@ def ordered_active_versions(self, user=None): } if user: kwargs['user'] = user - versions = Version.objects.public(**kwargs).select_related( + versions = Version.internal.public(**kwargs).select_related( 'project', 'project__main_language_project', ).prefetch_related( @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions.filter(active=True) + return self.versions(manager='internal').filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions.all() + versions = self.versions(manager='internal').all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index 64d82272738..ea15bfb168b 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) project = self.get_object() - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=project, ) @@ -179,7 +179,7 @@ def project_downloads(request, project_slug): Project.objects.protected(request.user), slug=project_slug, ) - versions = Version.objects.public(user=request.user, project=project) + versions = Version.internal.public(user=request.user, project=project) versions = sort_version_aware(versions) version_data = OrderedDict() for version in versions: @@ -268,7 +268,7 @@ def project_versions(request, project_slug): slug=project_slug, ) - versions = Version.objects.public( + versions = Version.internal.public( user=request.user, project=project, only_active=False, From 014477d7cafbb58092c0f3781fe433f9de0fa2c4 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 02:15:34 +0600 Subject: [PATCH 032/171] Manager names moved to Constants --- readthedocs/api/v2/views/model_views.py | 4 ++-- readthedocs/builds/constants.py | 7 +++++++ readthedocs/builds/models.py | 5 +++-- readthedocs/core/management/commands/update_repos.py | 5 +++-- readthedocs/core/views/serve.py | 8 +++++++- readthedocs/projects/forms.py | 3 ++- readthedocs/projects/models.py | 6 +++--- 7 files changed, 27 insertions(+), 11 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index 5f594a032b8..fca91b32626 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG +from readthedocs.builds.constants import BRANCH, TAG, INTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions(manager='internal').filter(active=True) + versions = project.versions(manager=INTERNAL).filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e2e4fc44ac3..e200fc5fe25 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -55,3 +55,10 @@ LATEST, STABLE, ) + +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding PULL_REQUEST Type. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only PULL_REQUEST Type Versions and Builds. +EXTERNAL = 'external' diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bcfc5dfff5c..d0c5f6a854e 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -32,12 +32,13 @@ from readthedocs.projects.models import APIProject, Project from readthedocs.projects.version_handling import determine_stable_version -from .constants import ( +from readthedocs.builds.constants import ( BRANCH, BUILD_STATE, BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, BUILD_TYPES, + INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, PULL_REQUEST, @@ -143,7 +144,7 @@ def __str__(self): def ref(self): if self.slug == STABLE: stable = determine_stable_version( - self.project.versions(manager='internal').all() + self.project.versions(manager=INTERNAL).all() ) if stable: return stable.slug diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 0e14f9ce339..b15ea8c3c9c 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -10,6 +10,7 @@ from django.core.management.base import BaseCommand +from readthedocs.builds.constants import EXTERNAL, INTERNAL from readthedocs.builds.models import Build, Version from readthedocs.core.utils import trigger_build from readthedocs.projects import tasks @@ -76,7 +77,7 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) - elif version == 'internal': + elif version == INTERNAL: log.info('Updating all internal versions for %s', slug) for version in Version.internal.filter( project__slug=slug, @@ -97,7 +98,7 @@ def handle(self, *args, **options): build_pk=build.pk, version_pk=version.pk, ) - elif version == 'external': + elif version == EXTERNAL: log.info('Updating all external versions for %s', slug) for version in Version.external.filter( project__slug=slug, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index f03f958b172..d82e8b60443 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,7 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, INTERNAL from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path @@ -427,10 +427,16 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): +<<<<<<< HEAD translation_versions = ( Version.internal.public(project=translation) .values_list('slug', flat=True) ) +======= + translation_versions = translation.versions( + manager=INTERNAL + ).public().values_list('slug', flat=True) +>>>>>>> Manager names moved to Constants if version.slug in translation_versions: href = project.get_docs_url( version_slug=version.slug, diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index 65c589c8b18..3c86cc39e12 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -14,6 +14,7 @@ from guardian.shortcuts import assign from textclassifier.validators import ClassifierValidator +from readthedocs.builds.constants import INTERNAL from readthedocs.core.utils import slugify, trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.integrations.models import Integration @@ -240,7 +241,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions(manager='internal').filter( + versions_choices = self.instance.versions(manager=INTERNAL).filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index dff87e3885f..a3f36feb5c8 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -20,7 +20,7 @@ from taggit.managers import TaggableManager from readthedocs.api.v2.client import api -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, INTERNAL from readthedocs.core.resolver import resolve, resolve_domain from readthedocs.core.utils import broadcast, slugify from readthedocs.projects import constants @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions(manager='internal').filter(active=True) + return self.versions(manager=INTERNAL).filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions(manager='internal').all() + versions = self.versions(manager=INTERNAL).all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() From d3f52542e96d027f282a5876e579daeec4ed81da Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:25:03 +0600 Subject: [PATCH 033/171] Tests added --- readthedocs/builds/constants.py | 2 +- .../rtd_tests/tests/test_doc_serving.py | 26 ++++- readthedocs/rtd_tests/tests/test_managers.py | 100 ++++++++++++++++++ readthedocs/rtd_tests/tests/test_project.py | 28 ++++- .../rtd_tests/tests/test_project_forms.py | 25 ++++- .../rtd_tests/tests/test_project_views.py | 30 +++++- readthedocs/rtd_tests/tests/test_version.py | 68 ++++++++++++ 7 files changed, 272 insertions(+), 7 deletions(-) create mode 100644 readthedocs/rtd_tests/tests/test_managers.py create mode 100644 readthedocs/rtd_tests/tests/test_version.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e200fc5fe25..37474eb5365 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -58,7 +58,7 @@ # Manager name for Internal Versions or Builds. # ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' +INTERNAL = 'internal' # Manager name for External Versions or Builds. # ie: Only PULL_REQUEST Type Versions and Builds. EXTERNAL = 'external' diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index fed119bbab5..05d1308f4c1 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -10,7 +10,11 @@ from django.urls import reverse from mock import mock_open, patch +<<<<<<< HEAD from readthedocs.builds.constants import LATEST +======= +from readthedocs.builds.constants import PULL_REQUEST, INTERNAL +>>>>>>> Tests added from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -249,6 +253,16 @@ def test_sitemap_xml(self): project=self.public, active=True ) + # This is a Pull Request Version + pr_version = fixture.get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.public, + active=True, + type=PULL_REQUEST + ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( Project, @@ -268,7 +282,7 @@ def test_sitemap_xml(self): ) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/xml') - for version in self.public.versions.filter(privacy_level=constants.PUBLIC): + for version in self.public.versions(manager=INTERNAL).filter(privacy_level=constants.PUBLIC): self.assertContains( response, self.public.get_docs_url( @@ -303,6 +317,16 @@ def test_sitemap_xml(self): # in language and country value. (zh_CN should be zh-CN) self.assertContains(response, 'zh-CN') + # External Versions should not be in the sitemap_xml. + self.assertNotContains( + response, + self.public.get_docs_url( + version_slug=pr_version.slug, + lang_slug=self.public.language, + private=True, + ), + ) + # Check if STABLE version has 'priority of 1 and changefreq of weekly. self.assertEqual( response.context['versions'][0]['loc'], diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py new file mode 100644 index 00000000000..8e440e18b85 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -0,0 +1,100 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED +from readthedocs.projects.models import Project + + +class TestVersionManagerBase(TestCase): + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.public_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC + ) + self.private_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PRIVATE + ) + self.protected_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PROTECTED + ) + self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + + +class TestInternalVersionManager(TestVersionManagerBase): + + """ + Queries using Internal Manager should only include Internal Versions. + + It will exclude PULL_REQUEST type Versions from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + + def test_internal_version_manager_with_all(self): + self.assertNotIn(self.public_pr_version, Version.internal.all()) + + def test_internal_version_manager_with_public(self): + self.assertNotIn(self.public_pr_version, Version.internal.public()) + + def test_internal_version_manager_with_protected(self): + self.assertNotIn(self.protected_pr_version, Version.internal.protected()) + + def test_internal_version_manager_with_private(self): + self.assertNotIn(self.private_pr_version, Version.internal.private()) + + def test_internal_version_manager_with_api(self): + self.assertNotIn(self.public_pr_version, Version.internal.api()) + + def test_internal_version_manager_with_for_project(self): + self.assertNotIn(self.public_pr_version, Version.internal.for_project(self.pip)) + + +class TestExternalVersionManager(TestVersionManagerBase): + + """ + Queries using External Manager should only include External Versions. + + It will only include PULL_REQUEST type Versions in the queries. + """ + + def test_external_version_manager_with_all(self): + self.assertNotIn(self.internal_versions, Version.external.all()) + self.assertIn(self.public_pr_version, Version.external.all()) + + def test_external_version_manager_with_public(self): + self.assertNotIn(self.internal_versions, Version.external.public()) + self.assertIn(self.public_pr_version, Version.external.public()) + + def test_external_version_manager_with_protected(self): + self.assertNotIn(self.internal_versions, Version.external.protected()) + self.assertIn(self.protected_pr_version, Version.external.protected()) + + def test_external_version_manager_with_private(self): + self.assertNotIn(self.internal_versions, Version.external.private()) + self.assertIn(self.private_pr_version, Version.external.private()) + + def test_external_version_manager_with_api(self): + self.assertNotIn(self.internal_versions, Version.external.api()) + self.assertIn(self.public_pr_version, Version.external.api()) + + def test_external_version_manager_with_for_project(self): + self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) + self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 40f86350ed4..c2db080f201 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,8 +14,9 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, + PULL_REQUEST, ) -from readthedocs.builds.models import Build +from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project from readthedocs.projects.tasks import finish_inactive_builds @@ -29,6 +30,16 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) class TestProject(ProjectMixin, TestCase): @@ -134,6 +145,21 @@ def test_get_storage_path(self): 'htmlzip/pip/latest/pip.zip', ) + def test_ordered_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.ordered_active_versions()) + + def test_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.active_versions()) + + def test_all_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.all_active_versions()) + + def test_update_stable_version_excludes_pr_versions(self): + # Delete all versions excluding PR Versions. + self.pip.versions.exclude(type=PULL_REQUEST).delete() + # Test that PR Version is not considered for stable. + self.assertEqual(self.pip.update_stable_version(), None) + class TestProjectTranslations(ProjectMixin, TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index 50d9815d051..b2e8687208a 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -314,7 +314,7 @@ def setUp(self): verbose_name='protected', ) - def test_list_only_non_auto_generated_versions_on_default_branch(self): + def test_list_only_non_auto_generated_versions_in_default_branch_choices(self): form = ProjectAdvancedForm(instance=self.project) # This version is created automatically by the project on save latest = self.project.versions.filter(slug=LATEST) @@ -336,7 +336,7 @@ def test_list_only_non_auto_generated_versions_on_default_branch(self): 'default_branch'].widget.choices], ) - def test_list_user_created_latest_and_stable_versions_on_default_branch(self): + def test_list_user_created_latest_and_stable_versions_in_default_branch_choices(self): self.project.versions.filter(slug=LATEST).first().delete() user_created_latest_version = get( Version, @@ -383,6 +383,25 @@ def test_commit_name_not_in_default_branch_choices(self): 'default_branch'].widget.choices], ) + def test_pr_version_not_in_default_branch_choices(self): + pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.project, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC, + ) + form = ProjectAdvancedForm(instance=self.project) + + self.assertNotIn( + pr_version.verbose_name, + [identifier for identifier, _ in form.fields[ + 'default_branch'].widget.choices], + ) + class TestTranslationForms(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 7c5b286bdd4..90578c8dceb 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST +from readthedocs.builds.constants import LATEST, PULL_REQUEST from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -370,12 +370,40 @@ def test_import_demo_imported_duplicate(self): class TestPublicViews(MockBuildTestCase): def setUp(self): self.pip = get(Project, slug='pip') + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) def test_project_download_media(self): url = reverse('project_download_media', args=[self.pip.slug, 'pdf', LATEST]) response = self.client.get(url) self.assertEqual(response.status_code, 302) + def test_project_detail_view_only_shows_internal_versons(self): + url = reverse('projects_detail', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_downloads_only_shows_internal_versons(self): + url = reverse('project_downloads', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_versions_only_shows_internal_versons(self): + url = reverse('project_version_list', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['active_versions']) + self.assertNotIn(self.pr_version, response.context['inactive_versions']) + class TestPrivateViews(MockBuildTestCase): def setUp(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py new file mode 100644 index 00000000000..8a6f5bf3db8 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -0,0 +1,68 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.models import Project + + +class VersionMixin: + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='9F86D081884C7D659A2FEAA0C55AD015A', + verbose_name='pr-version', + slug='9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) + self.branch_version = get( + Version, + identifier='origin/stable', + verbose_name='stable', + slug='stable', + project=self.pip, + active=True, + type=BRANCH + ) + self.tag_version = get( + Version, + identifier='origin/master', + verbose_name='latest', + slug='latest', + project=self.pip, + active=True, + type=TAG + ) + + +class TestVersionModel(VersionMixin, TestCase): + + def test_vcs_url_for_pr_version(self): + expected_url = f'https://github.com/pypa/pip/pull/{self.pr_version.slug}/' + self.assertEqual(self.pr_version.vcs_url, expected_url) + + def test_vcs_url_for_latest_version(self): + slug = self.pip.default_branch or self.pip.vcs_repo().fallback_branch + expected_url = f'https://github.com/pypa/pip/tree/{slug}/' + self.assertEqual(self.tag_version.vcs_url, expected_url) + + def test_vcs_url_for_stable_version(self): + expected_url = f'https://github.com/pypa/pip/tree/{self.branch_version.ref}/' + self.assertEqual(self.branch_version.vcs_url, expected_url) + + def test_commit_name_for_stable_version(self): + self.assertEqual(self.branch_version.commit_name, 'stable') + + def test_commit_name_for_latest_version(self): + self.assertEqual(self.tag_version.commit_name, 'master') + + def test_commit_name_for_pr_version(self): + self.assertEqual(self.pr_version.commit_name, self.pr_version.identifier) From 6bca1e42472e67da0b564c332debeaa44d599437 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:52:31 +0600 Subject: [PATCH 034/171] Version Update, Delete Html and wipe updated to exclude PR Versions --- readthedocs/core/views/__init__.py | 2 +- readthedocs/projects/views/private.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 6b438866ae5..de519a13759 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -73,7 +73,7 @@ def random_page(request, project_slug=None): # pylint: disable=unused-argument def wipe_version(request, project_slug, version_slug): version = get_object_or_404( - Version, + Version.internal.all(), project__slug=project_slug, slug=version_slug, ) diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index 46e83d96188..044a83641e6 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -157,7 +157,7 @@ def project_version_detail(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, @@ -682,7 +682,7 @@ def project_version_delete_html(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, From 8830a187ed84a56314ddba5c497b39065419c2cb Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 23:03:09 +0600 Subject: [PATCH 035/171] Updated migrations --- .../builds/migrations/0008_added_pull_request_version_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 14d8968866d..75a126d37b6 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-05-30 10:02 +# Generated by Django 1.11.20 on 2019-06-06 16:51 from __future__ import unicode_literals from django.db import migrations, models From 5ed38c4dc238122aa69d5cd06fc78a29c76e5ff4 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sun, 9 Jun 2019 02:17:05 +0600 Subject: [PATCH 036/171] footer API updated --- readthedocs/api/v2/views/footer_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/api/v2/views/footer_views.py b/readthedocs/api/v2/views/footer_views.py index 04d4fe8210b..04fedeadd3f 100644 --- a/readthedocs/api/v2/views/footer_views.py +++ b/readthedocs/api/v2/views/footer_views.py @@ -9,7 +9,7 @@ from rest_framework_jsonp.renderers import JSONPRenderer from readthedocs.api.v2.signals import footer_response -from readthedocs.builds.constants import LATEST, TAG +from readthedocs.builds.constants import LATEST, TAG, INTERNAL from readthedocs.builds.models import Version from readthedocs.projects.models import Project from readthedocs.projects.version_handling import ( @@ -25,7 +25,7 @@ def get_version_compare_data(project, base_version=None): :param base_version: We assert whether or not the base_version is also the highest version in the resulting "is_highest" value. """ - versions_qs = Version.objects.public(project=project) + versions_qs = Version.internal.public(project=project) # Take preferences over tags only if the project has at least one tag if versions_qs.filter(type=TAG).exists(): From 2d8b072255d2c42949ea4309c8864a146955a6c7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 00:45:00 +0600 Subject: [PATCH 037/171] manager tests updated --- readthedocs/rtd_tests/tests/test_managers.py | 34 +++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 8e440e18b85..e69abdc4dd8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -1,3 +1,4 @@ +from django.contrib.auth import get_user_model from django.test import TestCase from django_dynamic_fixture import get @@ -7,12 +8,16 @@ from readthedocs.projects.models import Project +User = get_user_model() + + class TestVersionManagerBase(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ['test_data'] def setUp(self): - self.client.login(username='eric', password='test') + self.user = User.objects.create(username='test_user', password='test') + self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') # Create a External Version. ie: PULL_REQUEST type Version. self.public_pr_version = get( @@ -54,6 +59,12 @@ def test_internal_version_manager_with_all(self): def test_internal_version_manager_with_public(self): self.assertNotIn(self.public_pr_version, Version.internal.public()) + def test_internal_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.public_pr_version, + Version.internal.public(self.user, self.pip) + ) + def test_internal_version_manager_with_protected(self): self.assertNotIn(self.protected_pr_version, Version.internal.protected()) @@ -83,6 +94,17 @@ def test_external_version_manager_with_public(self): self.assertNotIn(self.internal_versions, Version.external.public()) self.assertIn(self.public_pr_version, Version.external.public()) + def test_external_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.internal_versions, + Version.external.public(self.user, self.pip) + ) + self.assertIn( + self.public_pr_version, + Version.external.public(self.user, self.pip) + ) + + def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) @@ -96,5 +118,9 @@ def test_external_version_manager_with_api(self): self.assertIn(self.public_pr_version, Version.external.api()) def test_external_version_manager_with_for_project(self): - self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) - self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) + self.assertNotIn( + self.internal_versions, Version.external.for_project(self.pip) + ) + self.assertIn( + self.public_pr_version, Version.external.for_project(self.pip) + ) From 89f3c8af6c2d28625827bc0dc36c10f8b66424d3 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 18:56:06 +0600 Subject: [PATCH 038/171] _override_setting removed from new managers --- readthedocs/builds/managers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 2a8bb4840d9..0bc30f02807 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -118,9 +118,7 @@ class VersionManager(SettingsOverrideObject): class InternalVersionManager(SettingsOverrideObject): _default_class = InternalVersionManagerBase - _override_setting = 'INTERNAL_VERSION_MANAGER' class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase - _override_setting = 'EXTERNAL_VERSION_MANAGER' From 7b63973dc50f3bee61988e40387a9f47e2803f32 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:11:51 +0600 Subject: [PATCH 039/171] BuildTriggerMixin updated --- readthedocs/builds/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index 3d6e2cc0ca1..3333495f6c8 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -57,7 +57,7 @@ def post(self, request, project_slug): version_slug = request.POST.get('version_slug') version = get_object_or_404( - Version, + Version.internal.all(), project=project, slug=version_slug, ) From 3a2fc6dcb1ef267d5f5bd1d0db5989837b7fbfb3 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:25:28 +0600 Subject: [PATCH 040/171] More Update --- readthedocs/search/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 5f04f6736b7..cab67ba5ff8 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,7 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.objects.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) if version.exists(): project_list.append(version.first().project) return project_list From bca152a69c201c61ab508bbd00b26b7e51cd884a Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:55:32 +0600 Subject: [PATCH 041/171] lint fix --- readthedocs/rtd_tests/tests/test_managers.py | 1 - readthedocs/search/utils.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index e69abdc4dd8..f830c9af549 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -104,7 +104,6 @@ def test_external_version_manager_with_public_with_user_and_project(self): Version.external.public(self.user, self.pip) ) - def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index cab67ba5ff8..3433195bb55 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,9 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter( + project__slug=project.slug, slug=version_slug + ) if version.exists(): project_list.append(version.first().project) return project_list From 5f900529099fcd83d8f03b699a43e584638f0335 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 13 Jun 2019 14:53:57 -0500 Subject: [PATCH 042/171] Move search functions Follow up from #5798 --- readthedocs/projects/tasks.py | 2 +- readthedocs/search/signals.py | 61 -------------------------------- readthedocs/search/utils.py | 66 +++++++++++++++++++++++++++++++++-- 3 files changed, 64 insertions(+), 65 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index d0ebdb826a0..7efcd5a762a 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -60,7 +60,7 @@ from readthedocs.doc_builder.loader import get_builder_class from readthedocs.doc_builder.python_environments import Conda, Virtualenv from readthedocs.projects.models import APIProject, Feature -from readthedocs.search.signals import index_new_files, remove_indexed_files +from readthedocs.search.utils import index_new_files, remove_indexed_files from readthedocs.sphinx_domains.models import SphinxDomain from readthedocs.vcs_support import utils as vcs_support_utils from readthedocs.worker import app diff --git a/readthedocs/search/signals.py b/readthedocs/search/signals.py index 494725d22a7..294d96242d0 100644 --- a/readthedocs/search/signals.py +++ b/readthedocs/search/signals.py @@ -13,67 +13,6 @@ log = logging.getLogger(__name__) -def index_new_files(model, version, build): - """Index new files from the version into the search index.""" - - if not DEDConfig.autosync_enabled(): - log.info( - 'Autosync disabled, skipping indexing into the search index for: %s:%s', - version.project.slug, - version.slug, - ) - return - - try: - document = list(registry.get_documents(models=[model]))[0] - doc_obj = document() - queryset = ( - doc_obj.get_queryset() - .filter(project=version.project, version=version, build=build) - ) - log.info( - 'Indexing new objecst into search index for: %s:%s', - version.project.slug, - version.slug, - ) - doc_obj.update(queryset.iterator()) - except Exception: - log.exception('Unable to index a subset of files. Continuing.') - - -def remove_indexed_files(model, version, build): - """ - Remove files from the version from the search index. - - This excludes files from the current build. - """ - - if not DEDConfig.autosync_enabled(): - log.info( - 'Autosync disabled, skipping removal from the search index for: %s:%s', - version.project.slug, - version.slug, - ) - return - - try: - document = list(registry.get_documents(models=[model]))[0] - log.info( - 'Deleting old files from search index for: %s:%s', - version.project.slug, - version.slug, - ) - ( - document().search() - .filter('term', project=version.project.slug) - .filter('term', version=version.slug) - .exclude('term', build=build) - .delete() - ) - except Exception: - log.exception('Unable to delete a subset of files. Continuing.') - - @receiver(post_save, sender=Project) def index_project_save(instance, *args, **kwargs): """ diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 3433195bb55..6cbc0272722 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -1,20 +1,80 @@ -# -*- coding: utf-8 -*- - """Utilities related to reading and generating indexable search content.""" import logging from django.shortcuts import get_object_or_404 +from django_elasticsearch_dsl.apps import DEDConfig from django_elasticsearch_dsl.registries import registry from readthedocs.builds.models import Version -from readthedocs.projects.models import Project, HTMLFile +from readthedocs.projects.models import HTMLFile, Project from readthedocs.search.documents import PageDocument log = logging.getLogger(__name__) +def index_new_files(model, version, build): + """Index new files from the version into the search index.""" + + if not DEDConfig.autosync_enabled(): + log.info( + 'Autosync disabled, skipping indexing into the search index for: %s:%s', + version.project.slug, + version.slug, + ) + return + + try: + document = list(registry.get_documents(models=[model]))[0] + doc_obj = document() + queryset = ( + doc_obj.get_queryset() + .filter(project=version.project, version=version, build=build) + ) + log.info( + 'Indexing new objecst into search index for: %s:%s', + version.project.slug, + version.slug, + ) + doc_obj.update(queryset.iterator()) + except Exception: + log.exception('Unable to index a subset of files. Continuing.') + + +def remove_indexed_files(model, version, build): + """ + Remove files from the version from the search index. + + This excludes files from the current build. + """ + + if not DEDConfig.autosync_enabled(): + log.info( + 'Autosync disabled, skipping removal from the search index for: %s:%s', + version.project.slug, + version.slug, + ) + return + + try: + document = list(registry.get_documents(models=[model]))[0] + log.info( + 'Deleting old files from search index for: %s:%s', + version.project.slug, + version.slug, + ) + ( + document().search() + .filter('term', project=version.project.slug) + .filter('term', version=version.slug) + .exclude('term', build=build) + .delete() + ) + except Exception: + log.exception('Unable to delete a subset of files. Continuing.') + + # TODO: Rewrite all the views using this in Class Based View, # and move this function to a mixin def get_project_list_or_404(project_slug, user, version_slug=None): From 120eb5d38fb9ae456bd15637b1b45c471bdcd5e0 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 17:35:34 +0600 Subject: [PATCH 043/171] removed unused import --- readthedocs/core/views/serve.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index d82e8b60443..f03f958b172 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,7 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve -from readthedocs.builds.constants import LATEST, STABLE, INTERNAL +from readthedocs.builds.constants import LATEST, STABLE from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path @@ -427,16 +427,10 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): -<<<<<<< HEAD translation_versions = ( Version.internal.public(project=translation) .values_list('slug', flat=True) ) -======= - translation_versions = translation.versions( - manager=INTERNAL - ).public().values_list('slug', flat=True) ->>>>>>> Manager names moved to Constants if version.slug in translation_versions: href = project.get_docs_url( version_slug=version.slug, From ef3d06171f1f693e39dd5e51b48f678b3d7517a1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:30:26 +0600 Subject: [PATCH 044/171] External version name added everywhere --- readthedocs/builds/constants.py | 17 ++++++++--------- readthedocs/builds/managers.py | 10 +++++----- .../0008_added_pull_request_version_type.py | 6 +++--- readthedocs/builds/models.py | 14 +++++++------- readthedocs/rtd_tests/tests/test_doc_serving.py | 8 ++++++-- readthedocs/rtd_tests/tests/test_managers.py | 16 ++++++++-------- readthedocs/rtd_tests/tests/test_project.py | 8 ++++---- .../rtd_tests/tests/test_project_forms.py | 4 ++-- .../rtd_tests/tests/test_project_views.py | 4 ++-- readthedocs/rtd_tests/tests/test_version.py | 6 +++--- 10 files changed, 48 insertions(+), 45 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 37474eb5365..bdc9a70fe79 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -31,15 +31,21 @@ ('dash', _('Dash')), ) +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding pull request/merge request Versions and Builds. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only pull request/merge request Versions and Builds. +EXTERNAL = 'external' + BRANCH = 'branch' TAG = 'tag' -PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), - (PULL_REQUEST, _('Pull Request')), + (EXTERNAL, _('External')), (UNKNOWN, _('Unknown')), ) @@ -55,10 +61,3 @@ LATEST, STABLE, ) - -# Manager name for Internal Versions or Builds. -# ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' -# Manager name for External Versions or Builds. -# ie: Only PULL_REQUEST Type Versions and Builds. -EXTERNAL = 'external' diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 0bc30f02807..06262ed3e24 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,7 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, - PULL_REQUEST, + EXTERNAL, ) from .querysets import VersionQuerySet @@ -91,12 +91,12 @@ class InternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes internal version. - It will exclude PULL_REQUEST type from the queries + It will exclude pull request/merge request versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ def get_queryset(self): - return super().get_queryset().exclude(type=PULL_REQUEST) + return super().get_queryset().exclude(type=EXTERNAL) class ExternalVersionManagerBase(VersionManagerBase): @@ -104,11 +104,11 @@ class ExternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes external version. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull request/merge request Versions in the queries. """ def get_queryset(self): - return super().get_queryset().filter(type=PULL_REQUEST) + return super().get_queryset().filter(type=EXTERNAL) class VersionManager(SettingsOverrideObject): diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 75a126d37b6..2bf9dc62d66 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-06-06 16:51 +# Generated by Django 1.11.21 on 2019-06-17 19:26 from __future__ import unicode_literals from django.db import migrations, models @@ -15,11 +15,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='version', name='type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), ), migrations.AlterField( model_name='versionautomationrule', name='version_type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), ), ] diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index d0c5f6a854e..34a3fd372a9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -41,7 +41,7 @@ INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, - PULL_REQUEST, + EXTERNAL, STABLE, TAG, VERSION_TYPES, @@ -119,7 +119,7 @@ class Version(models.Model): objects = VersionManager.from_queryset(VersionQuerySet)() # Only include BRANCH, TAG, UNKONWN type Versions. internal = InternalVersionManager.from_queryset(VersionQuerySet)() - # Only include PULL_REQUEST type Versions. + # Only include EXTERNAL type Versions. external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: @@ -155,7 +155,7 @@ def vcs_url(self): Generate VCS (github, gitlab, bitbucket) URL for this version. Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. - Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. + Pull/merge Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -165,7 +165,7 @@ def vcs_url(self): else: slug_url = self.slug - if self.type == PULL_REQUEST: + if self.type == EXTERNAL: if 'github' in self.project.repo: url = f'/pull/{slug_url}/' @@ -245,14 +245,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - if self.type == PULL_REQUEST: - # If this version is a Pull Request, the identifier will + if self.type == EXTERNAL: + # If this version is a EXTERNAL version, the identifier will # contain the actual commit hash. which we can use to # generate url for a given file name return self.identifier # If we came that far it's not a special version - # nor a branch, tag or Pull Request. + # nor a branch, tag or EXTERNAL version. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 05d1308f4c1..5bfbbf13f4c 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -10,11 +10,15 @@ from django.urls import reverse from mock import mock_open, patch +<<<<<<< HEAD <<<<<<< HEAD from readthedocs.builds.constants import LATEST ======= from readthedocs.builds.constants import PULL_REQUEST, INTERNAL >>>>>>> Tests added +======= +from readthedocs.builds.constants import LATEST, EXTERNAL, INTERNAL +>>>>>>> External version name added everywhere from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -253,7 +257,7 @@ def test_sitemap_xml(self): project=self.public, active=True ) - # This is a Pull Request Version + # This is a EXTERNAL Version pr_version = fixture.get( Version, identifier='pr-version', @@ -261,7 +265,7 @@ def test_sitemap_xml(self): slug='pr-9999', project=self.public, active=True, - type=PULL_REQUEST + type=EXTERNAL ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index f830c9af549..924544a8be8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -2,7 +2,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED from readthedocs.projects.models import Project @@ -19,29 +19,29 @@ def setUp(self): self.user = User.objects.create(username='test_user', password='test') self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.public_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC ) self.private_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PRIVATE ) self.protected_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PROTECTED ) - self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + self.internal_versions = Version.objects.exclude(type=EXTERNAL) class TestInternalVersionManager(TestVersionManagerBase): @@ -49,7 +49,7 @@ class TestInternalVersionManager(TestVersionManagerBase): """ Queries using Internal Manager should only include Internal Versions. - It will exclude PULL_REQUEST type Versions from the queries + It will exclude EXTERNAL type Versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ @@ -83,7 +83,7 @@ class TestExternalVersionManager(TestVersionManagerBase): """ Queries using External Manager should only include External Versions. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull/merge request Version in the queries. """ def test_external_version_manager_with_all(self): diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index c2db080f201..5d5194d04d2 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,7 +14,7 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, - PULL_REQUEST, + EXTERNAL, ) from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError @@ -30,7 +30,7 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='pr-version', @@ -38,7 +38,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) @@ -156,7 +156,7 @@ def test_all_active_versions_excludes_pr_versions(self): def test_update_stable_version_excludes_pr_versions(self): # Delete all versions excluding PR Versions. - self.pip.versions.exclude(type=PULL_REQUEST).delete() + self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index b2e8687208a..ba5b3c88b9d 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST +from readthedocs.builds.constants import LATEST, STABLE, EXTERNAL from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -391,7 +391,7 @@ def test_pr_version_not_in_default_branch_choices(self): slug='pr-9999', project=self.project, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC, ) form = ProjectAdvancedForm(instance=self.project) diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 90578c8dceb..3bc948aff9b 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST, PULL_REQUEST +from readthedocs.builds.constants import LATEST, EXTERNAL from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -377,7 +377,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) def test_project_download_media(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py index 8a6f5bf3db8..e414ba79cc6 100644 --- a/readthedocs/rtd_tests/tests/test_version.py +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -1,7 +1,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.models import Project @@ -13,7 +13,7 @@ class VersionMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='9F86D081884C7D659A2FEAA0C55AD015A', @@ -21,7 +21,7 @@ def setUp(self): slug='9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) self.branch_version = get( Version, From 5b33bbd71a257fdce5ff49897cb8845e74707df6 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:43:48 +0600 Subject: [PATCH 045/171] Migration name changed --- ...uest_version_type.py => 0008_added_external_version_type.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename readthedocs/builds/migrations/{0008_added_pull_request_version_type.py => 0008_added_external_version_type.py} (94%) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_external_version_type.py similarity index 94% rename from readthedocs/builds/migrations/0008_added_pull_request_version_type.py rename to readthedocs/builds/migrations/0008_added_external_version_type.py index 2bf9dc62d66..8de9c4f504f 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_external_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.21 on 2019-06-17 19:26 +# Generated by Django 1.11.21 on 2019-06-17 19:43 from __future__ import unicode_literals from django.db import migrations, models From 518074335bcc722d4342c8d6ac9d34a68d215b1b Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 8 Jun 2019 21:10:38 +0600 Subject: [PATCH 046/171] Build Managers added --- readthedocs/builds/managers.py | 60 ++++++++++++++++++++++++++++++++- readthedocs/builds/models.py | 26 +++++++++----- readthedocs/builds/querysets.py | 1 - 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 06262ed3e24..78e274b5008 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -21,7 +21,7 @@ TAG, EXTERNAL, ) -from .querysets import VersionQuerySet +from .querysets import VersionQuerySet, BuildQuerySet log = logging.getLogger(__name__) @@ -122,3 +122,61 @@ class InternalVersionManager(SettingsOverrideObject): class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase + + +class BuildManagerBase(models.Manager): + + """ + Build manager for manager only queries. + + For creating different Managers. + """ + + @classmethod + def from_queryset(cls, queryset_class, class_name=None): + # This is overridden because :py:meth:`models.Manager.from_queryset` + # uses `inspect` to retrieve the class methods, and the proxy class has + # no direct members. + queryset_class = get_override_class( + BuildQuerySet, + BuildQuerySet._default_class, # pylint: disable=protected-access + ) + return super().from_queryset(queryset_class, class_name) + + +class InternalBuildManagerBase(BuildManagerBase): + + """ + Build manager that only includes internal version builds. + + It will exclude PULL_REQUEST type Version builds from the queries + and only include BRANCH, TAG, UNKONWN type Version builds. + """ + + def get_queryset(self): + return super().get_queryset().exclude(version__type=PULL_REQUEST) + + +class ExternalBuildManagerBase(BuildManagerBase): + + """ + Build manager that only includes external version builds. + + It will only include PULL_REQUEST type Versions builds in the queries. + """ + + def get_queryset(self): + return super().get_queryset().filter(version__type=PULL_REQUEST) + + +class BuildManager(SettingsOverrideObject): + _default_class = BuildManagerBase + _override_setting = 'BUILD_MANAGER' + + +class InternalBuildManager(SettingsOverrideObject): + _default_class = InternalBuildManagerBase + + +class ExternalBuildManager(SettingsOverrideObject): + _default_class = ExternalBuildManagerBase diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 34a3fd372a9..ba176acfb65 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -46,18 +46,25 @@ TAG, VERSION_TYPES, ) -from .managers import ( +from readthedocs.builds.managers import ( VersionManager, InternalVersionManager, - ExternalVersionManager + ExternalVersionManager, + BuildManager, + InternalBuildManager, + ExternalBuildManager, ) -from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet -from .utils import ( +from readthedocs.builds.querysets import ( + BuildQuerySet, + RelatedBuildQuerySet, + VersionQuerySet, +) +from readthedocs.builds.utils import ( get_bitbucket_username_repo, get_github_username_repo, get_gitlab_username_repo, ) -from .version_slug import VersionSlugField +from readthedocs.builds.version_slug import VersionSlugField log = logging.getLogger(__name__) @@ -627,9 +634,12 @@ class Build(models.Model): help_text='Build steps stored outside the database.', ) - # Manager - - objects = BuildQuerySet.as_manager() + # Managers + objects = BuildManager.from_queryset(BuildQuerySet)() + # Only include BRANCH, TAG, UNKONWN type Version builds. + internal = InternalBuildManager.from_queryset(BuildQuerySet)() + # Only include PULL_REQUEST type Version builds. + external = ExternalBuildManager.from_queryset(BuildQuerySet)() CONFIG_KEY = '__config' diff --git a/readthedocs/builds/querysets.py b/readthedocs/builds/querysets.py index 6b4c9132b1a..d6e4c1bda2c 100644 --- a/readthedocs/builds/querysets.py +++ b/readthedocs/builds/querysets.py @@ -116,7 +116,6 @@ def api(self, user=None, detail=True): class BuildQuerySet(SettingsOverrideObject): _default_class = BuildQuerySetBase - _override_setting = 'BUILD_MANAGER' class RelatedBuildQuerySetBase(models.QuerySet): From 8301679c94976297f7a4cb3253a2f91ab9a05cc1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 22:59:45 +0600 Subject: [PATCH 047/171] Managers used in all the places --- readthedocs/api/v3/views.py | 2 +- readthedocs/builds/models.py | 4 ++-- readthedocs/builds/views.py | 15 +++++++++++++++ readthedocs/core/templatetags/privacy_tags.py | 2 +- readthedocs/projects/models.py | 2 +- readthedocs/projects/querysets.py | 4 ++-- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 57b86360e96..e3ddb9daa93 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -269,7 +269,7 @@ class BuildsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, lookup_url_kwarg = 'build_pk' serializer_class = BuildSerializer filterset_class = BuildFilter - queryset = Build.objects.all() + queryset = Build.internal.all() permit_list_expands = [ 'config', ] diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index ba176acfb65..c77681c50a0 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -207,7 +207,7 @@ def config(self): :rtype: dict """ last_build = ( - self.builds.filter( + self.builds(manager=INTERNAL).filter( state='finished', success=True, ).order_by('-date').first() @@ -662,7 +662,7 @@ def previous(self): date = self.date or timezone.now() if self.project is not None and self.version is not None: return ( - Build.objects.filter( + Build.internal.filter( project=self.project, version=self.version, date__lt=date, diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index 3333495f6c8..479d0797458 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -84,6 +84,21 @@ def post(self, request, project_slug): class BuildList(BuildBase, BuildTriggerMixin, ListView): + def get_queryset(self): + # this is used to include only internal version + # builds in the build list page + self.project_slug = self.kwargs.get('project_slug', None) + self.project = get_object_or_404( + Project.objects.protected(self.request.user), + slug=self.project_slug, + ) + queryset = Build.internal.public( + user=self.request.user, + project=self.project, + ).select_related('project', 'version') + + return queryset + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) diff --git a/readthedocs/core/templatetags/privacy_tags.py b/readthedocs/core/templatetags/privacy_tags.py index 115bb9eadd2..e4ad684a8a0 100644 --- a/readthedocs/core/templatetags/privacy_tags.py +++ b/readthedocs/core/templatetags/privacy_tags.py @@ -26,7 +26,7 @@ def get_public_projects(context, user): viewer=context['request'].user, ).prefetch_latest_build().annotate( _good_build=Exists( - Build.objects.filter(success=True, project=OuterRef('pk'))) + Build.internal.filter(success=True, project=OuterRef('pk'))) ) context['public_projects'] = projects return '' diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index a3f36feb5c8..2e782c7296a 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -768,7 +768,7 @@ def has_good_build(self): # Used for Database optimization. if hasattr(self, '_good_build'): return self._good_build - return self.builds.filter(success=True).exists() + return self.builds(manager=INTERNAL).filter(success=True).exists() @property def has_versions(self): diff --git a/readthedocs/projects/querysets.py b/readthedocs/projects/querysets.py index 7ea7249ae0d..ff4221fd590 100644 --- a/readthedocs/projects/querysets.py +++ b/readthedocs/projects/querysets.py @@ -85,13 +85,13 @@ def prefetch_latest_build(self): # Prefetch the latest build for each project. subquery = Subquery( - Build.objects.filter( + Build.internal.filter( project=OuterRef('project_id') ).order_by('-date').values_list('id', flat=True)[:1] ) latest_build = Prefetch( 'builds', - Build.objects.filter(pk__in=subquery), + Build.internal.filter(pk__in=subquery), to_attr=self.model.LATEST_BUILD_CACHE, ) return self.prefetch_related(latest_build) From 7b75ee817a8c525f8ee3f0507134a0b370dedfc4 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 05:00:06 +0600 Subject: [PATCH 048/171] build manager tests added --- readthedocs/rtd_tests/tests/test_managers.py | 90 +++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 924544a8be8..0d649dfac1e 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -3,7 +3,7 @@ from django_dynamic_fixture import get from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG -from readthedocs.builds.models import Version +from readthedocs.builds.models import Version, Build from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED from readthedocs.projects.models import Project @@ -123,3 +123,91 @@ def test_external_version_manager_with_for_project(self): self.assertIn( self.public_pr_version, Version.external.for_project(self.pip) ) + + +class TestBuildManagerBase(TestCase): + + fixtures = ['test_data'] + + def setUp(self): + self.user = User.objects.create(username='test_user', password='test') + self.client.login(username='test_user', password='test') + self.pip = Project.objects.get(slug='pip') + print(self.pip.versions.all()) + # Create a External Version and build. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC + ) + self.pr_version_build = get( + Build, + project=self.pip, + version=self.pr_version + ) + # Create a Internal Version build. + self.internal_version_build = get( + Build, + project=self.pip, + version=self.pip.versions.get(slug='0.8') + ) + + self.internal_builds = Build.objects.exclude(version__type=PULL_REQUEST) + + +class TestInternalBuildManager(TestBuildManagerBase): + + """ + Queries using Internal Manager should only include Internal Version builds. + + It will exclude PULL_REQUEST type Version builds from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + + def test_internal_build_manager_with_all(self): + self.assertNotIn(self.pr_version_build, Build.internal.all()) + + def test_internal_build_manager_with_public(self): + self.assertNotIn(self.pr_version_build, Build.internal.public()) + + def test_internal_build_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.pr_version_build, + Build.internal.public(self.user, self.pip) + ) + + def test_internal_build_manager_with_api(self): + self.assertNotIn(self.pr_version_build, Build.internal.api()) + + +class TestExternalBuildManager(TestBuildManagerBase): + + """ + Queries using External Manager should only include External Version builds. + + It will only include PULL_REQUEST type Version builds in the queries. + """ + + def test_external_build_manager_with_all(self): + self.assertNotIn(self.internal_builds, Build.external.all()) + self.assertIn(self.pr_version_build, Build.external.all()) + + def test_external_build_manager_with_public(self): + self.assertNotIn(self.internal_builds, Build.external.public()) + self.assertIn(self.pr_version_build, Build.external.public()) + + def test_external_build_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.internal_builds, + Build.external.public(self.user, self.pip) + ) + self.assertIn( + self.pr_version_build, + Build.external.public(self.user, self.pip) + ) + + def test_external_build_manager_with_api(self): + self.assertNotIn(self.internal_builds, Build.external.api()) + self.assertIn(self.pr_version_build, Build.external.api()) From 8c7b02b2c15cfb71dee0b4c7b926de6db9b18eba Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 05:06:12 +0600 Subject: [PATCH 049/171] More Tests added --- readthedocs/builds/models.py | 2 +- readthedocs/rtd_tests/tests/test_project.py | 6 +++++ readthedocs/rtd_tests/tests/test_views.py | 25 +++++++++++++++++++-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index c77681c50a0..7e7e827f91e 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -662,7 +662,7 @@ def previous(self): date = self.date or timezone.now() if self.project is not None and self.version is not None: return ( - Build.internal.filter( + Build.objects.filter( project=self.project, version=self.version, date__lt=date, diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 5d5194d04d2..d9cb1cdca15 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -160,6 +160,12 @@ def test_update_stable_version_excludes_pr_versions(self): # Test that PR Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) + def test_has_good_build_excludes_pr_versions(self): + # Delete all versions excluding PR Versions. + self.pip.versions.exclude(type=PULL_REQUEST).delete() + # Test that PR Version is not considered for has_good_build. + self.assertFalse(self.pip.has_good_build) + class TestProjectTranslations(ProjectMixin, TestCase): diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index 25e7b60a756..bfbf07ce243 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -7,8 +7,8 @@ from django.urls import reverse from django_dynamic_fixture import get, new -from readthedocs.builds.constants import LATEST -from readthedocs.builds.models import Build +from readthedocs.builds.constants import LATEST, PULL_REQUEST +from readthedocs.builds.models import Build, Version from readthedocs.core.permissions import AdminPermission from readthedocs.projects.forms import UpdateProjectForm from readthedocs.projects.models import HTMLFile, Project @@ -266,6 +266,7 @@ class BuildViewTests(TestCase): def setUp(self): self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') @mock.patch('readthedocs.projects.tasks.update_docs_task') def test_build_redirect(self, mock): @@ -276,3 +277,23 @@ def test_build_redirect(self, mock): r._headers['location'][1], '/projects/pip/builds/%s/' % build.pk, ) + + def test_build_list_does_not_include_pr_versions(self): + pr_version = get( + Version, + project = self.pip, + active = True, + type = PULL_REQUEST, + ) + pr_version_build = get( + Build, + project = self.pip, + version = pr_version + ) + response = self.client.get( + reverse('builds_project_list', args=[self.pip.slug]), + ) + self.assertEqual(response.status_code, 200) + + self.assertNotIn(pr_version_build, response.context['build_qs']) + self.assertNotIn(pr_version_build, response.context['active_builds']) From 5d6c750fc2580c5e4b73755e1abbd5c4bb17e61f Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:55:10 +0600 Subject: [PATCH 050/171] External version name added --- readthedocs/builds/managers.py | 8 ++++---- readthedocs/builds/models.py | 2 +- readthedocs/rtd_tests/tests/test_managers.py | 10 +++++----- readthedocs/rtd_tests/tests/test_project.py | 2 +- readthedocs/rtd_tests/tests/test_views.py | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 78e274b5008..d80852478f2 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -149,12 +149,12 @@ class InternalBuildManagerBase(BuildManagerBase): """ Build manager that only includes internal version builds. - It will exclude PULL_REQUEST type Version builds from the queries + It will exclude pull request/merge request version builds from the queries and only include BRANCH, TAG, UNKONWN type Version builds. """ def get_queryset(self): - return super().get_queryset().exclude(version__type=PULL_REQUEST) + return super().get_queryset().exclude(version__type=EXTERNAL) class ExternalBuildManagerBase(BuildManagerBase): @@ -162,11 +162,11 @@ class ExternalBuildManagerBase(BuildManagerBase): """ Build manager that only includes external version builds. - It will only include PULL_REQUEST type Versions builds in the queries. + It will only include pull request/merge request version builds in the queries. """ def get_queryset(self): - return super().get_queryset().filter(version__type=PULL_REQUEST) + return super().get_queryset().filter(version__type=EXTERNAL) class BuildManager(SettingsOverrideObject): diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 7e7e827f91e..75abdcfda24 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -638,7 +638,7 @@ class Build(models.Model): objects = BuildManager.from_queryset(BuildQuerySet)() # Only include BRANCH, TAG, UNKONWN type Version builds. internal = InternalBuildManager.from_queryset(BuildQuerySet)() - # Only include PULL_REQUEST type Version builds. + # Only include EXTERNAL type Version builds. external = ExternalBuildManager.from_queryset(BuildQuerySet)() CONFIG_KEY = '__config' diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 0d649dfac1e..50aca0f8150 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -134,12 +134,12 @@ def setUp(self): self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') print(self.pip.versions.all()) - # Create a External Version and build. ie: PULL_REQUEST type Version. + # Create a External Version and build. ie: pull/merge request Version. self.pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC ) self.pr_version_build = get( @@ -154,7 +154,7 @@ def setUp(self): version=self.pip.versions.get(slug='0.8') ) - self.internal_builds = Build.objects.exclude(version__type=PULL_REQUEST) + self.internal_builds = Build.objects.exclude(version__type=EXTERNAL) class TestInternalBuildManager(TestBuildManagerBase): @@ -162,7 +162,7 @@ class TestInternalBuildManager(TestBuildManagerBase): """ Queries using Internal Manager should only include Internal Version builds. - It will exclude PULL_REQUEST type Version builds from the queries + It will exclude pull/merge request Version builds from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ @@ -187,7 +187,7 @@ class TestExternalBuildManager(TestBuildManagerBase): """ Queries using External Manager should only include External Version builds. - It will only include PULL_REQUEST type Version builds in the queries. + It will only include pull/merge request Version builds in the queries. """ def test_external_build_manager_with_all(self): diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index d9cb1cdca15..af5fb5d508a 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -162,7 +162,7 @@ def test_update_stable_version_excludes_pr_versions(self): def test_has_good_build_excludes_pr_versions(self): # Delete all versions excluding PR Versions. - self.pip.versions.exclude(type=PULL_REQUEST).delete() + self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for has_good_build. self.assertFalse(self.pip.has_good_build) diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index bfbf07ce243..71d69ab2e6b 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -7,7 +7,7 @@ from django.urls import reverse from django_dynamic_fixture import get, new -from readthedocs.builds.constants import LATEST, PULL_REQUEST +from readthedocs.builds.constants import LATEST, EXTERNAL from readthedocs.builds.models import Build, Version from readthedocs.core.permissions import AdminPermission from readthedocs.projects.forms import UpdateProjectForm @@ -283,7 +283,7 @@ def test_build_list_does_not_include_pr_versions(self): Version, project = self.pip, active = True, - type = PULL_REQUEST, + type = EXTERNAL, ) pr_version_build = get( Build, From 53290368d15766f2b9d95e100731a0df9a06900e Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 23 May 2019 18:15:08 +0600 Subject: [PATCH 051/171] Sitemap sort order priorities updated --- readthedocs/core/views/serve.py | 4 +-- readthedocs/projects/version_handling.py | 10 +++--- .../rtd_tests/tests/test_doc_serving.py | 33 ++++++++++++++++++- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index cd5ab8fdb64..b7687d0a132 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -373,14 +373,14 @@ def changefreqs_generator(): """ Generator returning ``changefreq`` needed by sitemap.xml. - It returns ``daily`` on first iteration, then ``weekly`` and then it + It returns ``weekly`` on first iteration, then ``daily`` and then it will return always ``monthly``. We are using ``monthly`` as last value because ``never`` is too aggressive. If the tag is removed and a branch is created with the same name, we will want bots to revisit this. """ - changefreqs = ['daily', 'weekly'] + changefreqs = ['weekly', 'daily'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) if project.privacy_level == constants.PRIVATE: diff --git a/readthedocs/projects/version_handling.py b/readthedocs/projects/version_handling.py index 7a730e61fd0..093bb705c68 100644 --- a/readthedocs/projects/version_handling.py +++ b/readthedocs/projects/version_handling.py @@ -43,23 +43,23 @@ def comparable_version(version_string): """ Can be used as ``key`` argument to ``sorted``. - The ``LATEST`` version shall always beat other versions in comparison. - ``STABLE`` should be listed second. If we cannot figure out the version + The ``STABLE`` version shall always beat other versions in comparison. + ``LATEST`` should be listed second. If we cannot figure out the version number then we sort it to the bottom of the list. :param version_string: version as string object (e.g. '3.10.1' or 'latest') :type version_string: str or unicode - :returns: a comparable version object (e.g. 'latest' -> Version('99999.0')) + :returns: a comparable version object (e.g. 'latest' -> Version('9999.0')) :rtype: packaging.version.Version """ comparable = parse_version_failsafe(version_string) if not comparable: if version_string == LATEST_VERBOSE_NAME: - comparable = Version('99999.0') - elif version_string == STABLE_VERBOSE_NAME: comparable = Version('9999.0') + elif version_string == STABLE_VERBOSE_NAME: + comparable = Version('99999.0') else: comparable = Version('0.01') return comparable diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 4f6cb93990c..fed119bbab5 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -240,6 +240,15 @@ def test_sitemap_xml(self): project=self.public, active=True ) + stable_version = fixture.get( + Version, + identifier='stable', + verbose_name='stable', + slug='stable', + privacy_level=constants.PUBLIC, + project=self.public, + active=True + ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( Project, @@ -269,7 +278,7 @@ def test_sitemap_xml(self): ), ) - # stable is marked as PRIVATE and should not appear here + # PRIVATE version should not appear here self.assertNotContains( response, self.public.get_docs_url( @@ -294,6 +303,28 @@ def test_sitemap_xml(self): # in language and country value. (zh_CN should be zh-CN) self.assertContains(response, 'zh-CN') + # Check if STABLE version has 'priority of 1 and changefreq of weekly. + self.assertEqual( + response.context['versions'][0]['loc'], + self.public.get_docs_url( + version_slug=stable_version.slug, + lang_slug=self.public.language, + private=False, + ),) + self.assertEqual(response.context['versions'][0]['priority'], 1) + self.assertEqual(response.context['versions'][0]['changefreq'], 'weekly') + + # Check if LATEST version has priority of 0.9 and changefreq of daily. + self.assertEqual( + response.context['versions'][1]['loc'], + self.public.get_docs_url( + version_slug='latest', + lang_slug=self.public.language, + private=False, + ),) + self.assertEqual(response.context['versions'][1]['priority'], 0.9) + self.assertEqual(response.context['versions'][1]['changefreq'], 'daily') + @override_settings( PYTHON_MEDIA=True, USE_SUBDOMAIN=False, From d64e2d0e672d3eef0784b6a9b12212547249a846 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 25 May 2019 17:33:14 +0600 Subject: [PATCH 052/171] ordering fix --- readthedocs/core/views/serve.py | 4 ++-- readthedocs/projects/version_handling.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index b7687d0a132..f7ab665bbc5 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -355,7 +355,7 @@ def priorities_generator(): It generates values from 1 to 0.1 by decreasing in 0.1 on each iteration. After 0.1 is reached, it will keep returning 0.1. """ - priorities = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] + priorities = [0.9, 1, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] yield from itertools.chain(priorities, itertools.repeat(0.1)) def hreflang_formatter(lang): @@ -380,7 +380,7 @@ def changefreqs_generator(): aggressive. If the tag is removed and a branch is created with the same name, we will want bots to revisit this. """ - changefreqs = ['weekly', 'daily'] + changefreqs = ['daily', 'weekly'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) if project.privacy_level == constants.PRIVATE: diff --git a/readthedocs/projects/version_handling.py b/readthedocs/projects/version_handling.py index 093bb705c68..7a730e61fd0 100644 --- a/readthedocs/projects/version_handling.py +++ b/readthedocs/projects/version_handling.py @@ -43,23 +43,23 @@ def comparable_version(version_string): """ Can be used as ``key`` argument to ``sorted``. - The ``STABLE`` version shall always beat other versions in comparison. - ``LATEST`` should be listed second. If we cannot figure out the version + The ``LATEST`` version shall always beat other versions in comparison. + ``STABLE`` should be listed second. If we cannot figure out the version number then we sort it to the bottom of the list. :param version_string: version as string object (e.g. '3.10.1' or 'latest') :type version_string: str or unicode - :returns: a comparable version object (e.g. 'latest' -> Version('9999.0')) + :returns: a comparable version object (e.g. 'latest' -> Version('99999.0')) :rtype: packaging.version.Version """ comparable = parse_version_failsafe(version_string) if not comparable: if version_string == LATEST_VERBOSE_NAME: - comparable = Version('9999.0') - elif version_string == STABLE_VERBOSE_NAME: comparable = Version('99999.0') + elif version_string == STABLE_VERBOSE_NAME: + comparable = Version('9999.0') else: comparable = Version('0.01') return comparable From 51ad395d4c8b041193b474fa4cd2fe944747aa69 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 25 May 2019 17:34:46 +0600 Subject: [PATCH 053/171] doc string fix --- readthedocs/core/views/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index f7ab665bbc5..49207c0b5bd 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -373,7 +373,7 @@ def changefreqs_generator(): """ Generator returning ``changefreq`` needed by sitemap.xml. - It returns ``weekly`` on first iteration, then ``daily`` and then it + It returns ``daily`` on first iteration, then ``weekly`` and then it will return always ``monthly``. We are using ``monthly`` as last value because ``never`` is too From 884857bea9d08ed022eaf40a160154676e251587 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 29 May 2019 17:04:04 +0600 Subject: [PATCH 054/171] sort_by_priority function added to sitemap --- readthedocs/core/views/serve.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 49207c0b5bd..80db933ab69 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -383,6 +383,14 @@ def changefreqs_generator(): changefreqs = ['daily', 'weekly'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) + def sort_by_priority(version_list): + """This will sort the versions by priority""" + return sorted( + version_list, + key=lambda version: version['priority'], + reverse=True + ) + if project.privacy_level == constants.PRIVATE: raise Http404 @@ -437,7 +445,7 @@ def changefreqs_generator(): versions.append(element) context = { - 'versions': versions, + 'versions': sort_by_priority(versions), } return render( request, From 43b24d79b05029bb711c9061568b9146330903e0 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 29 May 2019 17:14:19 +0600 Subject: [PATCH 055/171] lint fix --- readthedocs/core/views/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 80db933ab69..9a0dd3593cc 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -384,7 +384,7 @@ def changefreqs_generator(): yield from itertools.chain(changefreqs, itertools.repeat('monthly')) def sort_by_priority(version_list): - """This will sort the versions by priority""" + """Sorts the versions by priority. i.e: 1, 0.9, 0.8...""" return sorted( version_list, key=lambda version: version['priority'], From 3e278a29936c2c66568e695d796caeba1d53e3be Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 00:33:08 +0600 Subject: [PATCH 056/171] Sorting by swapping positions --- readthedocs/core/views/serve.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 9a0dd3593cc..3d1a971cfd4 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,6 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve +from readthedocs.builds.constants import LATEST, STABLE from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path @@ -355,7 +356,7 @@ def priorities_generator(): It generates values from 1 to 0.1 by decreasing in 0.1 on each iteration. After 0.1 is reached, it will keep returning 0.1. """ - priorities = [0.9, 1, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] + priorities = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] yield from itertools.chain(priorities, itertools.repeat(0.1)) def hreflang_formatter(lang): @@ -373,24 +374,16 @@ def changefreqs_generator(): """ Generator returning ``changefreq`` needed by sitemap.xml. - It returns ``daily`` on first iteration, then ``weekly`` and then it + It returns ``weekly`` on first iteration, then ``daily`` and then it will return always ``monthly``. We are using ``monthly`` as last value because ``never`` is too aggressive. If the tag is removed and a branch is created with the same name, we will want bots to revisit this. """ - changefreqs = ['daily', 'weekly'] + changefreqs = ['weekly', 'daily'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) - def sort_by_priority(version_list): - """Sorts the versions by priority. i.e: 1, 0.9, 0.8...""" - return sorted( - version_list, - key=lambda version: version['priority'], - reverse=True - ) - if project.privacy_level == constants.PRIVATE: raise Http404 @@ -400,6 +393,18 @@ def sort_by_priority(version_list): only_active=True, ), ) + + # This is a hack to swap the latest version with + # stable version to get the stable version first in the sitemap. + # We want stable with priority=1 and changefreq='weekly' and + # latest with priority=0.9 and changefreq='daily' + # More details on this: https://github.com/rtfd/readthedocs.org/issues/5447 + if ( + sorted_versions[0].slug == LATEST and + sorted_versions[1].slug == STABLE + ): + sorted_versions[0], sorted_versions[1] = sorted_versions[1], sorted_versions[0] + versions = [] for version, priority, changefreq in zip( sorted_versions, @@ -445,7 +450,7 @@ def sort_by_priority(version_list): versions.append(element) context = { - 'versions': sort_by_priority(versions), + 'versions': versions, } return render( request, From 7d3f934c0d8957cee08b3ac044ae38c157b28a2b Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 17:30:50 +0600 Subject: [PATCH 057/171] index out of range issue fix --- readthedocs/core/views/serve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 3d1a971cfd4..237b843e0fb 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -400,6 +400,7 @@ def changefreqs_generator(): # latest with priority=0.9 and changefreq='daily' # More details on this: https://github.com/rtfd/readthedocs.org/issues/5447 if ( + len(sorted_versions) >= 2 and sorted_versions[0].slug == LATEST and sorted_versions[1].slug == STABLE ): From 4194a42e7eeebd9af04245315b0c5b6344ebf0f4 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 12 Jun 2019 21:55:35 +0200 Subject: [PATCH 058/171] Use a real SessionBase object on FooterNoSessionMiddleware This allows all Django internals method to keep working in the same way but still using an empty session. --- readthedocs/core/middleware.py | 3 ++- readthedocs/rtd_tests/tests/test_footer.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 60cf4a277d3..692e45a2f8d 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -1,6 +1,7 @@ import logging from django.conf import settings +from django.contrib.sessions.backends.base import SessionBase from django.contrib.sessions.middleware import SessionMiddleware from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.http import Http404, HttpResponseBadRequest @@ -205,7 +206,7 @@ def process_request(self, request): settings.SESSION_COOKIE_NAME not in request.COOKIES ): # Hack request.session otherwise the Authentication middleware complains. - request.session = {} + request.session = SessionBase() # create an empty session return super().process_request(request) diff --git a/readthedocs/rtd_tests/tests/test_footer.py b/readthedocs/rtd_tests/tests/test_footer.py index 34c1fec1f92..241bc434f59 100644 --- a/readthedocs/rtd_tests/tests/test_footer.py +++ b/readthedocs/rtd_tests/tests/test_footer.py @@ -1,4 +1,5 @@ import mock +from django.contrib.sessions.backends.base import SessionBase from django.test import TestCase from rest_framework.test import APIRequestFactory, APITestCase @@ -80,7 +81,8 @@ def test_no_session_logged_out(self): # Null session here request = self.factory.get('/api/v2/footer_html/') mid.process_request(request) - self.assertEqual(request.session, {}) + self.assertIsInstance(request.session, SessionBase) + self.assertEqual(list(request.session.keys()), []) # Proper session here home_request = self.factory.get('/') From 5572dee67e80b47a2ec18675e7fae39e2d4f0172 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 16:59:23 +0600 Subject: [PATCH 059/171] Version Type Added --- readthedocs/builds/constants.py | 2 ++ .../0008_added_pull_request_version_type.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 readthedocs/builds/migrations/0008_added_pull_request_version_type.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index b0dc6cfba94..e2e4fc44ac3 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -33,11 +33,13 @@ BRANCH = 'branch' TAG = 'tag' +PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), + (PULL_REQUEST, _('Pull Request')), (UNKNOWN, _('Unknown')), ) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py new file mode 100644 index 00000000000..14d8968866d --- /dev/null +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-05-30 10:02 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('builds', '0007_add-automation-rules'), + ] + + operations = [ + migrations.AlterField( + model_name='version', + name='type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + ), + migrations.AlterField( + model_name='versionautomationrule', + name='version_type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + ), + ] From 7f90a88c3efd3ccea5fbea04b846e0237340f5f7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 23:52:36 +0600 Subject: [PATCH 060/171] Version Model Methods Updated --- readthedocs/builds/models.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 1039b815326..ca44b39f252 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -40,6 +40,7 @@ BUILD_TYPES, LATEST, NON_REPOSITORY_VERSIONS, + PULL_REQUEST, STABLE, TAG, VERSION_TYPES, @@ -142,7 +143,8 @@ def vcs_url(self): """ Generate VCS (github, gitlab, bitbucket) URL for this version. - Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -152,12 +154,24 @@ def vcs_url(self): else: slug_url = self.slug - if ('github' in self.project.repo) or ('gitlab' in self.project.repo): - url = f'/tree/{slug_url}/' + if self.type == PULL_REQUEST: + if 'github' in self.project.repo: + url = f'/pull/{slug_url}/' - if 'bitbucket' in self.project.repo: - slug_url = self.identifier - url = f'/src/{slug_url}' + if 'gitlab' in self.project.repo: + slug_url = self.identifier + url = f'/merge_requests/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/pull-requests/{slug_url}' + else: + if ('github' in self.project.repo) or ('gitlab' in self.project.repo): + url = f'/tree/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/src/{slug_url}' # TODO: improve this replacing return self.project.repo.replace('git://', 'https://').replace('.git', '') + url @@ -220,7 +234,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - # If we came that far it's not a special version nor a branch or tag. + if self.type == PULL_REQUEST: + # If this version is a Pull Request, the identifier will + # contain the actual commit hash. which we can use to + # generate url for a given file name + return self.identifier + + # If we came that far it's not a special version + # nor a branch, tag or Pull Request. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', From 19e7fa1526f8b092628bc62cf821c240801ca3bc Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 15:40:32 +0600 Subject: [PATCH 061/171] Internal and External Version Manager added --- readthedocs/builds/managers.py | 33 +++++++++++++++++++++++++++++++++ readthedocs/builds/models.py | 10 +++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 68cdec6efe9..991293beae2 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,6 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, + PULL_REQUEST, ) from .querysets import VersionQuerySet @@ -85,6 +86,38 @@ def get_object_or_log(self, **kwargs): log.warning('Version not found for given kwargs. %s' % kwargs) +class InternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes internal version. + + It will exclude PULL_REQUEST type from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + def get_queryset(self): + return super().get_queryset().exclude(type=PULL_REQUEST) + + +class ExternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes external version. + + It will only include PULL_REQUEST type Versions in the queries. + """ + def get_queryset(self): + return super().get_queryset().filter(type=PULL_REQUEST) + + class VersionManager(SettingsOverrideObject): _default_class = VersionManagerBase _override_setting = 'VERSION_MANAGER' + + +class InternalVersionManager(SettingsOverrideObject): + _default_class = InternalVersionManagerBase + _override_setting = 'INTERNAL_VERSION_MANAGER' + + +class ExternalVersionManager(SettingsOverrideObject): + _default_class = ExternalVersionManagerBase + _override_setting = 'EXTERNAL_VERSION_MANAGER' + diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index ca44b39f252..bca375570c9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -45,7 +45,11 @@ TAG, VERSION_TYPES, ) -from .managers import VersionManager +from .managers import ( + VersionManager, + InternalVersionManager, + ExternalVersionManager +) from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet from .utils import ( get_bitbucket_username_repo, @@ -112,6 +116,10 @@ class Version(models.Model): machine = models.BooleanField(_('Machine Created'), default=False) objects = VersionManager.from_queryset(VersionQuerySet)() + # Only include BRANCH, TAG, UNKONWN type Versions. + internal = InternalVersionManager.from_queryset(VersionQuerySet)() + # Only include PULL_REQUEST type Versions. + external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: unique_together = [('project', 'slug')] From bd256fda05906b33cbe4176f613304083cc6c446 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 16:20:19 +0600 Subject: [PATCH 062/171] lint fix --- readthedocs/builds/managers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 991293beae2..2a8bb4840d9 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -87,22 +87,26 @@ def get_object_or_log(self, **kwargs): class InternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes internal version. It will exclude PULL_REQUEST type from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ + def get_queryset(self): return super().get_queryset().exclude(type=PULL_REQUEST) class ExternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes external version. It will only include PULL_REQUEST type Versions in the queries. """ + def get_queryset(self): return super().get_queryset().filter(type=PULL_REQUEST) @@ -120,4 +124,3 @@ class InternalVersionManager(SettingsOverrideObject): class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase _override_setting = 'EXTERNAL_VERSION_MANAGER' - From fef719135ce7015dc389365620342d0366b94090 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 5 Jun 2019 01:18:20 +0600 Subject: [PATCH 063/171] All Version Querysets Updated with InternalVersionManager --- readthedocs/api/v2/views/model_views.py | 2 +- readthedocs/api/v3/views.py | 2 +- readthedocs/builds/models.py | 4 +- readthedocs/builds/views.py | 2 +- .../core/management/commands/update_repos.py | 42 +++++++++++++++++++ .../management/commands/update_versions.py | 2 +- readthedocs/core/views/serve.py | 4 +- readthedocs/projects/admin.py | 4 +- readthedocs/projects/forms.py | 2 +- readthedocs/projects/models.py | 8 ++-- readthedocs/projects/views/public.py | 6 +-- 11 files changed, 61 insertions(+), 17 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index af41f28ab84..5f594a032b8 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions.filter(active=True) + versions = project.versions(manager='internal').filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 0e8b2b55a0e..57b86360e96 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -230,7 +230,7 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, lookup_value_regex = r'[^/]+' filterset_class = VersionFilter - queryset = Version.objects.all() + queryset = Version.internal.all() permit_list_expands = [ 'last_build', 'last_build.config', diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bca375570c9..bcfc5dfff5c 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -142,7 +142,9 @@ def __str__(self): @property def ref(self): if self.slug == STABLE: - stable = determine_stable_version(self.project.versions.all()) + stable = determine_stable_version( + self.project.versions(manager='internal').all() + ) if stable: return stable.slug diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index e8e3d458e2c..3d6e2cc0ca1 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context['project'] = self.project context['active_builds'] = active_builds - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=self.project, ) diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 5852592d360..0e14f9ce339 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -76,6 +76,48 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) + elif version == 'internal': + log.info('Updating all internal versions for %s', slug) + for version in Version.internal.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) + elif version == 'external': + log.info('Updating all external versions for %s', slug) + for version in Version.external.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) else: p = Project.all_objects.get(slug=slug) log.info('Building %s', p) diff --git a/readthedocs/core/management/commands/update_versions.py b/readthedocs/core/management/commands/update_versions.py index b0aa1f877cb..6456accc430 100644 --- a/readthedocs/core/management/commands/update_versions.py +++ b/readthedocs/core/management/commands/update_versions.py @@ -13,7 +13,7 @@ class Command(BaseCommand): help = __doc__ def handle(self, *args, **options): - for version in Version.objects.filter(active=True, built=False): + for version in Version.internal.filter(active=True, built=False): # pylint: disable=no-value-for-parameter update_docs_task( version.pk, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 237b843e0fb..f03f958b172 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -388,7 +388,7 @@ def changefreqs_generator(): raise Http404 sorted_versions = sort_version_aware( - Version.objects.public( + Version.internal.public( project=project, only_active=True, ), @@ -428,7 +428,7 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): translation_versions = ( - Version.objects.public(project=translation) + Version.internal.public(project=translation) .values_list('slug', flat=True) ) if version.slug in translation_versions: diff --git a/readthedocs/projects/admin.py b/readthedocs/projects/admin.py index d74e8d99075..fb4aef4772d 100644 --- a/readthedocs/projects/admin.py +++ b/readthedocs/projects/admin.py @@ -239,7 +239,7 @@ def reindex_active_versions(self, request, queryset): """Reindex all active versions of the selected projects to ES.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) active_versions = version_qs.filter(active=True) if not active_versions.exists(): @@ -271,7 +271,7 @@ def wipe_all_versions(self, request, queryset): """Wipe indexes of all versions of selected projects.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) if not version_qs.exists(): self.message_user( request, diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index a701a49c020..65c589c8b18 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -240,7 +240,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions.filter( + versions_choices = self.instance.versions(manager='internal').filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index b636b348aa8..dff87e3885f 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -908,7 +908,7 @@ def api_versions(self): def active_versions(self): from readthedocs.builds.models import Version - versions = Version.objects.public(project=self, only_active=True) + versions = Version.internal.public(project=self, only_active=True) return ( versions.filter(built=True, active=True) | versions.filter(active=True, uploaded=True) @@ -922,7 +922,7 @@ def ordered_active_versions(self, user=None): } if user: kwargs['user'] = user - versions = Version.objects.public(**kwargs).select_related( + versions = Version.internal.public(**kwargs).select_related( 'project', 'project__main_language_project', ).prefetch_related( @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions.filter(active=True) + return self.versions(manager='internal').filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions.all() + versions = self.versions(manager='internal').all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index 64d82272738..ea15bfb168b 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) project = self.get_object() - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=project, ) @@ -179,7 +179,7 @@ def project_downloads(request, project_slug): Project.objects.protected(request.user), slug=project_slug, ) - versions = Version.objects.public(user=request.user, project=project) + versions = Version.internal.public(user=request.user, project=project) versions = sort_version_aware(versions) version_data = OrderedDict() for version in versions: @@ -268,7 +268,7 @@ def project_versions(request, project_slug): slug=project_slug, ) - versions = Version.objects.public( + versions = Version.internal.public( user=request.user, project=project, only_active=False, From 0dd9eb295ac444c9f65a41b2ed405896d8c75e0a Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 02:15:34 +0600 Subject: [PATCH 064/171] Manager names moved to Constants --- readthedocs/api/v2/views/model_views.py | 4 ++-- readthedocs/builds/constants.py | 7 +++++++ readthedocs/builds/models.py | 5 +++-- readthedocs/core/management/commands/update_repos.py | 5 +++-- readthedocs/core/views/serve.py | 4 ++++ readthedocs/projects/forms.py | 3 ++- readthedocs/projects/models.py | 6 +++--- 7 files changed, 24 insertions(+), 10 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index 5f594a032b8..fca91b32626 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG +from readthedocs.builds.constants import BRANCH, TAG, INTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions(manager='internal').filter(active=True) + versions = project.versions(manager=INTERNAL).filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e2e4fc44ac3..e200fc5fe25 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -55,3 +55,10 @@ LATEST, STABLE, ) + +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding PULL_REQUEST Type. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only PULL_REQUEST Type Versions and Builds. +EXTERNAL = 'external' diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bcfc5dfff5c..d0c5f6a854e 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -32,12 +32,13 @@ from readthedocs.projects.models import APIProject, Project from readthedocs.projects.version_handling import determine_stable_version -from .constants import ( +from readthedocs.builds.constants import ( BRANCH, BUILD_STATE, BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, BUILD_TYPES, + INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, PULL_REQUEST, @@ -143,7 +144,7 @@ def __str__(self): def ref(self): if self.slug == STABLE: stable = determine_stable_version( - self.project.versions(manager='internal').all() + self.project.versions(manager=INTERNAL).all() ) if stable: return stable.slug diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 0e14f9ce339..b15ea8c3c9c 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -10,6 +10,7 @@ from django.core.management.base import BaseCommand +from readthedocs.builds.constants import EXTERNAL, INTERNAL from readthedocs.builds.models import Build, Version from readthedocs.core.utils import trigger_build from readthedocs.projects import tasks @@ -76,7 +77,7 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) - elif version == 'internal': + elif version == INTERNAL: log.info('Updating all internal versions for %s', slug) for version in Version.internal.filter( project__slug=slug, @@ -97,7 +98,7 @@ def handle(self, *args, **options): build_pk=build.pk, version_pk=version.pk, ) - elif version == 'external': + elif version == EXTERNAL: log.info('Updating all external versions for %s', slug) for version in Version.external.filter( project__slug=slug, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index f03f958b172..061f5c5f4a6 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,7 +38,11 @@ from django.views.decorators.cache import cache_page from django.views.static import serve +<<<<<<< HEAD from readthedocs.builds.constants import LATEST, STABLE +======= +from readthedocs.builds.constants import INTERNAL +>>>>>>> Manager names moved to Constants from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index 65c589c8b18..3c86cc39e12 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -14,6 +14,7 @@ from guardian.shortcuts import assign from textclassifier.validators import ClassifierValidator +from readthedocs.builds.constants import INTERNAL from readthedocs.core.utils import slugify, trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.integrations.models import Integration @@ -240,7 +241,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions(manager='internal').filter( + versions_choices = self.instance.versions(manager=INTERNAL).filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index dff87e3885f..a3f36feb5c8 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -20,7 +20,7 @@ from taggit.managers import TaggableManager from readthedocs.api.v2.client import api -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, INTERNAL from readthedocs.core.resolver import resolve, resolve_domain from readthedocs.core.utils import broadcast, slugify from readthedocs.projects import constants @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions(manager='internal').filter(active=True) + return self.versions(manager=INTERNAL).filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions(manager='internal').all() + versions = self.versions(manager=INTERNAL).all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() From f2678810ce734c81da35c7f3316a998f740c37be Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:25:03 +0600 Subject: [PATCH 065/171] Tests added --- readthedocs/builds/constants.py | 2 +- .../rtd_tests/tests/test_doc_serving.py | 23 +++- readthedocs/rtd_tests/tests/test_managers.py | 100 ++++++++++++++++++ readthedocs/rtd_tests/tests/test_project.py | 28 ++++- .../rtd_tests/tests/test_project_forms.py | 25 ++++- .../rtd_tests/tests/test_project_views.py | 30 +++++- readthedocs/rtd_tests/tests/test_version.py | 68 ++++++++++++ 7 files changed, 268 insertions(+), 8 deletions(-) create mode 100644 readthedocs/rtd_tests/tests/test_managers.py create mode 100644 readthedocs/rtd_tests/tests/test_version.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e200fc5fe25..37474eb5365 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -58,7 +58,7 @@ # Manager name for Internal Versions or Builds. # ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' +INTERNAL = 'internal' # Manager name for External Versions or Builds. # ie: Only PULL_REQUEST Type Versions and Builds. EXTERNAL = 'external' diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index fed119bbab5..70ba1489294 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -10,7 +10,7 @@ from django.urls import reverse from mock import mock_open, patch -from readthedocs.builds.constants import LATEST +from readthedocs.builds.constants import PULL_REQUEST, INTERNAL from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -249,6 +249,16 @@ def test_sitemap_xml(self): project=self.public, active=True ) + # This is a Pull Request Version + pr_version = fixture.get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.public, + active=True, + type=PULL_REQUEST + ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( Project, @@ -268,7 +278,7 @@ def test_sitemap_xml(self): ) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/xml') - for version in self.public.versions.filter(privacy_level=constants.PUBLIC): + for version in self.public.versions(manager=INTERNAL).filter(privacy_level=constants.PUBLIC): self.assertContains( response, self.public.get_docs_url( @@ -303,6 +313,15 @@ def test_sitemap_xml(self): # in language and country value. (zh_CN should be zh-CN) self.assertContains(response, 'zh-CN') + # PR Versions should not be in the sitemap_xml. + self.assertNotContains( + response, + self.public.get_docs_url( + version_slug=pr_version.slug, + lang_slug=self.public.language, + private=True, + ), + ) # Check if STABLE version has 'priority of 1 and changefreq of weekly. self.assertEqual( response.context['versions'][0]['loc'], diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py new file mode 100644 index 00000000000..8e440e18b85 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -0,0 +1,100 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED +from readthedocs.projects.models import Project + + +class TestVersionManagerBase(TestCase): + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.public_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC + ) + self.private_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PRIVATE + ) + self.protected_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PROTECTED + ) + self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + + +class TestInternalVersionManager(TestVersionManagerBase): + + """ + Queries using Internal Manager should only include Internal Versions. + + It will exclude PULL_REQUEST type Versions from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + + def test_internal_version_manager_with_all(self): + self.assertNotIn(self.public_pr_version, Version.internal.all()) + + def test_internal_version_manager_with_public(self): + self.assertNotIn(self.public_pr_version, Version.internal.public()) + + def test_internal_version_manager_with_protected(self): + self.assertNotIn(self.protected_pr_version, Version.internal.protected()) + + def test_internal_version_manager_with_private(self): + self.assertNotIn(self.private_pr_version, Version.internal.private()) + + def test_internal_version_manager_with_api(self): + self.assertNotIn(self.public_pr_version, Version.internal.api()) + + def test_internal_version_manager_with_for_project(self): + self.assertNotIn(self.public_pr_version, Version.internal.for_project(self.pip)) + + +class TestExternalVersionManager(TestVersionManagerBase): + + """ + Queries using External Manager should only include External Versions. + + It will only include PULL_REQUEST type Versions in the queries. + """ + + def test_external_version_manager_with_all(self): + self.assertNotIn(self.internal_versions, Version.external.all()) + self.assertIn(self.public_pr_version, Version.external.all()) + + def test_external_version_manager_with_public(self): + self.assertNotIn(self.internal_versions, Version.external.public()) + self.assertIn(self.public_pr_version, Version.external.public()) + + def test_external_version_manager_with_protected(self): + self.assertNotIn(self.internal_versions, Version.external.protected()) + self.assertIn(self.protected_pr_version, Version.external.protected()) + + def test_external_version_manager_with_private(self): + self.assertNotIn(self.internal_versions, Version.external.private()) + self.assertIn(self.private_pr_version, Version.external.private()) + + def test_external_version_manager_with_api(self): + self.assertNotIn(self.internal_versions, Version.external.api()) + self.assertIn(self.public_pr_version, Version.external.api()) + + def test_external_version_manager_with_for_project(self): + self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) + self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 40f86350ed4..c2db080f201 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,8 +14,9 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, + PULL_REQUEST, ) -from readthedocs.builds.models import Build +from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project from readthedocs.projects.tasks import finish_inactive_builds @@ -29,6 +30,16 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) class TestProject(ProjectMixin, TestCase): @@ -134,6 +145,21 @@ def test_get_storage_path(self): 'htmlzip/pip/latest/pip.zip', ) + def test_ordered_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.ordered_active_versions()) + + def test_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.active_versions()) + + def test_all_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.all_active_versions()) + + def test_update_stable_version_excludes_pr_versions(self): + # Delete all versions excluding PR Versions. + self.pip.versions.exclude(type=PULL_REQUEST).delete() + # Test that PR Version is not considered for stable. + self.assertEqual(self.pip.update_stable_version(), None) + class TestProjectTranslations(ProjectMixin, TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index 50d9815d051..b2e8687208a 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -314,7 +314,7 @@ def setUp(self): verbose_name='protected', ) - def test_list_only_non_auto_generated_versions_on_default_branch(self): + def test_list_only_non_auto_generated_versions_in_default_branch_choices(self): form = ProjectAdvancedForm(instance=self.project) # This version is created automatically by the project on save latest = self.project.versions.filter(slug=LATEST) @@ -336,7 +336,7 @@ def test_list_only_non_auto_generated_versions_on_default_branch(self): 'default_branch'].widget.choices], ) - def test_list_user_created_latest_and_stable_versions_on_default_branch(self): + def test_list_user_created_latest_and_stable_versions_in_default_branch_choices(self): self.project.versions.filter(slug=LATEST).first().delete() user_created_latest_version = get( Version, @@ -383,6 +383,25 @@ def test_commit_name_not_in_default_branch_choices(self): 'default_branch'].widget.choices], ) + def test_pr_version_not_in_default_branch_choices(self): + pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.project, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC, + ) + form = ProjectAdvancedForm(instance=self.project) + + self.assertNotIn( + pr_version.verbose_name, + [identifier for identifier, _ in form.fields[ + 'default_branch'].widget.choices], + ) + class TestTranslationForms(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 7c5b286bdd4..90578c8dceb 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST +from readthedocs.builds.constants import LATEST, PULL_REQUEST from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -370,12 +370,40 @@ def test_import_demo_imported_duplicate(self): class TestPublicViews(MockBuildTestCase): def setUp(self): self.pip = get(Project, slug='pip') + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) def test_project_download_media(self): url = reverse('project_download_media', args=[self.pip.slug, 'pdf', LATEST]) response = self.client.get(url) self.assertEqual(response.status_code, 302) + def test_project_detail_view_only_shows_internal_versons(self): + url = reverse('projects_detail', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_downloads_only_shows_internal_versons(self): + url = reverse('project_downloads', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_versions_only_shows_internal_versons(self): + url = reverse('project_version_list', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['active_versions']) + self.assertNotIn(self.pr_version, response.context['inactive_versions']) + class TestPrivateViews(MockBuildTestCase): def setUp(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py new file mode 100644 index 00000000000..8a6f5bf3db8 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -0,0 +1,68 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.models import Project + + +class VersionMixin: + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='9F86D081884C7D659A2FEAA0C55AD015A', + verbose_name='pr-version', + slug='9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) + self.branch_version = get( + Version, + identifier='origin/stable', + verbose_name='stable', + slug='stable', + project=self.pip, + active=True, + type=BRANCH + ) + self.tag_version = get( + Version, + identifier='origin/master', + verbose_name='latest', + slug='latest', + project=self.pip, + active=True, + type=TAG + ) + + +class TestVersionModel(VersionMixin, TestCase): + + def test_vcs_url_for_pr_version(self): + expected_url = f'https://github.com/pypa/pip/pull/{self.pr_version.slug}/' + self.assertEqual(self.pr_version.vcs_url, expected_url) + + def test_vcs_url_for_latest_version(self): + slug = self.pip.default_branch or self.pip.vcs_repo().fallback_branch + expected_url = f'https://github.com/pypa/pip/tree/{slug}/' + self.assertEqual(self.tag_version.vcs_url, expected_url) + + def test_vcs_url_for_stable_version(self): + expected_url = f'https://github.com/pypa/pip/tree/{self.branch_version.ref}/' + self.assertEqual(self.branch_version.vcs_url, expected_url) + + def test_commit_name_for_stable_version(self): + self.assertEqual(self.branch_version.commit_name, 'stable') + + def test_commit_name_for_latest_version(self): + self.assertEqual(self.tag_version.commit_name, 'master') + + def test_commit_name_for_pr_version(self): + self.assertEqual(self.pr_version.commit_name, self.pr_version.identifier) From cc1297f112bdc374bf7a997857e41248682137da Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:52:31 +0600 Subject: [PATCH 066/171] Version Update, Delete Html and wipe updated to exclude PR Versions --- readthedocs/core/views/__init__.py | 2 +- readthedocs/projects/views/private.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 6b438866ae5..de519a13759 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -73,7 +73,7 @@ def random_page(request, project_slug=None): # pylint: disable=unused-argument def wipe_version(request, project_slug, version_slug): version = get_object_or_404( - Version, + Version.internal.all(), project__slug=project_slug, slug=version_slug, ) diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index 46e83d96188..044a83641e6 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -157,7 +157,7 @@ def project_version_detail(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, @@ -682,7 +682,7 @@ def project_version_delete_html(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, From 20ca139b8f9f1ff75010ccdec2d504eca8da2091 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 23:03:09 +0600 Subject: [PATCH 067/171] Updated migrations --- .../builds/migrations/0008_added_pull_request_version_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 14d8968866d..75a126d37b6 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-05-30 10:02 +# Generated by Django 1.11.20 on 2019-06-06 16:51 from __future__ import unicode_literals from django.db import migrations, models From cb17a79cfeae212036ef0d72319c924b6d771645 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sun, 9 Jun 2019 02:17:05 +0600 Subject: [PATCH 068/171] footer API updated --- readthedocs/api/v2/views/footer_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/api/v2/views/footer_views.py b/readthedocs/api/v2/views/footer_views.py index 04d4fe8210b..04fedeadd3f 100644 --- a/readthedocs/api/v2/views/footer_views.py +++ b/readthedocs/api/v2/views/footer_views.py @@ -9,7 +9,7 @@ from rest_framework_jsonp.renderers import JSONPRenderer from readthedocs.api.v2.signals import footer_response -from readthedocs.builds.constants import LATEST, TAG +from readthedocs.builds.constants import LATEST, TAG, INTERNAL from readthedocs.builds.models import Version from readthedocs.projects.models import Project from readthedocs.projects.version_handling import ( @@ -25,7 +25,7 @@ def get_version_compare_data(project, base_version=None): :param base_version: We assert whether or not the base_version is also the highest version in the resulting "is_highest" value. """ - versions_qs = Version.objects.public(project=project) + versions_qs = Version.internal.public(project=project) # Take preferences over tags only if the project has at least one tag if versions_qs.filter(type=TAG).exists(): From f6873023d0388d7eda15d25dc930114e10a2ac6c Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 00:45:00 +0600 Subject: [PATCH 069/171] manager tests updated --- readthedocs/rtd_tests/tests/test_managers.py | 34 +++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 8e440e18b85..e69abdc4dd8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -1,3 +1,4 @@ +from django.contrib.auth import get_user_model from django.test import TestCase from django_dynamic_fixture import get @@ -7,12 +8,16 @@ from readthedocs.projects.models import Project +User = get_user_model() + + class TestVersionManagerBase(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ['test_data'] def setUp(self): - self.client.login(username='eric', password='test') + self.user = User.objects.create(username='test_user', password='test') + self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') # Create a External Version. ie: PULL_REQUEST type Version. self.public_pr_version = get( @@ -54,6 +59,12 @@ def test_internal_version_manager_with_all(self): def test_internal_version_manager_with_public(self): self.assertNotIn(self.public_pr_version, Version.internal.public()) + def test_internal_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.public_pr_version, + Version.internal.public(self.user, self.pip) + ) + def test_internal_version_manager_with_protected(self): self.assertNotIn(self.protected_pr_version, Version.internal.protected()) @@ -83,6 +94,17 @@ def test_external_version_manager_with_public(self): self.assertNotIn(self.internal_versions, Version.external.public()) self.assertIn(self.public_pr_version, Version.external.public()) + def test_external_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.internal_versions, + Version.external.public(self.user, self.pip) + ) + self.assertIn( + self.public_pr_version, + Version.external.public(self.user, self.pip) + ) + + def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) @@ -96,5 +118,9 @@ def test_external_version_manager_with_api(self): self.assertIn(self.public_pr_version, Version.external.api()) def test_external_version_manager_with_for_project(self): - self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) - self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) + self.assertNotIn( + self.internal_versions, Version.external.for_project(self.pip) + ) + self.assertIn( + self.public_pr_version, Version.external.for_project(self.pip) + ) From bf9bd5e84f80197ee65d49cfe7029ae87e2a723d Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 18:56:06 +0600 Subject: [PATCH 070/171] _override_setting removed from new managers --- readthedocs/builds/managers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 2a8bb4840d9..0bc30f02807 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -118,9 +118,7 @@ class VersionManager(SettingsOverrideObject): class InternalVersionManager(SettingsOverrideObject): _default_class = InternalVersionManagerBase - _override_setting = 'INTERNAL_VERSION_MANAGER' class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase - _override_setting = 'EXTERNAL_VERSION_MANAGER' From 12cd686a82539444bd1d58b88e4e8efb828d40d6 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:11:51 +0600 Subject: [PATCH 071/171] BuildTriggerMixin updated --- readthedocs/builds/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index 3d6e2cc0ca1..3333495f6c8 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -57,7 +57,7 @@ def post(self, request, project_slug): version_slug = request.POST.get('version_slug') version = get_object_or_404( - Version, + Version.internal.all(), project=project, slug=version_slug, ) From d7330c1872b740d55340f7c703a930340b92118d Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:25:28 +0600 Subject: [PATCH 072/171] More Update --- readthedocs/search/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 5f04f6736b7..cab67ba5ff8 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,7 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.objects.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) if version.exists(): project_list.append(version.first().project) return project_list From eed1a9c4bff7ae357ee1511158ab667bbf0e2665 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:55:32 +0600 Subject: [PATCH 073/171] lint fix --- readthedocs/rtd_tests/tests/test_managers.py | 1 - readthedocs/search/utils.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index e69abdc4dd8..f830c9af549 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -104,7 +104,6 @@ def test_external_version_manager_with_public_with_user_and_project(self): Version.external.public(self.user, self.pip) ) - def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index cab67ba5ff8..3433195bb55 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,9 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter( + project__slug=project.slug, slug=version_slug + ) if version.exists(): project_list.append(version.first().project) return project_list From f3f86dc28798eb1b0d6ed4a065fe5e81168a16a9 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 13 Jun 2019 14:53:57 -0500 Subject: [PATCH 074/171] Move search functions Follow up from #5798 --- readthedocs/projects/tasks.py | 2 +- readthedocs/search/signals.py | 61 -------------------------------- readthedocs/search/utils.py | 66 +++++++++++++++++++++++++++++++++-- 3 files changed, 64 insertions(+), 65 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index d0ebdb826a0..7efcd5a762a 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -60,7 +60,7 @@ from readthedocs.doc_builder.loader import get_builder_class from readthedocs.doc_builder.python_environments import Conda, Virtualenv from readthedocs.projects.models import APIProject, Feature -from readthedocs.search.signals import index_new_files, remove_indexed_files +from readthedocs.search.utils import index_new_files, remove_indexed_files from readthedocs.sphinx_domains.models import SphinxDomain from readthedocs.vcs_support import utils as vcs_support_utils from readthedocs.worker import app diff --git a/readthedocs/search/signals.py b/readthedocs/search/signals.py index 494725d22a7..294d96242d0 100644 --- a/readthedocs/search/signals.py +++ b/readthedocs/search/signals.py @@ -13,67 +13,6 @@ log = logging.getLogger(__name__) -def index_new_files(model, version, build): - """Index new files from the version into the search index.""" - - if not DEDConfig.autosync_enabled(): - log.info( - 'Autosync disabled, skipping indexing into the search index for: %s:%s', - version.project.slug, - version.slug, - ) - return - - try: - document = list(registry.get_documents(models=[model]))[0] - doc_obj = document() - queryset = ( - doc_obj.get_queryset() - .filter(project=version.project, version=version, build=build) - ) - log.info( - 'Indexing new objecst into search index for: %s:%s', - version.project.slug, - version.slug, - ) - doc_obj.update(queryset.iterator()) - except Exception: - log.exception('Unable to index a subset of files. Continuing.') - - -def remove_indexed_files(model, version, build): - """ - Remove files from the version from the search index. - - This excludes files from the current build. - """ - - if not DEDConfig.autosync_enabled(): - log.info( - 'Autosync disabled, skipping removal from the search index for: %s:%s', - version.project.slug, - version.slug, - ) - return - - try: - document = list(registry.get_documents(models=[model]))[0] - log.info( - 'Deleting old files from search index for: %s:%s', - version.project.slug, - version.slug, - ) - ( - document().search() - .filter('term', project=version.project.slug) - .filter('term', version=version.slug) - .exclude('term', build=build) - .delete() - ) - except Exception: - log.exception('Unable to delete a subset of files. Continuing.') - - @receiver(post_save, sender=Project) def index_project_save(instance, *args, **kwargs): """ diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 3433195bb55..6cbc0272722 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -1,20 +1,80 @@ -# -*- coding: utf-8 -*- - """Utilities related to reading and generating indexable search content.""" import logging from django.shortcuts import get_object_or_404 +from django_elasticsearch_dsl.apps import DEDConfig from django_elasticsearch_dsl.registries import registry from readthedocs.builds.models import Version -from readthedocs.projects.models import Project, HTMLFile +from readthedocs.projects.models import HTMLFile, Project from readthedocs.search.documents import PageDocument log = logging.getLogger(__name__) +def index_new_files(model, version, build): + """Index new files from the version into the search index.""" + + if not DEDConfig.autosync_enabled(): + log.info( + 'Autosync disabled, skipping indexing into the search index for: %s:%s', + version.project.slug, + version.slug, + ) + return + + try: + document = list(registry.get_documents(models=[model]))[0] + doc_obj = document() + queryset = ( + doc_obj.get_queryset() + .filter(project=version.project, version=version, build=build) + ) + log.info( + 'Indexing new objecst into search index for: %s:%s', + version.project.slug, + version.slug, + ) + doc_obj.update(queryset.iterator()) + except Exception: + log.exception('Unable to index a subset of files. Continuing.') + + +def remove_indexed_files(model, version, build): + """ + Remove files from the version from the search index. + + This excludes files from the current build. + """ + + if not DEDConfig.autosync_enabled(): + log.info( + 'Autosync disabled, skipping removal from the search index for: %s:%s', + version.project.slug, + version.slug, + ) + return + + try: + document = list(registry.get_documents(models=[model]))[0] + log.info( + 'Deleting old files from search index for: %s:%s', + version.project.slug, + version.slug, + ) + ( + document().search() + .filter('term', project=version.project.slug) + .filter('term', version=version.slug) + .exclude('term', build=build) + .delete() + ) + except Exception: + log.exception('Unable to delete a subset of files. Continuing.') + + # TODO: Rewrite all the views using this in Class Based View, # and move this function to a mixin def get_project_list_or_404(project_slug, user, version_slug=None): From 3d6224878c8d7cbe85299efd788631d14335b7bd Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 17:35:34 +0600 Subject: [PATCH 075/171] removed unused import --- readthedocs/core/views/serve.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 061f5c5f4a6..f03f958b172 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,11 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve -<<<<<<< HEAD from readthedocs.builds.constants import LATEST, STABLE -======= -from readthedocs.builds.constants import INTERNAL ->>>>>>> Manager names moved to Constants from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path From 09a09a262947e31474cacda03a75771e056f3008 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:30:26 +0600 Subject: [PATCH 076/171] External version name added everywhere --- readthedocs/builds/constants.py | 17 ++++++++--------- readthedocs/builds/managers.py | 10 +++++----- .../0008_added_pull_request_version_type.py | 6 +++--- readthedocs/builds/models.py | 14 +++++++------- readthedocs/rtd_tests/tests/test_doc_serving.py | 6 +++--- readthedocs/rtd_tests/tests/test_managers.py | 16 ++++++++-------- readthedocs/rtd_tests/tests/test_project.py | 8 ++++---- .../rtd_tests/tests/test_project_forms.py | 4 ++-- .../rtd_tests/tests/test_project_views.py | 4 ++-- readthedocs/rtd_tests/tests/test_version.py | 6 +++--- 10 files changed, 45 insertions(+), 46 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 37474eb5365..bdc9a70fe79 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -31,15 +31,21 @@ ('dash', _('Dash')), ) +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding pull request/merge request Versions and Builds. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only pull request/merge request Versions and Builds. +EXTERNAL = 'external' + BRANCH = 'branch' TAG = 'tag' -PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), - (PULL_REQUEST, _('Pull Request')), + (EXTERNAL, _('External')), (UNKNOWN, _('Unknown')), ) @@ -55,10 +61,3 @@ LATEST, STABLE, ) - -# Manager name for Internal Versions or Builds. -# ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' -# Manager name for External Versions or Builds. -# ie: Only PULL_REQUEST Type Versions and Builds. -EXTERNAL = 'external' diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 0bc30f02807..06262ed3e24 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,7 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, - PULL_REQUEST, + EXTERNAL, ) from .querysets import VersionQuerySet @@ -91,12 +91,12 @@ class InternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes internal version. - It will exclude PULL_REQUEST type from the queries + It will exclude pull request/merge request versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ def get_queryset(self): - return super().get_queryset().exclude(type=PULL_REQUEST) + return super().get_queryset().exclude(type=EXTERNAL) class ExternalVersionManagerBase(VersionManagerBase): @@ -104,11 +104,11 @@ class ExternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes external version. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull request/merge request Versions in the queries. """ def get_queryset(self): - return super().get_queryset().filter(type=PULL_REQUEST) + return super().get_queryset().filter(type=EXTERNAL) class VersionManager(SettingsOverrideObject): diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 75a126d37b6..2bf9dc62d66 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-06-06 16:51 +# Generated by Django 1.11.21 on 2019-06-17 19:26 from __future__ import unicode_literals from django.db import migrations, models @@ -15,11 +15,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='version', name='type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), ), migrations.AlterField( model_name='versionautomationrule', name='version_type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), ), ] diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index d0c5f6a854e..34a3fd372a9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -41,7 +41,7 @@ INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, - PULL_REQUEST, + EXTERNAL, STABLE, TAG, VERSION_TYPES, @@ -119,7 +119,7 @@ class Version(models.Model): objects = VersionManager.from_queryset(VersionQuerySet)() # Only include BRANCH, TAG, UNKONWN type Versions. internal = InternalVersionManager.from_queryset(VersionQuerySet)() - # Only include PULL_REQUEST type Versions. + # Only include EXTERNAL type Versions. external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: @@ -155,7 +155,7 @@ def vcs_url(self): Generate VCS (github, gitlab, bitbucket) URL for this version. Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. - Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. + Pull/merge Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -165,7 +165,7 @@ def vcs_url(self): else: slug_url = self.slug - if self.type == PULL_REQUEST: + if self.type == EXTERNAL: if 'github' in self.project.repo: url = f'/pull/{slug_url}/' @@ -245,14 +245,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - if self.type == PULL_REQUEST: - # If this version is a Pull Request, the identifier will + if self.type == EXTERNAL: + # If this version is a EXTERNAL version, the identifier will # contain the actual commit hash. which we can use to # generate url for a given file name return self.identifier # If we came that far it's not a special version - # nor a branch, tag or Pull Request. + # nor a branch, tag or EXTERNAL version. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 70ba1489294..8ddbca2a410 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -10,7 +10,7 @@ from django.urls import reverse from mock import mock_open, patch -from readthedocs.builds.constants import PULL_REQUEST, INTERNAL +from readthedocs.builds.constants import LATEST, EXTERNAL, INTERNAL from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -249,7 +249,7 @@ def test_sitemap_xml(self): project=self.public, active=True ) - # This is a Pull Request Version + # This is a EXTERNAL Version pr_version = fixture.get( Version, identifier='pr-version', @@ -257,7 +257,7 @@ def test_sitemap_xml(self): slug='pr-9999', project=self.public, active=True, - type=PULL_REQUEST + type=EXTERNAL ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index f830c9af549..924544a8be8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -2,7 +2,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED from readthedocs.projects.models import Project @@ -19,29 +19,29 @@ def setUp(self): self.user = User.objects.create(username='test_user', password='test') self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.public_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC ) self.private_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PRIVATE ) self.protected_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PROTECTED ) - self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + self.internal_versions = Version.objects.exclude(type=EXTERNAL) class TestInternalVersionManager(TestVersionManagerBase): @@ -49,7 +49,7 @@ class TestInternalVersionManager(TestVersionManagerBase): """ Queries using Internal Manager should only include Internal Versions. - It will exclude PULL_REQUEST type Versions from the queries + It will exclude EXTERNAL type Versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ @@ -83,7 +83,7 @@ class TestExternalVersionManager(TestVersionManagerBase): """ Queries using External Manager should only include External Versions. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull/merge request Version in the queries. """ def test_external_version_manager_with_all(self): diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index c2db080f201..5d5194d04d2 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,7 +14,7 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, - PULL_REQUEST, + EXTERNAL, ) from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError @@ -30,7 +30,7 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='pr-version', @@ -38,7 +38,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) @@ -156,7 +156,7 @@ def test_all_active_versions_excludes_pr_versions(self): def test_update_stable_version_excludes_pr_versions(self): # Delete all versions excluding PR Versions. - self.pip.versions.exclude(type=PULL_REQUEST).delete() + self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index b2e8687208a..ba5b3c88b9d 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST +from readthedocs.builds.constants import LATEST, STABLE, EXTERNAL from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -391,7 +391,7 @@ def test_pr_version_not_in_default_branch_choices(self): slug='pr-9999', project=self.project, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC, ) form = ProjectAdvancedForm(instance=self.project) diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 90578c8dceb..3bc948aff9b 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST, PULL_REQUEST +from readthedocs.builds.constants import LATEST, EXTERNAL from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -377,7 +377,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) def test_project_download_media(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py index 8a6f5bf3db8..e414ba79cc6 100644 --- a/readthedocs/rtd_tests/tests/test_version.py +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -1,7 +1,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.models import Project @@ -13,7 +13,7 @@ class VersionMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='9F86D081884C7D659A2FEAA0C55AD015A', @@ -21,7 +21,7 @@ def setUp(self): slug='9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) self.branch_version = get( Version, From 119a3aeefbdea297926b23715cd7387bb41bc2a3 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:43:48 +0600 Subject: [PATCH 077/171] Migration name changed --- ...uest_version_type.py => 0008_added_external_version_type.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename readthedocs/builds/migrations/{0008_added_pull_request_version_type.py => 0008_added_external_version_type.py} (94%) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_external_version_type.py similarity index 94% rename from readthedocs/builds/migrations/0008_added_pull_request_version_type.py rename to readthedocs/builds/migrations/0008_added_external_version_type.py index 2bf9dc62d66..8de9c4f504f 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_external_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.21 on 2019-06-17 19:26 +# Generated by Django 1.11.21 on 2019-06-17 19:43 from __future__ import unicode_literals from django.db import migrations, models From df4705af045eff42716e0846f7f259d66d4fd30b Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 19:07:12 +0600 Subject: [PATCH 078/171] Removed PR Versions from Elasticsearch Indexing --- readthedocs/search/documents.py | 3 ++- readthedocs/search/tests/test_api.py | 1 - readthedocs/search/tests/test_documents.py | 24 ++++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 readthedocs/search/tests/test_documents.py diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index 28a3a61f477..f5dad2e2ea6 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -5,6 +5,7 @@ from elasticsearch import Elasticsearch +from readthedocs.builds.constants import PULL_REQUEST from readthedocs.projects.models import HTMLFile, Project from readthedocs.sphinx_domains.models import SphinxDomain @@ -159,7 +160,7 @@ def get_queryset(self): # Also do not index certain files queryset = queryset.filter( project__documentation_type__contains='sphinx' - ) + ).exclude(version__type=PULL_REQUEST) # TODO: Make this smarter # This was causing issues excluding some valid user documentation pages diff --git a/readthedocs/search/tests/test_api.py b/readthedocs/search/tests/test_api.py index 42e8187a856..0167172cca8 100644 --- a/readthedocs/search/tests/test_api.py +++ b/readthedocs/search/tests/test_api.py @@ -2,7 +2,6 @@ from django.core.urlresolvers import reverse from django_dynamic_fixture import G - from readthedocs.builds.models import Version from readthedocs.projects.models import HTMLFile from readthedocs.search.tests.utils import get_search_query_from_project_file diff --git a/readthedocs/search/tests/test_documents.py b/readthedocs/search/tests/test_documents.py new file mode 100644 index 00000000000..c4725825d45 --- /dev/null +++ b/readthedocs/search/tests/test_documents.py @@ -0,0 +1,24 @@ +import pytest + +from readthedocs.builds.constants import PULL_REQUEST +from readthedocs.projects.models import HTMLFile +from readthedocs.search.documents import PageDocument + + +@pytest.mark.django_db +@pytest.mark.search +class TestPageDocument: + + def test_get_queryset_does_not_include_pr_versions(self, project): + # turn version into PR Version + version = project.versions.all()[0] + version.type = PULL_REQUEST + version.save() + + html_file = HTMLFile.objects.filter( + project__slug=project.slug, version=version + ) + qs = PageDocument().get_queryset() + + assert qs.model == HTMLFile + assert not set(html_file).issubset(set(qs)) From 9140a57e745045d08c5fa74e4e15568953a822dd Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 22:45:36 +0600 Subject: [PATCH 079/171] HTMLFile manager added --- readthedocs/core/views/__init__.py | 2 +- readthedocs/projects/managers.py | 22 +++++++++- readthedocs/projects/models.py | 3 +- readthedocs/projects/querysets.py | 25 +++++++++++ readthedocs/rtd_tests/tests/test_managers.py | 44 +++++++++++++++++++- readthedocs/search/documents.py | 4 +- 6 files changed, 94 insertions(+), 6 deletions(-) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index de519a13759..935860f4a62 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -61,7 +61,7 @@ def get_context_data(self, **kwargs): def random_page(request, project_slug=None): # pylint: disable=unused-argument - html_file = HTMLFile.objects.order_by('?') + html_file = HTMLFile.objects.internal().order_by('?') if project_slug: html_file = html_file.filter(project__slug=project_slug) html_file = html_file.first() diff --git a/readthedocs/projects/managers.py b/readthedocs/projects/managers.py index 995ea3da4f2..40f445e3cc3 100644 --- a/readthedocs/projects/managers.py +++ b/readthedocs/projects/managers.py @@ -1,7 +1,27 @@ from django.db import models +from readthedocs.builds.constants import PULL_REQUEST +from readthedocs.core.utils.extend import ( + get_override_class, + SettingsOverrideObject +) +from readthedocs.projects.querysets import HTMLFileQuerySet -class HTMLFileManager(models.Manager): + +class HTMLFileManagerBase(models.Manager): + + @classmethod + def from_queryset(cls, queryset_class, class_name=None): + queryset_class = get_override_class( + HTMLFileQuerySet, + HTMLFileQuerySet._default_class, # pylint: disable=protected-access + ) + return super().from_queryset(queryset_class, class_name) def get_queryset(self): return super().get_queryset().filter(name__endswith='.html') + + +class HTMLFileManager(SettingsOverrideObject): + _default_class = HTMLFileManagerBase + diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index a3f36feb5c8..ac725d4b311 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -31,6 +31,7 @@ FeatureQuerySet, ProjectQuerySet, RelatedProjectQuerySet, + HTMLFileQuerySet, ) from readthedocs.projects.templatetags.projects_tags import sort_version_aware from readthedocs.projects.validators import ( @@ -1216,7 +1217,7 @@ class HTMLFile(ImportedFile): class Meta: proxy = True - objects = HTMLFileManager() + objects = HTMLFileManager.from_queryset(HTMLFileQuerySet)() def get_processed_json(self): """ diff --git a/readthedocs/projects/querysets.py b/readthedocs/projects/querysets.py index 7ea7249ae0d..44879b094f6 100644 --- a/readthedocs/projects/querysets.py +++ b/readthedocs/projects/querysets.py @@ -4,6 +4,7 @@ from django.db.models import Q, OuterRef, Subquery, Prefetch from guardian.shortcuts import get_objects_for_user +from readthedocs.builds.constants import PULL_REQUEST from readthedocs.core.utils.extend import SettingsOverrideObject from . import constants @@ -213,3 +214,27 @@ def for_project(self, project): Q(projects=project) | Q(default_true=True, add_date__gt=project.pub_date), ).distinct() + + +class HTMLFileQuerySetBase(models.QuerySet): + + def internal(self): + """ + HTMLFileQuerySet method that only includes internal version html files. + + It will exclude PULL_REQUEST type from the queries + and only include BRANCH, TAG, UNKONWN type Version html files. + """ + return self.exclude(version__type=PULL_REQUEST) + + def external(self): + """ + HTMLFileQuerySet method that only includes external version html files. + + It will only include PULL_REQUEST type Version html files in the queries. + """ + return self.filter(version__type=PULL_REQUEST) + + +class HTMLFileQuerySet(SettingsOverrideObject): + _default_class = HTMLFileQuerySetBase diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 924544a8be8..3cb2f97ccb2 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -5,7 +5,7 @@ from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED -from readthedocs.projects.models import Project +from readthedocs.projects.models import Project, HTMLFile User = get_user_model() @@ -123,3 +123,45 @@ def test_external_version_manager_with_for_project(self): self.assertIn( self.public_pr_version, Version.external.for_project(self.pip) ) + + +class TestHTMLFileManager(TestCase): + + fixtures = ['test_data'] + + def setUp(self): + self.user = User.objects.create(username='test_user', password='test') + self.client.login(username='test_user', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC + ) + self.html_file = HTMLFile.objects.create( + project=self.pip, + version=self.pr_version, + name='file.html', + slug='file', + path='file.html', + md5='abcdef', + commit='1234567890abcdef', + ) + self.internal_html_files = HTMLFile.objects.exclude(version__type=PULL_REQUEST) + + def test_internal_html_file_manager(self): + """ + It will exclude PULL_REQUEST type Version html files from the queries + and only include BRANCH, TAG, UNKONWN type Version files. + """ + self.assertNotIn(self.html_file, HTMLFile.objects.internal()) + + def test_external_html_file_manager(self): + """ + It will only include PULL_REQUEST type Version html files in the queries. + """ + self.assertNotIn(self.internal_html_files, HTMLFile.objects.external()) + self.assertIn(self.html_file, HTMLFile.objects.external()) diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index f5dad2e2ea6..f20ef37ec2b 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -158,9 +158,9 @@ def get_queryset(self): # Do not index files that belong to non sphinx project # Also do not index certain files - queryset = queryset.filter( + queryset = queryset.internal().filter( project__documentation_type__contains='sphinx' - ).exclude(version__type=PULL_REQUEST) + ) # TODO: Make this smarter # This was causing issues excluding some valid user documentation pages From f902d6564547a67d724ea37424fab9b916dc26fd Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 22:52:08 +0600 Subject: [PATCH 080/171] typo fix --- readthedocs/rtd_tests/tests/test_managers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 3cb2f97ccb2..99266cde2c9 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -152,14 +152,14 @@ def setUp(self): ) self.internal_html_files = HTMLFile.objects.exclude(version__type=PULL_REQUEST) - def test_internal_html_file_manager(self): + def test_internal_html_file_queryset(self): """ It will exclude PULL_REQUEST type Version html files from the queries and only include BRANCH, TAG, UNKONWN type Version files. """ self.assertNotIn(self.html_file, HTMLFile.objects.internal()) - def test_external_html_file_manager(self): + def test_external_html_file_queryset(self): """ It will only include PULL_REQUEST type Version html files in the queries. """ From afd5e4758319b2101732309c63b73e65e3aba918 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Mon, 17 Jun 2019 19:10:39 +0600 Subject: [PATCH 081/171] lint fix --- readthedocs/projects/managers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/projects/managers.py b/readthedocs/projects/managers.py index 40f445e3cc3..d0ba71a8d52 100644 --- a/readthedocs/projects/managers.py +++ b/readthedocs/projects/managers.py @@ -24,4 +24,3 @@ def get_queryset(self): class HTMLFileManager(SettingsOverrideObject): _default_class = HTMLFileManagerBase - From f03db54964c18cad324a8dec7affd8dff83c98e7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 15:36:49 +0600 Subject: [PATCH 082/171] External naming updated --- readthedocs/projects/managers.py | 1 - readthedocs/projects/querysets.py | 10 +++++----- readthedocs/rtd_tests/tests/test_managers.py | 10 +++++----- readthedocs/search/documents.py | 1 - readthedocs/search/tests/test_documents.py | 6 +++--- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/readthedocs/projects/managers.py b/readthedocs/projects/managers.py index d0ba71a8d52..97724b12afd 100644 --- a/readthedocs/projects/managers.py +++ b/readthedocs/projects/managers.py @@ -1,6 +1,5 @@ from django.db import models -from readthedocs.builds.constants import PULL_REQUEST from readthedocs.core.utils.extend import ( get_override_class, SettingsOverrideObject diff --git a/readthedocs/projects/querysets.py b/readthedocs/projects/querysets.py index 44879b094f6..885425c8eeb 100644 --- a/readthedocs/projects/querysets.py +++ b/readthedocs/projects/querysets.py @@ -4,7 +4,7 @@ from django.db.models import Q, OuterRef, Subquery, Prefetch from guardian.shortcuts import get_objects_for_user -from readthedocs.builds.constants import PULL_REQUEST +from readthedocs.builds.constants import EXTERNAL from readthedocs.core.utils.extend import SettingsOverrideObject from . import constants @@ -222,18 +222,18 @@ def internal(self): """ HTMLFileQuerySet method that only includes internal version html files. - It will exclude PULL_REQUEST type from the queries + It will exclude pull request/merge request Version html files from the queries and only include BRANCH, TAG, UNKONWN type Version html files. """ - return self.exclude(version__type=PULL_REQUEST) + return self.exclude(version__type=EXTERNAL) def external(self): """ HTMLFileQuerySet method that only includes external version html files. - It will only include PULL_REQUEST type Version html files in the queries. + It will only include pull request/merge request Version html files in the queries. """ - return self.filter(version__type=PULL_REQUEST) + return self.filter(version__type=EXTERNAL) class HTMLFileQuerySet(SettingsOverrideObject): diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 99266cde2c9..1996aa4c1c8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -133,12 +133,12 @@ def setUp(self): self.user = User.objects.create(username='test_user', password='test') self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC ) self.html_file = HTMLFile.objects.create( @@ -150,18 +150,18 @@ def setUp(self): md5='abcdef', commit='1234567890abcdef', ) - self.internal_html_files = HTMLFile.objects.exclude(version__type=PULL_REQUEST) + self.internal_html_files = HTMLFile.objects.exclude(version__type=EXTERNAL) def test_internal_html_file_queryset(self): """ - It will exclude PULL_REQUEST type Version html files from the queries + It will exclude pull/merge request Version html files from the queries and only include BRANCH, TAG, UNKONWN type Version files. """ self.assertNotIn(self.html_file, HTMLFile.objects.internal()) def test_external_html_file_queryset(self): """ - It will only include PULL_REQUEST type Version html files in the queries. + It will only include pull/merge request Version html files in the queries. """ self.assertNotIn(self.internal_html_files, HTMLFile.objects.external()) self.assertIn(self.html_file, HTMLFile.objects.external()) diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index f20ef37ec2b..9595ef2b372 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -5,7 +5,6 @@ from elasticsearch import Elasticsearch -from readthedocs.builds.constants import PULL_REQUEST from readthedocs.projects.models import HTMLFile, Project from readthedocs.sphinx_domains.models import SphinxDomain diff --git a/readthedocs/search/tests/test_documents.py b/readthedocs/search/tests/test_documents.py index c4725825d45..cb046802ca4 100644 --- a/readthedocs/search/tests/test_documents.py +++ b/readthedocs/search/tests/test_documents.py @@ -1,6 +1,6 @@ import pytest -from readthedocs.builds.constants import PULL_REQUEST +from readthedocs.builds.constants import EXTERNAL from readthedocs.projects.models import HTMLFile from readthedocs.search.documents import PageDocument @@ -9,10 +9,10 @@ @pytest.mark.search class TestPageDocument: - def test_get_queryset_does_not_include_pr_versions(self, project): + def test_get_queryset_does_not_include_external_versions(self, project): # turn version into PR Version version = project.versions.all()[0] - version.type = PULL_REQUEST + version.type = EXTERNAL version.save() html_file = HTMLFile.objects.filter( From 5a05b6cf792533a01c630a1d950d9a2d3857bffb Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 15:51:28 +0600 Subject: [PATCH 083/171] naming update --- .../rtd_tests/tests/test_doc_serving.py | 4 +-- readthedocs/rtd_tests/tests/test_managers.py | 34 +++++++++---------- readthedocs/rtd_tests/tests/test_project.py | 16 ++++----- .../rtd_tests/tests/test_project_forms.py | 6 ++-- .../rtd_tests/tests/test_project_views.py | 10 +++--- readthedocs/rtd_tests/tests/test_version.py | 12 +++---- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 8ddbca2a410..ea9a198986f 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -250,7 +250,7 @@ def test_sitemap_xml(self): active=True ) # This is a EXTERNAL Version - pr_version = fixture.get( + external_version = fixture.get( Version, identifier='pr-version', verbose_name='pr-version', @@ -317,7 +317,7 @@ def test_sitemap_xml(self): self.assertNotContains( response, self.public.get_docs_url( - version_slug=pr_version.slug, + version_slug=external_version.slug, lang_slug=self.public.language, private=True, ), diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 924544a8be8..1d5cbd78645 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -20,21 +20,21 @@ def setUp(self): self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') # Create a External Version. ie: pull/merge request Version. - self.public_pr_version = get( + self.public_external_version = get( Version, project=self.pip, active=True, type=EXTERNAL, privacy_level=PUBLIC ) - self.private_pr_version = get( + self.private_external_version = get( Version, project=self.pip, active=True, type=EXTERNAL, privacy_level=PRIVATE ) - self.protected_pr_version = get( + self.protected_external_version = get( Version, project=self.pip, active=True, @@ -54,28 +54,28 @@ class TestInternalVersionManager(TestVersionManagerBase): """ def test_internal_version_manager_with_all(self): - self.assertNotIn(self.public_pr_version, Version.internal.all()) + self.assertNotIn(self.public_external_version, Version.internal.all()) def test_internal_version_manager_with_public(self): - self.assertNotIn(self.public_pr_version, Version.internal.public()) + self.assertNotIn(self.public_external_version, Version.internal.public()) def test_internal_version_manager_with_public_with_user_and_project(self): self.assertNotIn( - self.public_pr_version, + self.public_external_version, Version.internal.public(self.user, self.pip) ) def test_internal_version_manager_with_protected(self): - self.assertNotIn(self.protected_pr_version, Version.internal.protected()) + self.assertNotIn(self.protected_external_version, Version.internal.protected()) def test_internal_version_manager_with_private(self): - self.assertNotIn(self.private_pr_version, Version.internal.private()) + self.assertNotIn(self.private_external_version, Version.internal.private()) def test_internal_version_manager_with_api(self): - self.assertNotIn(self.public_pr_version, Version.internal.api()) + self.assertNotIn(self.public_external_version, Version.internal.api()) def test_internal_version_manager_with_for_project(self): - self.assertNotIn(self.public_pr_version, Version.internal.for_project(self.pip)) + self.assertNotIn(self.public_external_version, Version.internal.for_project(self.pip)) class TestExternalVersionManager(TestVersionManagerBase): @@ -88,11 +88,11 @@ class TestExternalVersionManager(TestVersionManagerBase): def test_external_version_manager_with_all(self): self.assertNotIn(self.internal_versions, Version.external.all()) - self.assertIn(self.public_pr_version, Version.external.all()) + self.assertIn(self.public_external_version, Version.external.all()) def test_external_version_manager_with_public(self): self.assertNotIn(self.internal_versions, Version.external.public()) - self.assertIn(self.public_pr_version, Version.external.public()) + self.assertIn(self.public_external_version, Version.external.public()) def test_external_version_manager_with_public_with_user_and_project(self): self.assertNotIn( @@ -100,26 +100,26 @@ def test_external_version_manager_with_public_with_user_and_project(self): Version.external.public(self.user, self.pip) ) self.assertIn( - self.public_pr_version, + self.public_external_version, Version.external.public(self.user, self.pip) ) def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) - self.assertIn(self.protected_pr_version, Version.external.protected()) + self.assertIn(self.protected_external_version, Version.external.protected()) def test_external_version_manager_with_private(self): self.assertNotIn(self.internal_versions, Version.external.private()) - self.assertIn(self.private_pr_version, Version.external.private()) + self.assertIn(self.private_external_version, Version.external.private()) def test_external_version_manager_with_api(self): self.assertNotIn(self.internal_versions, Version.external.api()) - self.assertIn(self.public_pr_version, Version.external.api()) + self.assertIn(self.public_external_version, Version.external.api()) def test_external_version_manager_with_for_project(self): self.assertNotIn( self.internal_versions, Version.external.for_project(self.pip) ) self.assertIn( - self.public_pr_version, Version.external.for_project(self.pip) + self.public_external_version, Version.external.for_project(self.pip) ) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 5d5194d04d2..b7c0afa883a 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -31,7 +31,7 @@ def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') # Create a External Version. ie: pull/merge request Version. - self.pr_version = get( + self.external_version = get( Version, identifier='pr-version', verbose_name='pr-version', @@ -145,16 +145,16 @@ def test_get_storage_path(self): 'htmlzip/pip/latest/pip.zip', ) - def test_ordered_active_versions_excludes_pr_versions(self): - self.assertNotIn(self.pr_version, self.pip.ordered_active_versions()) + def test_ordered_active_versions_excludes_external_versions(self): + self.assertNotIn(self.external_version, self.pip.ordered_active_versions()) - def test_active_versions_excludes_pr_versions(self): - self.assertNotIn(self.pr_version, self.pip.active_versions()) + def test_active_versions_excludes_external_versions(self): + self.assertNotIn(self.external_version, self.pip.active_versions()) - def test_all_active_versions_excludes_pr_versions(self): - self.assertNotIn(self.pr_version, self.pip.all_active_versions()) + def test_all_active_versions_excludes_external_versions(self): + self.assertNotIn(self.external_version, self.pip.all_active_versions()) - def test_update_stable_version_excludes_pr_versions(self): + def test_update_stable_version_excludes_external_versions(self): # Delete all versions excluding PR Versions. self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for stable. diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index ba5b3c88b9d..9d2aaa01716 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -383,8 +383,8 @@ def test_commit_name_not_in_default_branch_choices(self): 'default_branch'].widget.choices], ) - def test_pr_version_not_in_default_branch_choices(self): - pr_version = get( + def test_external_version_not_in_default_branch_choices(self): + external_version = get( Version, identifier='pr-version', verbose_name='pr-version', @@ -397,7 +397,7 @@ def test_pr_version_not_in_default_branch_choices(self): form = ProjectAdvancedForm(instance=self.project) self.assertNotIn( - pr_version.verbose_name, + external_version.verbose_name, [identifier for identifier, _ in form.fields[ 'default_branch'].widget.choices], ) diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 3bc948aff9b..331f6982aa3 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -370,7 +370,7 @@ def test_import_demo_imported_duplicate(self): class TestPublicViews(MockBuildTestCase): def setUp(self): self.pip = get(Project, slug='pip') - self.pr_version = get( + self.external_version = get( Version, identifier='pr-version', verbose_name='pr-version', @@ -389,20 +389,20 @@ def test_project_detail_view_only_shows_internal_versons(self): url = reverse('projects_detail', args=[self.pip.slug]) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertNotIn(self.pr_version, response.context['versions']) + self.assertNotIn(self.external_version, response.context['versions']) def test_project_downloads_only_shows_internal_versons(self): url = reverse('project_downloads', args=[self.pip.slug]) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertNotIn(self.pr_version, response.context['versions']) + self.assertNotIn(self.external_version, response.context['versions']) def test_project_versions_only_shows_internal_versons(self): url = reverse('project_version_list', args=[self.pip.slug]) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertNotIn(self.pr_version, response.context['active_versions']) - self.assertNotIn(self.pr_version, response.context['inactive_versions']) + self.assertNotIn(self.external_version, response.context['active_versions']) + self.assertNotIn(self.external_version, response.context['inactive_versions']) class TestPrivateViews(MockBuildTestCase): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py index e414ba79cc6..c0055708db0 100644 --- a/readthedocs/rtd_tests/tests/test_version.py +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -14,7 +14,7 @@ def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') # Create a External Version. ie: pull/merge request Version. - self.pr_version = get( + self.external_version = get( Version, identifier='9F86D081884C7D659A2FEAA0C55AD015A', verbose_name='pr-version', @@ -45,9 +45,9 @@ def setUp(self): class TestVersionModel(VersionMixin, TestCase): - def test_vcs_url_for_pr_version(self): - expected_url = f'https://github.com/pypa/pip/pull/{self.pr_version.slug}/' - self.assertEqual(self.pr_version.vcs_url, expected_url) + def test_vcs_url_for_external_version(self): + expected_url = f'https://github.com/pypa/pip/pull/{self.external_version.slug}/' + self.assertEqual(self.external_version.vcs_url, expected_url) def test_vcs_url_for_latest_version(self): slug = self.pip.default_branch or self.pip.vcs_repo().fallback_branch @@ -64,5 +64,5 @@ def test_commit_name_for_stable_version(self): def test_commit_name_for_latest_version(self): self.assertEqual(self.tag_version.commit_name, 'master') - def test_commit_name_for_pr_version(self): - self.assertEqual(self.pr_version.commit_name, self.pr_version.identifier) + def test_commit_name_for_external_version(self): + self.assertEqual(self.external_version.commit_name, self.external_version.identifier) From 75efef807a8f8d7e340d83e8841ee9a09bae0a7c Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 16:32:29 +0600 Subject: [PATCH 084/171] naming updated --- .../rtd_tests/tests/test_doc_serving.py | 4 ++-- readthedocs/rtd_tests/tests/test_managers.py | 22 +++++++++---------- readthedocs/rtd_tests/tests/test_project.py | 2 +- readthedocs/rtd_tests/tests/test_views.py | 12 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 8809be15f59..379387aac5c 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -241,7 +241,7 @@ def test_sitemap_xml(self): active=True ) # This is a EXTERNAL Version - pr_version = fixture.get( + external_version = fixture.get( Version, identifier='pr-version', verbose_name='pr-version', @@ -327,7 +327,7 @@ def test_sitemap_xml(self): self.assertNotContains( response, self.public.get_docs_url( - version_slug=pr_version.slug, + version_slug=external_version.slug, lang_slug=self.public.language, private=True, ), diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 2e7364e35b3..a17aee9c7a7 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -135,17 +135,17 @@ def setUp(self): self.pip = Project.objects.get(slug='pip') print(self.pip.versions.all()) # Create a External Version and build. ie: pull/merge request Version. - self.pr_version = get( + self.external_version = get( Version, project=self.pip, active=True, type=EXTERNAL, privacy_level=PUBLIC ) - self.pr_version_build = get( + self.external_version_build = get( Build, project=self.pip, - version=self.pr_version + version=self.external_version ) # Create a Internal Version build. self.internal_version_build = get( @@ -167,19 +167,19 @@ class TestInternalBuildManager(TestBuildManagerBase): """ def test_internal_build_manager_with_all(self): - self.assertNotIn(self.pr_version_build, Build.internal.all()) + self.assertNotIn(self.external_version_build, Build.internal.all()) def test_internal_build_manager_with_public(self): - self.assertNotIn(self.pr_version_build, Build.internal.public()) + self.assertNotIn(self.external_version_build, Build.internal.public()) def test_internal_build_manager_with_public_with_user_and_project(self): self.assertNotIn( - self.pr_version_build, + self.external_version_build, Build.internal.public(self.user, self.pip) ) def test_internal_build_manager_with_api(self): - self.assertNotIn(self.pr_version_build, Build.internal.api()) + self.assertNotIn(self.external_version_build, Build.internal.api()) class TestExternalBuildManager(TestBuildManagerBase): @@ -192,11 +192,11 @@ class TestExternalBuildManager(TestBuildManagerBase): def test_external_build_manager_with_all(self): self.assertNotIn(self.internal_builds, Build.external.all()) - self.assertIn(self.pr_version_build, Build.external.all()) + self.assertIn(self.external_version_build, Build.external.all()) def test_external_build_manager_with_public(self): self.assertNotIn(self.internal_builds, Build.external.public()) - self.assertIn(self.pr_version_build, Build.external.public()) + self.assertIn(self.external_version_build, Build.external.public()) def test_external_build_manager_with_public_with_user_and_project(self): self.assertNotIn( @@ -204,10 +204,10 @@ def test_external_build_manager_with_public_with_user_and_project(self): Build.external.public(self.user, self.pip) ) self.assertIn( - self.pr_version_build, + self.external_version_build, Build.external.public(self.user, self.pip) ) def test_external_build_manager_with_api(self): self.assertNotIn(self.internal_builds, Build.external.api()) - self.assertIn(self.pr_version_build, Build.external.api()) + self.assertIn(self.external_version_build, Build.external.api()) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 023db2734d2..ded11f52c53 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -160,7 +160,7 @@ def test_update_stable_version_excludes_external_versions(self): # Test that PR Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) - def test_has_good_build_excludes_pr_versions(self): + def test_has_good_build_excludes_external_versions(self): # Delete all versions excluding PR Versions. self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for has_good_build. diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index 71d69ab2e6b..de77945b311 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -278,22 +278,22 @@ def test_build_redirect(self, mock): '/projects/pip/builds/%s/' % build.pk, ) - def test_build_list_does_not_include_pr_versions(self): - pr_version = get( + def test_build_list_does_not_include_external_versions(self): + external_version = get( Version, project = self.pip, active = True, type = EXTERNAL, ) - pr_version_build = get( + external_version_build = get( Build, project = self.pip, - version = pr_version + version = external_version ) response = self.client.get( reverse('builds_project_list', args=[self.pip.slug]), ) self.assertEqual(response.status_code, 200) - self.assertNotIn(pr_version_build, response.context['build_qs']) - self.assertNotIn(pr_version_build, response.context['active_builds']) + self.assertNotIn(external_version_build, response.context['build_qs']) + self.assertNotIn(external_version_build, response.context['active_builds']) From 3c13c3862e25b0e1dd9cbbaa30ae3ed4a701b895 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 16:37:37 +0600 Subject: [PATCH 085/171] naming update for External --- readthedocs/rtd_tests/tests/test_doc_serving.py | 2 +- readthedocs/rtd_tests/tests/test_managers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index dcd9433dd18..b8e334ac223 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -241,7 +241,7 @@ def test_sitemap_xml(self): active=True ) # This is a EXTERNAL Version - pr_version = fixture.get( + external_version = fixture.get( Version, identifier='pr-version', verbose_name='pr-version', diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 619f3e90e27..2e366799f15 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -134,7 +134,7 @@ def setUp(self): self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') # Create a External Version. ie: pull/merge request Version. - self.pr_version = get( + self.external_version = get( Version, project=self.pip, active=True, @@ -143,7 +143,7 @@ def setUp(self): ) self.html_file = HTMLFile.objects.create( project=self.pip, - version=self.pr_version, + version=self.external_version, name='file.html', slug='file', path='file.html', From 51188e7e7be8c4147f11247d7dbf278b65d24ef5 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 18:33:57 +0600 Subject: [PATCH 086/171] fix --- readthedocs/rtd_tests/tests/test_doc_serving.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 379387aac5c..734ed3338b1 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -240,16 +240,6 @@ def test_sitemap_xml(self): project=self.public, active=True ) - # This is a EXTERNAL Version - external_version = fixture.get( - Version, - identifier='pr-version', - verbose_name='pr-version', - slug='pr-9999', - project=self.public, - active=True, - type=EXTERNAL - ) stable_version = fixture.get( Version, identifier='stable', From 3bdf03cd4137a4dcb4179171ccdd7912ab6635ab Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 19:00:53 +0600 Subject: [PATCH 087/171] fix --- readthedocs/rtd_tests/tests/test_doc_serving.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index b8e334ac223..ea9a198986f 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -240,16 +240,6 @@ def test_sitemap_xml(self): project=self.public, active=True ) - # This is a EXTERNAL Version - external_version = fixture.get( - Version, - identifier='pr-version', - verbose_name='pr-version', - slug='pr-9999', - project=self.public, - active=True, - type=EXTERNAL - ) stable_version = fixture.get( Version, identifier='stable', From 9b9953887c69238c4dd4f1b216cc5533b53dab39 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 19 Jun 2019 00:27:17 +0600 Subject: [PATCH 088/171] vcs_url() update --- readthedocs/builds/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 34a3fd372a9..6432283c293 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -167,6 +167,7 @@ def vcs_url(self): if self.type == EXTERNAL: if 'github' in self.project.repo: + slug_url = self.verbose_name url = f'/pull/{slug_url}/' if 'gitlab' in self.project.repo: From cfd085caa91ac962884de4807882db23fb1dd00b Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 19 Jun 2019 00:30:39 +0600 Subject: [PATCH 089/171] tests updated --- readthedocs/rtd_tests/tests/test_version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py index c0055708db0..6ae29c3e241 100644 --- a/readthedocs/rtd_tests/tests/test_version.py +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -17,8 +17,8 @@ def setUp(self): self.external_version = get( Version, identifier='9F86D081884C7D659A2FEAA0C55AD015A', - verbose_name='pr-version', - slug='9999', + verbose_name='9999', + slug='pr-9999', project=self.pip, active=True, type=EXTERNAL @@ -46,7 +46,7 @@ def setUp(self): class TestVersionModel(VersionMixin, TestCase): def test_vcs_url_for_external_version(self): - expected_url = f'https://github.com/pypa/pip/pull/{self.external_version.slug}/' + expected_url = f'https://github.com/pypa/pip/pull/{self.external_version.verbose_name}/' self.assertEqual(self.external_version.vcs_url, expected_url) def test_vcs_url_for_latest_version(self): From 6fd5f682397cae319b10db19fa9c36d20d59236e Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 19 Jun 2019 13:13:14 +0600 Subject: [PATCH 090/171] lint fix --- readthedocs/rtd_tests/tests/test_managers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index e6cb6ac8a85..c9bb54020e1 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -161,6 +161,7 @@ class TestInternalBuildManager(TestBuildManagerBase): """ Queries using Internal Manager should only include Internal Version builds. + It will exclude pull/merge request Version builds from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ @@ -185,6 +186,7 @@ class TestExternalBuildManager(TestBuildManagerBase): """ Queries using External Manager should only include External Version builds. + It will only include pull/merge request Version builds in the queries. """ From be8f4de475b4a434ed9d595680025b06b9ff4bfc Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:07:54 -0700 Subject: [PATCH 091/171] Sync External versions to our database This is just an initial proof of concept. We still need to update the External type properly, and do lots of other work. This is just a start to show how we can sync and build them. --- readthedocs/projects/constants.py | 5 +++++ readthedocs/vcs_support/backends/git.py | 11 ++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 7c83cffd7ea..1a97a89a397 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -333,3 +333,8 @@ 'https://gitlab.com/{user}/{repo}/' '{action}/{version}{docroot}{path}{source_suffix}' ) + +DEFAULT_GIT_PATTERN = 'refs/heads/*:refs/remotes/origin/*' +GITHUB_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' +GITLAB_GIT_PATTERN = 'refs/merge-requests/*/head:refs/remotes/origin/external/*' +BITBUCKET_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index aa3adc6c2bc..5619ca44e13 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -11,6 +11,7 @@ from git.exc import BadName, InvalidGitRepositoryError from readthedocs.config import ALL +from readthedocs.projects.constants import GITHUB_GIT_PATTERN, DEFAULT_GIT_PATTERN from readthedocs.projects.exceptions import RepositoryError from readthedocs.projects.validators import validate_submodule_url from readthedocs.vcs_support.base import BaseVCS, VCSVersion @@ -55,9 +56,12 @@ def update(self): super().update() if self.repo_exists(): self.set_remote_url(self.repo_url) - return self.fetch() + self.fetch() + return self.make_clean_working_dir() - return self.clone() + self.clone() + # Do a fetch to make sure we get all external versions + self.fetch() def repo_exists(self): try: @@ -144,7 +148,8 @@ def use_shallow_clone(self): return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE) def fetch(self): - cmd = ['git', 'fetch', '--tags', '--prune', '--prune-tags'] + cmd = ['git', 'fetch', 'origin', DEFAULT_GIT_PATTERN, GITHUB_GIT_PATTERN, + '--tags', '--prune', '--prune-tags'] if self.use_shallow_clone(): cmd.extend(['--depth', str(self.repo_depth)]) From 30ac6bab3a2fea67e7b8b16090c6c2f505855c22 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:12:07 -0700 Subject: [PATCH 092/171] Remove unused patterns --- readthedocs/projects/constants.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 1a97a89a397..de8151b993d 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -335,6 +335,5 @@ ) DEFAULT_GIT_PATTERN = 'refs/heads/*:refs/remotes/origin/*' +# https://help.github.com/en/articles/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally GITHUB_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' -GITLAB_GIT_PATTERN = 'refs/merge-requests/*/head:refs/remotes/origin/external/*' -BITBUCKET_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' From df122daf4baecf6c28416d95a51f40f6b27bf39e Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:12:35 -0700 Subject: [PATCH 093/171] Small whitespace cleanup --- readthedocs/vcs_support/backends/git.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 5619ca44e13..c61fff5cc51 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -148,7 +148,8 @@ def use_shallow_clone(self): return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE) def fetch(self): - cmd = ['git', 'fetch', 'origin', DEFAULT_GIT_PATTERN, GITHUB_GIT_PATTERN, + cmd = ['git', 'fetch', 'origin', + DEFAULT_GIT_PATTERN, GITHUB_GIT_PATTERN, '--tags', '--prune', '--prune-tags'] if self.use_shallow_clone(): From da4f71d8ae3a731705c4be7c48d88211bb55a510 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:14:25 -0700 Subject: [PATCH 094/171] Clean up fetch logic a bit --- readthedocs/vcs_support/backends/git.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index c61fff5cc51..8bcb2c09a51 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -56,11 +56,9 @@ def update(self): super().update() if self.repo_exists(): self.set_remote_url(self.repo_url) - self.fetch() - return - self.make_clean_working_dir() - self.clone() - # Do a fetch to make sure we get all external versions + else: + self.make_clean_working_dir() + self.clone() self.fetch() def repo_exists(self): From 1207afb700e2c1534b3b7ce123b491616f981a3b Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 18 Jun 2019 12:16:55 -0700 Subject: [PATCH 095/171] Add Comment --- readthedocs/vcs_support/backends/git.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 8bcb2c09a51..1718b4401cc 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -59,6 +59,7 @@ def update(self): else: self.make_clean_working_dir() self.clone() + # A fetch is always required to get external versions properly self.fetch() def repo_exists(self): From c74d29414bcd047dc61db0d29eb3bdbea5eff307 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 20 Jun 2019 00:29:01 +0600 Subject: [PATCH 096/171] Initial changes for PR Builder --- readthedocs/api/v2/utils.py | 33 ++++++++++++++++++++++++ readthedocs/api/v2/views/integrations.py | 7 +++++ readthedocs/api/v2/views/model_views.py | 24 ++++++++++++++--- readthedocs/projects/tasks.py | 6 +++++ readthedocs/vcs_support/backends/git.py | 26 ++++++++++++++++--- 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/readthedocs/api/v2/utils.py b/readthedocs/api/v2/utils.py index a9fcffea6d8..30184cefa13 100644 --- a/readthedocs/api/v2/utils.py +++ b/readthedocs/api/v2/utils.py @@ -13,7 +13,9 @@ STABLE, STABLE_VERBOSE_NAME, TAG, + EXTERNAL, ) +from readthedocs.core.utils import trigger_build from readthedocs.builds.models import Version @@ -77,6 +79,15 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin version_name, version_id, ) + elif type == EXTERNAL: + created_version = Version.objects.create( + project=project, + type=type, + identifier=version_id, + verbose_name=version_name, + ) + added.add(created_version.slug) + else: # New Version created_version = Version.objects.create( @@ -135,6 +146,9 @@ def delete_versions(project, version_data): versions_tags = [ version['verbose_name'] for version in version_data.get('tags', []) ] + external_versions = [ + version['verbose_name'] for version in version_data.get('external_branches', []) + ] versions_branches = [ version['identifier'] for version in version_data.get('branches', []) ] @@ -147,6 +161,10 @@ def delete_versions(project, version_data): type=BRANCH, identifier__in=versions_branches, ) + to_delete_qs = to_delete_qs.exclude( + type=EXTERNAL, + verbose_name__in=external_versions, + ) to_delete_qs = to_delete_qs.exclude(uploaded=True) to_delete_qs = to_delete_qs.exclude(active=True) to_delete_qs = to_delete_qs.exclude(slug__in=NON_REPOSITORY_VERSIONS) @@ -176,6 +194,21 @@ def run_automation_rules(project, versions_slug): rule.run(version) +def trigger_external_build(project, version_list): + """ + Trigger Builds for all external versions provided. + + The rules are sorted by priority. + + """ + for version_slug in version_list: + version = project.versions(manager=EXTERNAL).get(slug=version_slug) + version.active = True + version.save() + + trigger_build(project=project, version=version) + + class RemoteOrganizationPagination(PageNumberPagination): page_size = 25 diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 4424238a78b..75979a450ba 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -29,6 +29,9 @@ GITHUB_EVENT_HEADER = 'HTTP_X_GITHUB_EVENT' GITHUB_SIGNATURE_HEADER = 'HTTP_X_HUB_SIGNATURE' GITHUB_PUSH = 'push' +GITHUB_PULL_REQUEST = 'pull_request' +GITHUB_PULL_REQUEST_OPEN = 'open' +GITHUB_PULL_REQUEST_SYNC = 'synchronize' GITHUB_CREATE = 'create' GITHUB_DELETE = 'delete' GITLAB_TOKEN_HEADER = 'HTTP_X_GITLAB_TOKEN' @@ -271,6 +274,10 @@ def handle_webhook(self): raise ParseError('Parameter "ref" is required') if event in (GITHUB_CREATE, GITHUB_DELETE): return self.sync_versions(self.project) + + if event == GITHUB_PULL_REQUEST: + return self.sync_versions(self.project) + return None def _normalize_ref(self, ref): diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index fca91b32626..ecc57a0a8f0 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG, INTERNAL +from readthedocs.builds.constants import BRANCH, TAG, INTERNAL, EXTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -189,6 +189,7 @@ def sync_versions(self, request, **kwargs): # noqa: D205 # Update All Versions data = request.data added_versions = set() + added_external_versions = set() if 'tags' in data: ret_set = api_utils.sync_versions( project=project, @@ -203,6 +204,15 @@ def sync_versions(self, request, **kwargs): # noqa: D205 type=BRANCH, ) added_versions.update(ret_set) + + if 'external_branches' in data: + ret_set = api_utils.sync_versions( + project=project, + versions=data['external_branches'], + type=EXTERNAL, + ) + added_external_versions.update(ret_set) + deleted_versions = api_utils.delete_versions(project, data) except Exception as e: log.exception('Sync Versions Error') @@ -213,11 +223,13 @@ def sync_versions(self, request, **kwargs): # noqa: D205 status=status.HTTP_400_BAD_REQUEST, ) + all_added_versions = added_versions | added_external_versions + try: # The order of added_versions isn't deterministic. # We don't track the commit time or any other metadata. # We usually have one version added per webhook. - api_utils.run_automation_rules(project, added_versions) + api_utils.run_automation_rules(project, all_added_versions) except Exception: # Don't interrupt the request if something goes wrong # in the automation rules. @@ -226,6 +238,12 @@ def sync_versions(self, request, **kwargs): # noqa: D205 project.slug, added_versions ) + if added_external_versions: + api_utils.trigger_external_build( + project=project, + version_list=added_external_versions + ) + # TODO: move this to an automation rule promoted_version = project.update_stable_version() new_stable = project.get_stable_version() @@ -250,7 +268,7 @@ def sync_versions(self, request, **kwargs): # noqa: D205 trigger_build(project=project, version=promoted_version) return Response({ - 'added_versions': added_versions, + 'added_versions': all_added_versions, 'deleted_versions': deleted_versions, }) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 7efcd5a762a..1bf721249cb 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -165,6 +165,12 @@ def sync_versions(self, version_repo): 'verbose_name': v.verbose_name, } for v in version_repo.branches] + if version_repo.supports_external_branches: + version_post_data['external_branches'] = [{ + 'identifier': v.identifier, + 'verbose_name': v.verbose_name, + } for v in version_repo.external_branches] + self.validate_duplicate_reserved_versions(version_post_data) try: diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 1718b4401cc..193ebc2d07a 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -26,6 +26,7 @@ class Backend(BaseVCS): supports_tags = True supports_branches = True + supports_external_branches = True supports_submodules = True fallback_branch = 'master' # default branch repo_depth = 50 @@ -213,11 +214,30 @@ def branches(self): for branch in branches: verbose_name = branch.name - if verbose_name.startswith('origin/'): + if not verbose_name.startswith('origin/external/'): + if verbose_name.startswith('origin/'): + verbose_name = verbose_name.replace('origin/', '') + if verbose_name == 'HEAD': + continue + versions.append(VCSVersion(self, str(branch), verbose_name)) + return versions + + @property + def external_branches(self): + repo = git.Repo(self.working_dir) + versions = [] + branches = [] + + # ``repo.remotes.origin.refs`` returns remote branches + if repo.remotes: + branches += repo.remotes.origin.refs + + for branch in branches: + verbose_name = branch.name + if verbose_name.startswith('origin/external/'): verbose_name = verbose_name.replace('origin/', '') - if verbose_name == 'HEAD': + versions.append(VCSVersion(self, str(branch.commit), verbose_name)) continue - versions.append(VCSVersion(self, str(branch), verbose_name)) return versions @property From e7c741711ea15fb1f8c89657eb3b6a1e98b84dec Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 20 Jun 2019 21:52:53 +0600 Subject: [PATCH 097/171] removed _default_class for BuildManager --- readthedocs/builds/managers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index d80852478f2..cbf0d96bc27 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -171,7 +171,6 @@ def get_queryset(self): class BuildManager(SettingsOverrideObject): _default_class = BuildManagerBase - _override_setting = 'BUILD_MANAGER' class InternalBuildManager(SettingsOverrideObject): From 5fb2e4eba6da0190180dcb626c47a4690758ed1c Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 21 Jun 2019 18:11:27 +0600 Subject: [PATCH 098/171] Only build external version from webhook payload --- readthedocs/api/v2/utils.py | 33 ------------------ readthedocs/api/v2/views/integrations.py | 34 +++++++++++++++--- readthedocs/api/v2/views/model_views.py | 26 +++----------- readthedocs/core/views/hooks.py | 20 +++++++++++ readthedocs/projects/constants.py | 4 +-- readthedocs/projects/tasks.py | 8 +---- readthedocs/vcs_support/backends/bzr.py | 2 +- readthedocs/vcs_support/backends/git.py | 44 ++++++++++-------------- readthedocs/vcs_support/backends/hg.py | 2 +- readthedocs/vcs_support/backends/svn.py | 2 +- 10 files changed, 77 insertions(+), 98 deletions(-) diff --git a/readthedocs/api/v2/utils.py b/readthedocs/api/v2/utils.py index 30184cefa13..a9fcffea6d8 100644 --- a/readthedocs/api/v2/utils.py +++ b/readthedocs/api/v2/utils.py @@ -13,9 +13,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, - EXTERNAL, ) -from readthedocs.core.utils import trigger_build from readthedocs.builds.models import Version @@ -79,15 +77,6 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin version_name, version_id, ) - elif type == EXTERNAL: - created_version = Version.objects.create( - project=project, - type=type, - identifier=version_id, - verbose_name=version_name, - ) - added.add(created_version.slug) - else: # New Version created_version = Version.objects.create( @@ -146,9 +135,6 @@ def delete_versions(project, version_data): versions_tags = [ version['verbose_name'] for version in version_data.get('tags', []) ] - external_versions = [ - version['verbose_name'] for version in version_data.get('external_branches', []) - ] versions_branches = [ version['identifier'] for version in version_data.get('branches', []) ] @@ -161,10 +147,6 @@ def delete_versions(project, version_data): type=BRANCH, identifier__in=versions_branches, ) - to_delete_qs = to_delete_qs.exclude( - type=EXTERNAL, - verbose_name__in=external_versions, - ) to_delete_qs = to_delete_qs.exclude(uploaded=True) to_delete_qs = to_delete_qs.exclude(active=True) to_delete_qs = to_delete_qs.exclude(slug__in=NON_REPOSITORY_VERSIONS) @@ -194,21 +176,6 @@ def run_automation_rules(project, versions_slug): rule.run(version) -def trigger_external_build(project, version_list): - """ - Trigger Builds for all external versions provided. - - The rules are sorted by priority. - - """ - for version_slug in version_list: - version = project.versions(manager=EXTERNAL).get(slug=version_slug) - version.active = True - version.save() - - trigger_build(project=project, version=version) - - class RemoteOrganizationPagination(PageNumberPagination): page_size = 25 diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 75979a450ba..d194623b7b9 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -19,7 +19,11 @@ webhook_github, webhook_gitlab, ) -from readthedocs.core.views.hooks import build_branches, sync_versions +from readthedocs.core.views.hooks import ( + build_branches, + sync_versions, + get_or_create_external_version, +) from readthedocs.integrations.models import HttpExchange, Integration from readthedocs.projects.models import Project @@ -30,7 +34,7 @@ GITHUB_SIGNATURE_HEADER = 'HTTP_X_HUB_SIGNATURE' GITHUB_PUSH = 'push' GITHUB_PULL_REQUEST = 'pull_request' -GITHUB_PULL_REQUEST_OPEN = 'open' +GITHUB_PULL_REQUEST_OPEN = 'opened' GITHUB_PULL_REQUEST_SYNC = 'synchronize' GITHUB_CREATE = 'create' GITHUB_DELETE = 'delete' @@ -113,6 +117,10 @@ def handle_webhook(self): """Handle webhook payload.""" raise NotImplementedError + def get_external_version_data(self): + """Get External Version data from payload.""" + raise NotImplementedError + def is_payload_valid(self): """Validates the webhook's payload using the integration's secret.""" return False @@ -221,6 +229,13 @@ def get_data(self): pass return super().get_data() + def get_external_version_data(self): + """Get Commit Sha and pull request number from payload""" + identifier = self.data['pull_request']['head']['sha'] + verbose_name = str(self.data['number']) + + return identifier, verbose_name + def is_payload_valid(self): """ GitHub use a HMAC hexdigest hash to sign the payload. @@ -275,8 +290,19 @@ def handle_webhook(self): if event in (GITHUB_CREATE, GITHUB_DELETE): return self.sync_versions(self.project) - if event == GITHUB_PULL_REQUEST: - return self.sync_versions(self.project) + if ( + event == GITHUB_PULL_REQUEST and + self.data['action'] in [GITHUB_PULL_REQUEST_OPEN, GITHUB_PULL_REQUEST_SYNC] + ): + try: + identifier, verbose_name = self.get_external_version_data() + external_version = get_or_create_external_version( + self.project, identifier, verbose_name + ) + return self.get_response_push(self.project, external_version.verbose_name) + + except KeyError: + raise ParseError('Parameters "sha" and "number" are required') return None diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index ecc57a0a8f0..af41f28ab84 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG, INTERNAL, EXTERNAL +from readthedocs.builds.constants import BRANCH, TAG from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions(manager=INTERNAL).filter(active=True) + versions = project.versions.filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) @@ -189,7 +189,6 @@ def sync_versions(self, request, **kwargs): # noqa: D205 # Update All Versions data = request.data added_versions = set() - added_external_versions = set() if 'tags' in data: ret_set = api_utils.sync_versions( project=project, @@ -204,15 +203,6 @@ def sync_versions(self, request, **kwargs): # noqa: D205 type=BRANCH, ) added_versions.update(ret_set) - - if 'external_branches' in data: - ret_set = api_utils.sync_versions( - project=project, - versions=data['external_branches'], - type=EXTERNAL, - ) - added_external_versions.update(ret_set) - deleted_versions = api_utils.delete_versions(project, data) except Exception as e: log.exception('Sync Versions Error') @@ -223,13 +213,11 @@ def sync_versions(self, request, **kwargs): # noqa: D205 status=status.HTTP_400_BAD_REQUEST, ) - all_added_versions = added_versions | added_external_versions - try: # The order of added_versions isn't deterministic. # We don't track the commit time or any other metadata. # We usually have one version added per webhook. - api_utils.run_automation_rules(project, all_added_versions) + api_utils.run_automation_rules(project, added_versions) except Exception: # Don't interrupt the request if something goes wrong # in the automation rules. @@ -238,12 +226,6 @@ def sync_versions(self, request, **kwargs): # noqa: D205 project.slug, added_versions ) - if added_external_versions: - api_utils.trigger_external_build( - project=project, - version_list=added_external_versions - ) - # TODO: move this to an automation rule promoted_version = project.update_stable_version() new_stable = project.get_stable_version() @@ -268,7 +250,7 @@ def sync_versions(self, request, **kwargs): # noqa: D205 trigger_build(project=project, version=promoted_version) return Response({ - 'added_versions': all_added_versions, + 'added_versions': added_versions, 'deleted_versions': deleted_versions, }) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index a3273a88871..bd5b8c40cb0 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -2,6 +2,8 @@ import logging +from readthedocs.builds.constants import EXTERNAL +from readthedocs.builds.models import Version from readthedocs.core.utils import trigger_build from readthedocs.projects.tasks import sync_repository_task @@ -88,3 +90,21 @@ def sync_versions(project): except Exception: log.exception('Unknown sync versions exception') return None + + +def get_or_create_external_version(project, identifier, verbose_name): + external_version = project.versions(manager=EXTERNAL).filter(verbose_name=verbose_name).first() + if external_version: + if external_version.identifier != identifier: + external_version.identifier = identifier + external_version.save() + else: + created_external_version = Version.objects.create( + project=project, + type=EXTERNAL, + identifier=identifier, + verbose_name=verbose_name, + active=True + ) + return created_external_version + return external_version diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index de8151b993d..8b770830f79 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -334,6 +334,4 @@ '{action}/{version}{docroot}{path}{source_suffix}' ) -DEFAULT_GIT_PATTERN = 'refs/heads/*:refs/remotes/origin/*' -# https://help.github.com/en/articles/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally -GITHUB_GIT_PATTERN = 'refs/pull/*/head:refs/remotes/origin/external/*' +GITHUB_GIT_PATTERN = 'pull/{id}/head:external-{id}' diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 1bf721249cb..2d0e087b2a2 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -138,7 +138,7 @@ def sync_repo(self): } ) version_repo = self.get_vcs_repo() - version_repo.update() + version_repo.update(version=self.version) self.sync_versions(version_repo) version_repo.checkout(self.version.identifier) finally: @@ -165,12 +165,6 @@ def sync_versions(self, version_repo): 'verbose_name': v.verbose_name, } for v in version_repo.branches] - if version_repo.supports_external_branches: - version_post_data['external_branches'] = [{ - 'identifier': v.identifier, - 'verbose_name': v.verbose_name, - } for v in version_repo.external_branches] - self.validate_duplicate_reserved_versions(version_post_data) try: diff --git a/readthedocs/vcs_support/backends/bzr.py b/readthedocs/vcs_support/backends/bzr.py index e228ac720d3..99029d6dc82 100644 --- a/readthedocs/vcs_support/backends/bzr.py +++ b/readthedocs/vcs_support/backends/bzr.py @@ -17,7 +17,7 @@ class Backend(BaseVCS): supports_tags = True fallback_branch = '' - def update(self): + def update(self, version=None): # pylint: disable=arguments-differ super().update() retcode = self.run('bzr', 'status', record=False)[0] if retcode == 0: diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 193ebc2d07a..ba265fe3704 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -10,8 +10,9 @@ from django.core.exceptions import ValidationError from git.exc import BadName, InvalidGitRepositoryError +from readthedocs.builds.constants import EXTERNAL from readthedocs.config import ALL -from readthedocs.projects.constants import GITHUB_GIT_PATTERN, DEFAULT_GIT_PATTERN +from readthedocs.projects.constants import GITHUB_GIT_PATTERN from readthedocs.projects.exceptions import RepositoryError from readthedocs.projects.validators import validate_submodule_url from readthedocs.vcs_support.base import BaseVCS, VCSVersion @@ -52,16 +53,21 @@ def _get_clone_url(self): def set_remote_url(self, url): return self.run('git', 'remote', 'set-url', 'origin', url) - def update(self): + def update(self, version=None): # pylint: disable=arguments-differ """Clone or update the repository.""" super().update() if self.repo_exists(): self.set_remote_url(self.repo_url) - else: - self.make_clean_working_dir() - self.clone() + # A fetch is always required to get external versions properly + if version and version.type == EXTERNAL: + return self.fetch(version.verbose_name) + return self.fetch() + self.make_clean_working_dir() # A fetch is always required to get external versions properly - self.fetch() + if version and version.type == EXTERNAL: + self.clone() + return self.fetch(version.verbose_name) + return self.clone() def repo_exists(self): try: @@ -147,11 +153,15 @@ def use_shallow_clone(self): from readthedocs.projects.models import Feature return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE) - def fetch(self): + def fetch(self, verbose_name=None): cmd = ['git', 'fetch', 'origin', - DEFAULT_GIT_PATTERN, GITHUB_GIT_PATTERN, '--tags', '--prune', '--prune-tags'] + if verbose_name and 'github.com' in self.repo_url: + cmd.append( + GITHUB_GIT_PATTERN.format(id=verbose_name) + ) + if self.use_shallow_clone(): cmd.extend(['--depth', str(self.repo_depth)]) @@ -222,24 +232,6 @@ def branches(self): versions.append(VCSVersion(self, str(branch), verbose_name)) return versions - @property - def external_branches(self): - repo = git.Repo(self.working_dir) - versions = [] - branches = [] - - # ``repo.remotes.origin.refs`` returns remote branches - if repo.remotes: - branches += repo.remotes.origin.refs - - for branch in branches: - verbose_name = branch.name - if verbose_name.startswith('origin/external/'): - verbose_name = verbose_name.replace('origin/', '') - versions.append(VCSVersion(self, str(branch.commit), verbose_name)) - continue - return versions - @property def commit(self): if self.repo_exists(): diff --git a/readthedocs/vcs_support/backends/hg.py b/readthedocs/vcs_support/backends/hg.py index 0361bfa462c..9b44faeed2e 100644 --- a/readthedocs/vcs_support/backends/hg.py +++ b/readthedocs/vcs_support/backends/hg.py @@ -13,7 +13,7 @@ class Backend(BaseVCS): supports_branches = True fallback_branch = 'default' - def update(self): + def update(self, version=None): # pylint: disable=arguments-differ super().update() retcode = self.run('hg', 'status', record=False)[0] if retcode == 0: diff --git a/readthedocs/vcs_support/backends/svn.py b/readthedocs/vcs_support/backends/svn.py index b1a945aec2c..50bab45a512 100644 --- a/readthedocs/vcs_support/backends/svn.py +++ b/readthedocs/vcs_support/backends/svn.py @@ -27,7 +27,7 @@ def __init__(self, *args, **kwargs): else: self.base_url = self.repo_url - def update(self): + def update(self, version=None): # pylint: disable=arguments-differ super().update() # For some reason `svn status` gives me retcode 0 in non-svn # directories that's why I use `svn info` here. From a859eacc5d331c679c9722cee521aec66d2ffb48 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 21 Jun 2019 18:17:19 +0600 Subject: [PATCH 099/171] fix --- readthedocs/api/v2/views/model_views.py | 4 ++-- readthedocs/vcs_support/backends/git.py | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index af41f28ab84..fca91b32626 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG +from readthedocs.builds.constants import BRANCH, TAG, INTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions.filter(active=True) + versions = project.versions(manager=INTERNAL).filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index ba265fe3704..506143132ed 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -224,12 +224,11 @@ def branches(self): for branch in branches: verbose_name = branch.name - if not verbose_name.startswith('origin/external/'): - if verbose_name.startswith('origin/'): - verbose_name = verbose_name.replace('origin/', '') - if verbose_name == 'HEAD': - continue - versions.append(VCSVersion(self, str(branch), verbose_name)) + if verbose_name.startswith('origin/'): + verbose_name = verbose_name.replace('origin/', '') + if verbose_name == 'HEAD': + continue + versions.append(VCSVersion(self, str(branch), verbose_name)) return versions @property From 5b883df743276aebb41c1765e54f04c2f2a3ab03 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 21 Jun 2019 18:50:37 +0600 Subject: [PATCH 100/171] need to pass verbose_name as a list for get_response_push --- readthedocs/api/v2/views/integrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index d194623b7b9..1567e71b18e 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -299,7 +299,7 @@ def handle_webhook(self): external_version = get_or_create_external_version( self.project, identifier, verbose_name ) - return self.get_response_push(self.project, external_version.verbose_name) + return self.get_response_push(self.project, [external_version.verbose_name]) except KeyError: raise ParseError('Parameters "sha" and "number" are required') From 31041a3eec8ada1837f1fc454cb596f27d1d5f61 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 22 Jun 2019 03:04:17 +0600 Subject: [PATCH 101/171] add all versions and builds to version and build listing --- readthedocs/builds/views.py | 2 +- readthedocs/projects/views/public.py | 4 ++-- readthedocs/rtd_tests/tests/test_project_views.py | 11 ++--------- readthedocs/rtd_tests/tests/test_views.py | 6 +++--- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index 479d0797458..ab1c7dd5b5b 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -92,7 +92,7 @@ def get_queryset(self): Project.objects.protected(self.request.user), slug=self.project_slug, ) - queryset = Build.internal.public( + queryset = Build.objects.public( user=self.request.user, project=self.project, ).select_related('project', 'version') diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index ea15bfb168b..174e618c9aa 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) project = self.get_object() - context['versions'] = Version.internal.public( + context['versions'] = Version.objects.public( user=self.request.user, project=project, ) @@ -268,7 +268,7 @@ def project_versions(request, project_slug): slug=project_slug, ) - versions = Version.internal.public( + versions = Version.objects.public( user=request.user, project=project, only_active=False, diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 331f6982aa3..817d2bf55a5 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -385,11 +385,11 @@ def test_project_download_media(self): response = self.client.get(url) self.assertEqual(response.status_code, 302) - def test_project_detail_view_only_shows_internal_versons(self): + def test_project_detail_view_shows_external_versons(self): url = reverse('projects_detail', args=[self.pip.slug]) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertNotIn(self.external_version, response.context['versions']) + self.assertIn(self.external_version, response.context['versions']) def test_project_downloads_only_shows_internal_versons(self): url = reverse('project_downloads', args=[self.pip.slug]) @@ -397,13 +397,6 @@ def test_project_downloads_only_shows_internal_versons(self): self.assertEqual(response.status_code, 200) self.assertNotIn(self.external_version, response.context['versions']) - def test_project_versions_only_shows_internal_versons(self): - url = reverse('project_version_list', args=[self.pip.slug]) - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertNotIn(self.external_version, response.context['active_versions']) - self.assertNotIn(self.external_version, response.context['inactive_versions']) - class TestPrivateViews(MockBuildTestCase): def setUp(self): diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index de77945b311..e94232f5da2 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -278,7 +278,7 @@ def test_build_redirect(self, mock): '/projects/pip/builds/%s/' % build.pk, ) - def test_build_list_does_not_include_external_versions(self): + def test_build_list_includes_external_versions(self): external_version = get( Version, project = self.pip, @@ -295,5 +295,5 @@ def test_build_list_does_not_include_external_versions(self): ) self.assertEqual(response.status_code, 200) - self.assertNotIn(external_version_build, response.context['build_qs']) - self.assertNotIn(external_version_build, response.context['active_builds']) + self.assertIn(external_version_build, response.context['build_qs']) + self.assertIn(external_version_build, response.context['active_builds']) From 9cf8ec39a2fddbac128e89485a110efe0aa28957 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 22 Jun 2019 03:10:23 +0600 Subject: [PATCH 102/171] fix tests --- readthedocs/rtd_tests/tests/test_views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index e94232f5da2..cdc83e4b19b 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -296,4 +296,3 @@ def test_build_list_includes_external_versions(self): self.assertEqual(response.status_code, 200) self.assertIn(external_version_build, response.context['build_qs']) - self.assertIn(external_version_build, response.context['active_builds']) From 711ed9a26805c2c2dcf7908aad6fb25a576fcd8a Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 22 Jun 2019 03:16:24 +0600 Subject: [PATCH 103/171] fix tests --- readthedocs/rtd_tests/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index 3ee7820d24d..3b443b2a246 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -928,7 +928,7 @@ def test_github_invalid_webhook(self, trigger_build): '/api/v2/webhook/github/{}/'.format(self.project.slug), {'foo': 'bar'}, format='json', - HTTP_X_GITHUB_EVENT='pull_request', + HTTP_X_GITHUB_EVENT='issues', ) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.data['detail'], 'Unhandled webhook event') From 65697cb492c0806105ca57348aea9406a6c2077c Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 22 Jun 2019 20:54:22 +0600 Subject: [PATCH 104/171] refine webhook handling --- readthedocs/api/v2/views/integrations.py | 50 +++++++++++++++++------- readthedocs/core/views/hooks.py | 15 ++++++- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 1567e71b18e..d497cc4fc96 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -34,7 +34,9 @@ GITHUB_SIGNATURE_HEADER = 'HTTP_X_HUB_SIGNATURE' GITHUB_PUSH = 'push' GITHUB_PULL_REQUEST = 'pull_request' -GITHUB_PULL_REQUEST_OPEN = 'opened' +GITHUB_PULL_REQUEST_OPENED = 'opened' +GITHUB_PULL_REQUEST_CLOSED = 'closed' +GITHUB_PULL_REQUEST_REOPENED = 'reopened' GITHUB_PULL_REQUEST_SYNC = 'synchronize' GITHUB_CREATE = 'create' GITHUB_DELETE = 'delete' @@ -229,6 +231,14 @@ def get_data(self): pass return super().get_data() + def get_action(self): + """Get Pull Request Action Data. ie: opened, closed, synchronize, reopened""" + try: + return self.data['action'] + except KeyError: + return None + + def get_external_version_data(self): """Get Commit Sha and pull request number from payload""" identifier = self.data['pull_request']['head']['sha'] @@ -290,19 +300,31 @@ def handle_webhook(self): if event in (GITHUB_CREATE, GITHUB_DELETE): return self.sync_versions(self.project) - if ( - event == GITHUB_PULL_REQUEST and - self.data['action'] in [GITHUB_PULL_REQUEST_OPEN, GITHUB_PULL_REQUEST_SYNC] - ): - try: - identifier, verbose_name = self.get_external_version_data() - external_version = get_or_create_external_version( - self.project, identifier, verbose_name - ) - return self.get_response_push(self.project, [external_version.verbose_name]) - - except KeyError: - raise ParseError('Parameters "sha" and "number" are required') + if event == GITHUB_PULL_REQUEST and self.get_action(): + if ( + self.get_action() in + [ + GITHUB_PULL_REQUEST_OPENED, + GITHUB_PULL_REQUEST_REOPENED, + GITHUB_PULL_REQUEST_SYNC + ] + ): + # Handle opened, synchronize, reopened pull_request event. + try: + identifier, verbose_name = self.get_external_version_data() + # create or get external version object using `verbose_name`. + external_version = get_or_create_external_version( + self.project, identifier, verbose_name + ) + # send the external version instance to `self.get_response_push()`. + return self.get_response_push(self.project, [external_version]) + + except KeyError: + raise ParseError('Parameters "sha" and "number" are required') + + if self.get_action() == GITHUB_PULL_REQUEST_CLOSED: + # Handle closed pull_request event. + pass return None diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index bd5b8c40cb0..124134d1dc5 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -48,7 +48,13 @@ def build_branches(project, branch_list): to_build = set() not_building = set() for branch in branch_list: - versions = project.versions_from_branch_name(branch) + # Used for External Versions + # pull/merge request `handle_webhook()` sends version instance for Building. + if isinstance(branch, Version): + versions = [branch] + else: + versions = project.versions_from_branch_name(branch) + for version in versions: log.info( '(Branch Build) Processing %s:%s', @@ -93,12 +99,17 @@ def sync_versions(project): def get_or_create_external_version(project, identifier, verbose_name): - external_version = project.versions(manager=EXTERNAL).filter(verbose_name=verbose_name).first() + external_version = project.versions( + manager=EXTERNAL + ).filter(verbose_name=verbose_name).first() + if external_version: + # identifier will change if there is a new commit to the Pull/Merge Request if external_version.identifier != identifier: external_version.identifier = identifier external_version.save() else: + # create an external version if the version does not exist. created_external_version = Version.objects.create( project=project, type=EXTERNAL, From 7fa5a01310becf06903cbd5c39b30d6e6d01b653 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 22 Jun 2019 21:06:54 +0600 Subject: [PATCH 105/171] lint fix --- readthedocs/api/v2/views/integrations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index d497cc4fc96..022b1441d53 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -238,7 +238,6 @@ def get_action(self): except KeyError: return None - def get_external_version_data(self): """Get Commit Sha and pull request number from payload""" identifier = self.data['pull_request']['head']['sha'] From 5718ea7e8846679ee867e2f9f2805174a1c9658b Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 22 Jun 2019 23:41:00 +0600 Subject: [PATCH 106/171] Doc string updated --- readthedocs/api/v2/views/integrations.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 022b1441d53..2700fc84aa0 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -208,16 +208,29 @@ class GitHubWebhookView(WebhookMixin, APIView): Expects the following JSON:: - { - "ref": "branch-name", - ... - } + For push, create, delete Events: + { + "ref": "branch-name", + ... + } + + For pull_request Events: + { + "action": "opened", + "number": 2, + "pull_request": { + "head": { + "sha": "ec26de721c3235aad62de7213c562f8c821" + } + } + } See full payload here: - https://developer.github.com/v3/activity/events/types/#pushevent - https://developer.github.com/v3/activity/events/types/#createevent - https://developer.github.com/v3/activity/events/types/#deleteevent + - https://developer.github.com/v3/activity/events/types/#pullrequestevent """ integration_type = Integration.GITHUB_WEBHOOK From 5663dc39c936f67497dbafc410800ef235f19621 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sun, 23 Jun 2019 00:06:01 +0600 Subject: [PATCH 107/171] Doc string updated --- readthedocs/api/v2/views/integrations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 2700fc84aa0..832c74595af 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -202,9 +202,9 @@ class GitHubWebhookView(WebhookMixin, APIView): """ Webhook consumer for GitHub. - Accepts webhook events from GitHub, 'push' events trigger builds. Expects the - webhook event type will be included in HTTP header ``X-GitHub-Event``, and - we will have a JSON payload. + Accepts webhook events from GitHub, 'push' and 'pull_request' events trigger builds. + Expects the webhook event type will be included in HTTP header ``X-GitHub-Event``, + and we will have a JSON payload. Expects the following JSON:: From e5d9725ed0a029c2c5af75f4452d65c14e16c52e Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Mon, 24 Jun 2019 22:14:33 +0600 Subject: [PATCH 108/171] Updated Action --- readthedocs/api/v2/views/integrations.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 832c74595af..e61c2c366ec 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -295,6 +295,7 @@ def get_digest(secret, msg): def handle_webhook(self): # Get event and trigger other webhook events + action = self.get_action() event = self.request.META.get(GITHUB_EVENT_HEADER, GITHUB_PUSH) webhook_github.send( Project, @@ -312,9 +313,9 @@ def handle_webhook(self): if event in (GITHUB_CREATE, GITHUB_DELETE): return self.sync_versions(self.project) - if event == GITHUB_PULL_REQUEST and self.get_action(): + if event == GITHUB_PULL_REQUEST and action: if ( - self.get_action() in + action in [ GITHUB_PULL_REQUEST_OPENED, GITHUB_PULL_REQUEST_REOPENED, @@ -334,7 +335,7 @@ def handle_webhook(self): except KeyError: raise ParseError('Parameters "sha" and "number" are required') - if self.get_action() == GITHUB_PULL_REQUEST_CLOSED: + if action == GITHUB_PULL_REQUEST_CLOSED: # Handle closed pull_request event. pass From 3427564ef8995e7450da65741b20f537952f71f3 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 25 Jun 2019 21:20:36 +0600 Subject: [PATCH 109/171] Pull Request Closed webhook handled --- readthedocs/api/v2/views/integrations.py | 24 +++++++++++++++++++++++- readthedocs/core/views/hooks.py | 24 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index e61c2c366ec..ec1f59050f8 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -23,6 +23,7 @@ build_branches, sync_versions, get_or_create_external_version, + delete_external_version, ) from readthedocs.integrations.models import HttpExchange, Integration from readthedocs.projects.models import Project @@ -196,6 +197,19 @@ def sync_versions(self, project): 'versions': [version], } + def delete_external_version(self, project, identifier, verbose_name): + # if external version exists returns + # verbose name (Pull/Merge Request Number) + # else returns None + deleted_version = delete_external_version( + project, identifier, verbose_name + ) + return { + 'version_deleted': deleted_version is not None, + 'project': project.slug, + 'versions': [deleted_version], + } + class GitHubWebhookView(WebhookMixin, APIView): @@ -337,7 +351,15 @@ def handle_webhook(self): if action == GITHUB_PULL_REQUEST_CLOSED: # Handle closed pull_request event. - pass + try: + identifier, verbose_name = self.get_external_version_data() + # Delete external version object if exists using `verbose_name`. + return self.delete_external_version( + self.project, identifier, verbose_name + ) + + except KeyError: + raise ParseError('Parameters "sha" and "number" are required') return None diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 124134d1dc5..5152f344665 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -117,5 +117,29 @@ def get_or_create_external_version(project, identifier, verbose_name): verbose_name=verbose_name, active=True ) + log.info( + '(Create External Version) Added Version: [%s] ', ' '.join( + created_external_version.slug + ) + ) return created_external_version return external_version + + +def delete_external_version(project, identifier, verbose_name): + external_version = project.versions(manager=EXTERNAL).filter( + verbose_name=verbose_name, identifier=identifier + ).first() + + if external_version: + verbose_name = external_version.verbose_name + # Delete External Version + external_version.delete() + log.info( + '(Delete External Version) Deleted Version: [%s]', ' '.join( + external_version.slug + ) + ) + + return verbose_name + return None From 2f739f0d6d34986e8c2bd1be37c96b850b36d6b5 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 26 Jun 2019 16:52:42 +0600 Subject: [PATCH 110/171] More refining --- readthedocs/api/v2/views/integrations.py | 51 +++++++++++++----------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index ec1f59050f8..a3a88843bb7 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -197,10 +197,31 @@ def sync_versions(self, project): 'versions': [version], } - def delete_external_version(self, project, identifier, verbose_name): - # if external version exists returns - # verbose name (Pull/Merge Request Number) - # else returns None + def get_external_version_response(self, project): + try: + # get identifier (commit hash) and verbose name (pull/merge request number) + # from webhook payload + identifier, verbose_name = self.get_external_version_data() + except KeyError: + raise ParseError('Parameters "sha" and "number" are required') + # create or get external version object using `verbose_name`. + external_version = get_or_create_external_version( + project, identifier, verbose_name + ) + # send the external version instance to `self.get_response_push()`. + return self.get_response_push(project, [external_version]) + + def delete_external_version(self, project): + try: + # get identifier (commit hash) and verbose name (pull/merge request number) + # from webhook payload + identifier, verbose_name = self.get_external_version_data() + except KeyError: + raise ParseError('Parameters "sha" and "number" are required') + # if external version exists + # Delete external version and return + # verbose name (Pull/Merge Request Number) of the deleted version + # else return None deleted_version = delete_external_version( project, identifier, verbose_name ) @@ -337,29 +358,11 @@ def handle_webhook(self): ] ): # Handle opened, synchronize, reopened pull_request event. - try: - identifier, verbose_name = self.get_external_version_data() - # create or get external version object using `verbose_name`. - external_version = get_or_create_external_version( - self.project, identifier, verbose_name - ) - # send the external version instance to `self.get_response_push()`. - return self.get_response_push(self.project, [external_version]) - - except KeyError: - raise ParseError('Parameters "sha" and "number" are required') + return self.get_external_version_response(self.project) if action == GITHUB_PULL_REQUEST_CLOSED: # Handle closed pull_request event. - try: - identifier, verbose_name = self.get_external_version_data() - # Delete external version object if exists using `verbose_name`. - return self.delete_external_version( - self.project, identifier, verbose_name - ) - - except KeyError: - raise ParseError('Parameters "sha" and "number" are required') + return self.delete_external_version(self.project) return None From 9fbb839c55f9e911b96a29151b33a6460b94b33e Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 27 Jun 2019 17:04:01 +0600 Subject: [PATCH 111/171] tests added --- readthedocs/core/views/hooks.py | 2 +- readthedocs/rtd_tests/tests/test_api.py | 211 +++++++++++++++++- readthedocs/rtd_tests/tests/test_backend.py | 26 +++ .../tests/test_version_commit_name.py | 17 +- readthedocs/vcs_support/backends/bzr.py | 2 +- readthedocs/vcs_support/backends/git.py | 3 +- readthedocs/vcs_support/backends/hg.py | 2 +- readthedocs/vcs_support/backends/svn.py | 2 +- readthedocs/vcs_support/base.py | 2 +- 9 files changed, 258 insertions(+), 9 deletions(-) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 5152f344665..5a898b40bae 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -141,5 +141,5 @@ def delete_external_version(project, identifier, verbose_name): ) ) - return verbose_name + return external_version.verbose_name return None diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index 3b443b2a246..429cc3827e8 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -18,6 +18,11 @@ GITHUB_EVENT_HEADER, GITHUB_PUSH, GITHUB_SIGNATURE_HEADER, + GITHUB_PULL_REQUEST, + GITHUB_PULL_REQUEST_OPENED, + GITHUB_PULL_REQUEST_REOPENED, + GITHUB_PULL_REQUEST_CLOSED, + GITHUB_PULL_REQUEST_SYNC, GITLAB_NULL_HASH, GITLAB_PUSH, GITLAB_TAG_PUSH, @@ -26,7 +31,7 @@ GitLabWebhookView, ) from readthedocs.api.v2.views.task_views import get_status_data -from readthedocs.builds.constants import LATEST +from readthedocs.builds.constants import LATEST, EXTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.integrations.models import Integration from readthedocs.oauth.models import RemoteOrganization, RemoteRepository @@ -774,6 +779,15 @@ def setUp(self): self.github_payload = { 'ref': 'master', } + self.github_pull_request_payload = { + "action": "opened", + "number": 2, + "pull_request": { + "head": { + "sha": "ec26de721c3235aad62de7213c562f8c821" + } + } + } self.gitlab_payload = { 'object_kind': GITLAB_PUSH, 'ref': 'master', @@ -892,6 +906,201 @@ def test_github_create_event(self, sync_repository_task, trigger_build): latest_version = self.project.versions.get(slug=LATEST) sync_repository_task.delay.assert_called_with(latest_version.pk) + def test_github_pull_request_opened_event(self, trigger_build): + client = APIClient() + + headers = {GITHUB_EVENT_HEADER: GITHUB_PULL_REQUEST} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + self.github_pull_request_payload, + format='json', + **headers + ) + # get the created external version + external_version = self.project.versions( + manager=EXTERNAL + ).get(verbose_name='2') + + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertTrue(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [external_version.slug]) + trigger_build.assert_called_once_with( + force=True, project=self.project, version=external_version + ) + self.assertTrue(external_version) + + def test_github_pull_request_reopened_event(self, trigger_build): + client = APIClient() + + # Update the payload for `reopened` webhook event + pull_request_number = '5' + payload = self.github_pull_request_payload + payload["action"] = GITHUB_PULL_REQUEST_REOPENED + payload["number"] = pull_request_number + + headers = {GITHUB_EVENT_HEADER: GITHUB_PULL_REQUEST} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + payload, + format='json', + **headers + ) + # get the created external version + external_version = self.project.versions( + manager=EXTERNAL + ).get(verbose_name=pull_request_number) + + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertTrue(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [external_version.slug]) + trigger_build.assert_called_once_with( + force=True, project=self.project, version=external_version + ) + self.assertTrue(external_version) + + def test_github_pull_request_synchronize_event(self, trigger_build): + client = APIClient() + + pull_request_number = '6' + prev_identifier = '95790bf891e76fee5e1747ab589903a6a1f80f23' + # create an existing external version for pull request + version = get( + Version, + project=self.project, + type=EXTERNAL, + built=True, + uploaded=True, + active=True, + verbose_name=pull_request_number, + identifier=prev_identifier + ) + + # Update the payload for `synchronize` webhook event + payload = self.github_pull_request_payload + payload["action"] = GITHUB_PULL_REQUEST_SYNC + payload["number"] = pull_request_number + + headers = {GITHUB_EVENT_HEADER: GITHUB_PULL_REQUEST} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + payload, + format='json', + **headers + ) + # get updated external version + external_version = self.project.versions( + manager=EXTERNAL + ).get(verbose_name=pull_request_number) + + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertTrue(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [external_version.slug]) + trigger_build.assert_called_once_with( + force=True, project=self.project, version=external_version + ) + # `synchronize` webhook event updated the identifier (commit hash) + self.assertNotEqual(prev_identifier, external_version.identifier) + + def test_github_pull_request_closed_event(self, trigger_build): + client = APIClient() + + pull_request_number = '7' + identifier = '95790bf891e76fee5e1747ab589903a6a1f80f23' + # create an existing external version for pull request + version = get( + Version, + project=self.project, + type=EXTERNAL, + built=True, + uploaded=True, + active=True, + verbose_name=pull_request_number, + identifier=identifier + ) + + # Update the payload for `closed` webhook event + payload = self.github_pull_request_payload + payload["action"] = GITHUB_PULL_REQUEST_CLOSED + payload["number"] = pull_request_number + payload["pull_request"]["head"]["sha"] = identifier + + headers = {GITHUB_EVENT_HEADER: GITHUB_PULL_REQUEST} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + payload, + format='json', + **headers + ) + external_version = self.project.versions( + manager=EXTERNAL + ).filter(verbose_name=pull_request_number) + + # external version should be deleted + self.assertFalse(external_version.exists()) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertTrue(resp.data['version_deleted']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [version.verbose_name]) + trigger_build.assert_not_called() + + def test_github_pull_request_no_action(self, trigger_build): + client = APIClient() + + payload = { + "number": 2, + "pull_request": { + "head": { + "sha": "ec26de721c3235aad62de7213c562f8c821" + } + } + } + headers = {GITHUB_EVENT_HEADER: GITHUB_PULL_REQUEST} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + payload, + format='json', + **headers + ) + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.data['detail'], 'Unhandled webhook event') + + def test_github_pull_request_opened_event_invalid_payload(self, trigger_build): + client = APIClient() + + payload = { + "action": GITHUB_PULL_REQUEST_OPENED, + "number": 2, + } + headers = {GITHUB_EVENT_HEADER: GITHUB_PULL_REQUEST} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + payload, + format='json', + **headers + ) + + self.assertEqual(resp.status_code, 400) + + def test_github_pull_request_closed_event_invalid_payload(self, trigger_build): + client = APIClient() + + payload = { + "action": GITHUB_PULL_REQUEST_CLOSED, + "number": 2, + } + headers = {GITHUB_EVENT_HEADER: GITHUB_PULL_REQUEST} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + payload, + format='json', + **headers + ) + + self.assertEqual(resp.status_code, 400) + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') def test_github_delete_event(self, sync_repository_task, trigger_build): client = APIClient() diff --git a/readthedocs/rtd_tests/tests/test_backend.py b/readthedocs/rtd_tests/tests/test_backend.py index 5618dd92151..2de4b9b83b7 100644 --- a/readthedocs/rtd_tests/tests/test_backend.py +++ b/readthedocs/rtd_tests/tests/test_backend.py @@ -8,6 +8,8 @@ from django.contrib.auth.models import User from mock import Mock, patch +from readthedocs.builds.constants import EXTERNAL +from readthedocs.builds.models import Version from readthedocs.config import ALL from readthedocs.projects.exceptions import RepositoryError from readthedocs.projects.models import Feature, Project @@ -108,6 +110,30 @@ def test_git_update_and_checkout(self): self.assertEqual(code, 0) self.assertTrue(exists(repo.working_dir)) + @patch('readthedocs.vcs_support.backends.git.Backend.fetch') + def test_git_update_with_external_version(self, fetch): + version = fixture.get( + Version, + project=self.project, + type=EXTERNAL, + active=True + ) + repo = self.project.vcs_repo() + repo.update(version) + fetch.assert_called_with(version.verbose_name) + + def test_git_fetch_with_external_version(self): + version = fixture.get( + Version, + project=self.project, + type=EXTERNAL, + active=True + ) + repo = self.project.vcs_repo() + repo.update() + code, _, _ = repo.fetch(version.verbose_name) + self.assertEqual(code, 0) + def test_git_checkout_invalid_revision(self): repo = self.project.vcs_repo() repo.update() diff --git a/readthedocs/rtd_tests/tests/test_version_commit_name.py b/readthedocs/rtd_tests/tests/test_version_commit_name.py index bf181c3d5da..f7367865c9e 100644 --- a/readthedocs/rtd_tests/tests/test_version_commit_name.py +++ b/readthedocs/rtd_tests/tests/test_version_commit_name.py @@ -2,7 +2,13 @@ from django.test import TestCase from django_dynamic_fixture import get, new -from readthedocs.builds.constants import BRANCH, LATEST, STABLE, TAG +from readthedocs.builds.constants import ( + BRANCH, + LATEST, + STABLE, + TAG, + EXTERNAL, +) from readthedocs.builds.models import Version from readthedocs.projects.constants import REPO_TYPE_GIT, REPO_TYPE_HG from readthedocs.projects.models import Project @@ -72,3 +78,12 @@ def test_git_latest_branch(self): verbose_name=LATEST, type=BRANCH, ) self.assertEqual(version.commit_name, 'master') + + def test_external_version(self): + identifier = 'ec26de721c3235aad62de7213c562f8c821' + version = new( + Version, identifier=identifier, + slug='11', verbose_name='11', + type=EXTERNAL, + ) + self.assertEqual(version.commit_name, identifier) diff --git a/readthedocs/vcs_support/backends/bzr.py b/readthedocs/vcs_support/backends/bzr.py index 99029d6dc82..dd166c72310 100644 --- a/readthedocs/vcs_support/backends/bzr.py +++ b/readthedocs/vcs_support/backends/bzr.py @@ -17,7 +17,7 @@ class Backend(BaseVCS): supports_tags = True fallback_branch = '' - def update(self, version=None): # pylint: disable=arguments-differ + def update(self, version=None): super().update() retcode = self.run('bzr', 'status', record=False)[0] if retcode == 0: diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 506143132ed..0c2d169f8dd 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -27,7 +27,6 @@ class Backend(BaseVCS): supports_tags = True supports_branches = True - supports_external_branches = True supports_submodules = True fallback_branch = 'master' # default branch repo_depth = 50 @@ -53,7 +52,7 @@ def _get_clone_url(self): def set_remote_url(self, url): return self.run('git', 'remote', 'set-url', 'origin', url) - def update(self, version=None): # pylint: disable=arguments-differ + def update(self, version=None): """Clone or update the repository.""" super().update() if self.repo_exists(): diff --git a/readthedocs/vcs_support/backends/hg.py b/readthedocs/vcs_support/backends/hg.py index 9b44faeed2e..4a43f213250 100644 --- a/readthedocs/vcs_support/backends/hg.py +++ b/readthedocs/vcs_support/backends/hg.py @@ -13,7 +13,7 @@ class Backend(BaseVCS): supports_branches = True fallback_branch = 'default' - def update(self, version=None): # pylint: disable=arguments-differ + def update(self, version=None): super().update() retcode = self.run('hg', 'status', record=False)[0] if retcode == 0: diff --git a/readthedocs/vcs_support/backends/svn.py b/readthedocs/vcs_support/backends/svn.py index 50bab45a512..cc2792e1113 100644 --- a/readthedocs/vcs_support/backends/svn.py +++ b/readthedocs/vcs_support/backends/svn.py @@ -27,7 +27,7 @@ def __init__(self, *args, **kwargs): else: self.base_url = self.repo_url - def update(self, version=None): # pylint: disable=arguments-differ + def update(self, version=None): super().update() # For some reason `svn status` gives me retcode 0 in non-svn # directories that's why I use `svn info` here. diff --git a/readthedocs/vcs_support/base.py b/readthedocs/vcs_support/base.py index 653629110fc..392b50385c1 100644 --- a/readthedocs/vcs_support/base.py +++ b/readthedocs/vcs_support/base.py @@ -81,7 +81,7 @@ def env(self): return environment - def update(self): + def update(self, version=None): """ Update a local copy of the repository in self.working_dir. From cdd38a1537210fe89191769eb1a74de1266d6179 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 27 Jun 2019 21:33:01 +0600 Subject: [PATCH 112/171] vcs_repo updated --- readthedocs/projects/tasks.py | 2 +- readthedocs/rtd_tests/tests/test_backend.py | 10 +++++----- readthedocs/rtd_tests/tests/test_footer.py | 2 +- readthedocs/vcs_support/backends/bzr.py | 2 +- readthedocs/vcs_support/backends/git.py | 14 ++++++-------- readthedocs/vcs_support/backends/hg.py | 2 +- readthedocs/vcs_support/backends/svn.py | 2 +- readthedocs/vcs_support/base.py | 3 ++- 8 files changed, 18 insertions(+), 19 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 2d0e087b2a2..7efcd5a762a 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -138,7 +138,7 @@ def sync_repo(self): } ) version_repo = self.get_vcs_repo() - version_repo.update(version=self.version) + version_repo.update() self.sync_versions(version_repo) version_repo.checkout(self.version.identifier) finally: diff --git a/readthedocs/rtd_tests/tests/test_backend.py b/readthedocs/rtd_tests/tests/test_backend.py index 2de4b9b83b7..e08ced06172 100644 --- a/readthedocs/rtd_tests/tests/test_backend.py +++ b/readthedocs/rtd_tests/tests/test_backend.py @@ -118,9 +118,9 @@ def test_git_update_with_external_version(self, fetch): type=EXTERNAL, active=True ) - repo = self.project.vcs_repo() - repo.update(version) - fetch.assert_called_with(version.verbose_name) + repo = self.project.vcs_repo(version=version.slug) + repo.update() + fetch.assert_called_once() def test_git_fetch_with_external_version(self): version = fixture.get( @@ -129,9 +129,9 @@ def test_git_fetch_with_external_version(self): type=EXTERNAL, active=True ) - repo = self.project.vcs_repo() + repo = self.project.vcs_repo(version=version.slug) repo.update() - code, _, _ = repo.fetch(version.verbose_name) + code, _, _ = repo.fetch() self.assertEqual(code, 0) def test_git_checkout_invalid_revision(self): diff --git a/readthedocs/rtd_tests/tests/test_footer.py b/readthedocs/rtd_tests/tests/test_footer.py index 241bc434f59..36c81c2cd89 100644 --- a/readthedocs/rtd_tests/tests/test_footer.py +++ b/readthedocs/rtd_tests/tests/test_footer.py @@ -235,7 +235,7 @@ class TestFooterPerformance(APITestCase): # The expected number of queries for generating the footer # This shouldn't increase unless we modify the footer API - EXPECTED_QUERIES = 9 + EXPECTED_QUERIES = 11 def setUp(self): self.pip = Project.objects.get(slug='pip') diff --git a/readthedocs/vcs_support/backends/bzr.py b/readthedocs/vcs_support/backends/bzr.py index dd166c72310..e228ac720d3 100644 --- a/readthedocs/vcs_support/backends/bzr.py +++ b/readthedocs/vcs_support/backends/bzr.py @@ -17,7 +17,7 @@ class Backend(BaseVCS): supports_tags = True fallback_branch = '' - def update(self, version=None): + def update(self): super().update() retcode = self.run('bzr', 'status', record=False)[0] if retcode == 0: diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 0c2d169f8dd..f9ec17867b5 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -52,20 +52,18 @@ def _get_clone_url(self): def set_remote_url(self, url): return self.run('git', 'remote', 'set-url', 'origin', url) - def update(self, version=None): + def update(self): """Clone or update the repository.""" super().update() if self.repo_exists(): self.set_remote_url(self.repo_url) # A fetch is always required to get external versions properly - if version and version.type == EXTERNAL: - return self.fetch(version.verbose_name) return self.fetch() self.make_clean_working_dir() # A fetch is always required to get external versions properly - if version and version.type == EXTERNAL: + if self.version and self.version.type == EXTERNAL: self.clone() - return self.fetch(version.verbose_name) + return self.fetch() return self.clone() def repo_exists(self): @@ -152,13 +150,13 @@ def use_shallow_clone(self): from readthedocs.projects.models import Feature return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE) - def fetch(self, verbose_name=None): + def fetch(self): cmd = ['git', 'fetch', 'origin', '--tags', '--prune', '--prune-tags'] - if verbose_name and 'github.com' in self.repo_url: + if self.version.type == EXTERNAL and 'github.com' in self.repo_url: cmd.append( - GITHUB_GIT_PATTERN.format(id=verbose_name) + GITHUB_GIT_PATTERN.format(id=self.version.verbose_name) ) if self.use_shallow_clone(): diff --git a/readthedocs/vcs_support/backends/hg.py b/readthedocs/vcs_support/backends/hg.py index 4a43f213250..0361bfa462c 100644 --- a/readthedocs/vcs_support/backends/hg.py +++ b/readthedocs/vcs_support/backends/hg.py @@ -13,7 +13,7 @@ class Backend(BaseVCS): supports_branches = True fallback_branch = 'default' - def update(self, version=None): + def update(self): super().update() retcode = self.run('hg', 'status', record=False)[0] if retcode == 0: diff --git a/readthedocs/vcs_support/backends/svn.py b/readthedocs/vcs_support/backends/svn.py index cc2792e1113..b1a945aec2c 100644 --- a/readthedocs/vcs_support/backends/svn.py +++ b/readthedocs/vcs_support/backends/svn.py @@ -27,7 +27,7 @@ def __init__(self, *args, **kwargs): else: self.base_url = self.repo_url - def update(self, version=None): + def update(self): super().update() # For some reason `svn status` gives me retcode 0 in non-svn # directories that's why I use `svn info` here. diff --git a/readthedocs/vcs_support/base.py b/readthedocs/vcs_support/base.py index 392b50385c1..0546895b273 100644 --- a/readthedocs/vcs_support/base.py +++ b/readthedocs/vcs_support/base.py @@ -56,6 +56,7 @@ def __init__(self, project, version_slug, environment=None, **kwargs): self.name = project.name self.repo_url = project.clean_repo self.working_dir = project.checkout_path(version_slug) + self.version = project.versions.filter(slug=version_slug).first() from readthedocs.doc_builder.environments import LocalEnvironment self.environment = environment or LocalEnvironment(project) @@ -81,7 +82,7 @@ def env(self): return environment - def update(self, version=None): + def update(self): """ Update a local copy of the repository in self.working_dir. From 7ef5175f7333562cda1f6bb0244fcfc708c1a8b0 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 27 Jun 2019 21:36:47 +0600 Subject: [PATCH 113/171] vcs_repo more update --- readthedocs/vcs_support/backends/git.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index f9ec17867b5..7bf810ed427 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -154,7 +154,11 @@ def fetch(self): cmd = ['git', 'fetch', 'origin', '--tags', '--prune', '--prune-tags'] - if self.version.type == EXTERNAL and 'github.com' in self.repo_url: + if ( + self.version and + self.version.type == EXTERNAL and + 'github.com' in self.repo_url + ): cmd.append( GITHUB_GIT_PATTERN.format(id=self.version.verbose_name) ) From 30390fe56164032ab209ef4e6ae6eae385694a40 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 27 Jun 2019 23:03:41 +0600 Subject: [PATCH 114/171] updated event handling --- readthedocs/api/v2/views/integrations.py | 12 ++++++++-- readthedocs/core/views/hooks.py | 28 +++++++++++++++++++----- readthedocs/rtd_tests/tests/test_api.py | 6 ++--- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index a3a88843bb7..d0819d61e81 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -24,6 +24,7 @@ sync_versions, get_or_create_external_version, delete_external_version, + build_external_version, ) from readthedocs.integrations.models import HttpExchange, Integration from readthedocs.projects.models import Project @@ -208,8 +209,15 @@ def get_external_version_response(self, project): external_version = get_or_create_external_version( project, identifier, verbose_name ) - # send the external version instance to `self.get_response_push()`. - return self.get_response_push(project, [external_version]) + # Build External Version + # returns external version verbose_name (pull/merge request number) + to_build = build_external_version(project, external_version) + + return { + 'build_triggered': True, + 'project': project.slug, + 'versions': [to_build], + } def delete_external_version(self, project): try: diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 5a898b40bae..e567e45f051 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -48,12 +48,7 @@ def build_branches(project, branch_list): to_build = set() not_building = set() for branch in branch_list: - # Used for External Versions - # pull/merge request `handle_webhook()` sends version instance for Building. - if isinstance(branch, Version): - versions = [branch] - else: - versions = project.versions_from_branch_name(branch) + versions = project.versions_from_branch_name(branch) for version in versions: log.info( @@ -143,3 +138,24 @@ def delete_external_version(project, identifier, verbose_name): return external_version.verbose_name return None + + +def build_external_version(project, version): + """ + Where we actually trigger builds for external versions. + + All pull/merge request webhook logic should route here to call ``trigger_build``. + """ + if not project.has_valid_webhook: + project.has_valid_webhook = True + project.save() + + # Build External version + log.info( + '(External Version build) Building %s:%s', + project.slug, + version.slug, + ) + trigger_build(project=project, version=version, force=True) + + return version.verbose_name diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index 429cc3827e8..c1a78d496da 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -924,7 +924,7 @@ def test_github_pull_request_opened_event(self, trigger_build): self.assertEqual(resp.status_code, status.HTTP_200_OK) self.assertTrue(resp.data['build_triggered']) self.assertEqual(resp.data['project'], self.project.slug) - self.assertEqual(resp.data['versions'], [external_version.slug]) + self.assertEqual(resp.data['versions'], [external_version.verbose_name]) trigger_build.assert_called_once_with( force=True, project=self.project, version=external_version ) @@ -954,7 +954,7 @@ def test_github_pull_request_reopened_event(self, trigger_build): self.assertEqual(resp.status_code, status.HTTP_200_OK) self.assertTrue(resp.data['build_triggered']) self.assertEqual(resp.data['project'], self.project.slug) - self.assertEqual(resp.data['versions'], [external_version.slug]) + self.assertEqual(resp.data['versions'], [external_version.verbose_name]) trigger_build.assert_called_once_with( force=True, project=self.project, version=external_version ) @@ -997,7 +997,7 @@ def test_github_pull_request_synchronize_event(self, trigger_build): self.assertEqual(resp.status_code, status.HTTP_200_OK) self.assertTrue(resp.data['build_triggered']) self.assertEqual(resp.data['project'], self.project.slug) - self.assertEqual(resp.data['versions'], [external_version.slug]) + self.assertEqual(resp.data['versions'], [external_version.verbose_name]) trigger_build.assert_called_once_with( force=True, project=self.project, version=external_version ) From 694a27bcfda78ffb40a9b986affabdb3aa12472e Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 28 Jun 2019 03:04:45 +0600 Subject: [PATCH 115/171] vcs_repo method updated --- readthedocs/projects/models.py | 10 ++++++++-- readthedocs/projects/tasks.py | 2 ++ readthedocs/rtd_tests/tests/test_backend.py | 10 ++++++++-- readthedocs/rtd_tests/tests/test_footer.py | 2 +- readthedocs/vcs_support/backends/git.py | 9 ++++----- readthedocs/vcs_support/base.py | 9 +++++++-- 6 files changed, 30 insertions(+), 12 deletions(-) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 20945715121..50cb6f7a50b 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -808,7 +808,10 @@ def has_htmlzip(self, version_slug=LATEST): def sponsored(self): return False - def vcs_repo(self, version=LATEST, environment=None): + def vcs_repo( + self, version=LATEST, environment=None, + verbose_name=None, version_type=None + ): """ Return a Backend object for this project able to handle VCS commands. @@ -825,7 +828,10 @@ def vcs_repo(self, version=LATEST, environment=None): if not backend: repo = None else: - repo = backend(self, version, environment) + repo = backend( + self, version, environment, + verbose_name, version_type + ) return repo def repo_nonblockinglock(self, version, max_lock_age=None): diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 7efcd5a762a..5877e3e82df 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -106,6 +106,8 @@ def get_vcs_repo(self): # a ``setup_env`` so we use just ``None`` and commands won't # be recorded getattr(self, 'setup_env', None), + verbose_name=self.version.verbose_name, + version_type=self.version.type ) return version_repo diff --git a/readthedocs/rtd_tests/tests/test_backend.py b/readthedocs/rtd_tests/tests/test_backend.py index e08ced06172..43f8419a0b5 100644 --- a/readthedocs/rtd_tests/tests/test_backend.py +++ b/readthedocs/rtd_tests/tests/test_backend.py @@ -118,7 +118,10 @@ def test_git_update_with_external_version(self, fetch): type=EXTERNAL, active=True ) - repo = self.project.vcs_repo(version=version.slug) + repo = self.project.vcs_repo( + verbose_name=version.verbose_name, + version_type=version.type + ) repo.update() fetch.assert_called_once() @@ -129,7 +132,10 @@ def test_git_fetch_with_external_version(self): type=EXTERNAL, active=True ) - repo = self.project.vcs_repo(version=version.slug) + repo = self.project.vcs_repo( + verbose_name=version.verbose_name, + version_type=version.type + ) repo.update() code, _, _ = repo.fetch() self.assertEqual(code, 0) diff --git a/readthedocs/rtd_tests/tests/test_footer.py b/readthedocs/rtd_tests/tests/test_footer.py index 36c81c2cd89..241bc434f59 100644 --- a/readthedocs/rtd_tests/tests/test_footer.py +++ b/readthedocs/rtd_tests/tests/test_footer.py @@ -235,7 +235,7 @@ class TestFooterPerformance(APITestCase): # The expected number of queries for generating the footer # This shouldn't increase unless we modify the footer API - EXPECTED_QUERIES = 11 + EXPECTED_QUERIES = 9 def setUp(self): self.pip = Project.objects.get(slug='pip') diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 7bf810ed427..3095517ae41 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -61,7 +61,7 @@ def update(self): return self.fetch() self.make_clean_working_dir() # A fetch is always required to get external versions properly - if self.version and self.version.type == EXTERNAL: + if self.version_type and self.version_type == EXTERNAL: self.clone() return self.fetch() return self.clone() @@ -155,12 +155,11 @@ def fetch(self): '--tags', '--prune', '--prune-tags'] if ( - self.version and - self.version.type == EXTERNAL and - 'github.com' in self.repo_url + self.version_type and self.verbose_name and + self.version_type == EXTERNAL and 'github.com' in self.repo_url ): cmd.append( - GITHUB_GIT_PATTERN.format(id=self.version.verbose_name) + GITHUB_GIT_PATTERN.format(id=self.verbose_name) ) if self.use_shallow_clone(): diff --git a/readthedocs/vcs_support/base.py b/readthedocs/vcs_support/base.py index 0546895b273..1b7da2017d0 100644 --- a/readthedocs/vcs_support/base.py +++ b/readthedocs/vcs_support/base.py @@ -50,13 +50,18 @@ class BaseVCS: # Defining a base API, so we'll have unused args # pylint: disable=unused-argument - def __init__(self, project, version_slug, environment=None, **kwargs): + def __init__( + self, project, version_slug, environment=None, + verbose_name=None, version_type=None, **kwargs + ): self.default_branch = project.default_branch self.project = project self.name = project.name self.repo_url = project.clean_repo self.working_dir = project.checkout_path(version_slug) - self.version = project.versions.filter(slug=version_slug).first() + # required for External versions + self.verbose_name = verbose_name + self.version_type = version_type from readthedocs.doc_builder.environments import LocalEnvironment self.environment = environment or LocalEnvironment(project) From d721fc069af306086680c275dbe3197f98515755 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 28 Jun 2019 03:33:17 +0600 Subject: [PATCH 116/171] fix --- readthedocs/vcs_support/backends/git.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 3095517ae41..34556bf78c0 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -61,7 +61,7 @@ def update(self): return self.fetch() self.make_clean_working_dir() # A fetch is always required to get external versions properly - if self.version_type and self.version_type == EXTERNAL: + if self.version_type == EXTERNAL: self.clone() return self.fetch() return self.clone() @@ -155,8 +155,9 @@ def fetch(self): '--tags', '--prune', '--prune-tags'] if ( - self.version_type and self.verbose_name and - self.version_type == EXTERNAL and 'github.com' in self.repo_url + self.verbose_name and + self.version_type == EXTERNAL and + 'github.com' in self.repo_url ): cmd.append( GITHUB_GIT_PATTERN.format(id=self.verbose_name) From 1692dfbbdb38e08465c8f47f0aa0e648b3a7f591 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 28 Jun 2019 03:42:32 +0600 Subject: [PATCH 117/171] use named args --- readthedocs/projects/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 50cb6f7a50b..3ec2457776a 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -829,8 +829,8 @@ def vcs_repo( repo = None else: repo = backend( - self, version, environment, - verbose_name, version_type + self, version, environment=environment, + verbose_name=verbose_name, version_type=version_type ) return repo From 7b2bc6c2478e2d0e343f39690d8dec11c6957370 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 28 Jun 2019 04:49:53 +0600 Subject: [PATCH 118/171] Docstring added and some changes added --- readthedocs/api/v2/views/integrations.py | 71 ++++++++++++++---------- readthedocs/core/views/hooks.py | 23 +++++++- 2 files changed, 63 insertions(+), 31 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index d0819d61e81..2a9d715531e 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -199,17 +199,25 @@ def sync_versions(self, project): } def get_external_version_response(self, project): - try: - # get identifier (commit hash) and verbose name (pull/merge request number) - # from webhook payload - identifier, verbose_name = self.get_external_version_data() - except KeyError: - raise ParseError('Parameters "sha" and "number" are required') + """ + Build External version on pull/merge request events and return API response. + + Return a JSON response with the following:: + + { + "build_triggered": true, + "project": "project_name", + "versions": [verbose_name] + } + + :param project: Project instance + :type project: Project + """ + identifier, verbose_name = self.get_external_version_data() # create or get external version object using `verbose_name`. external_version = get_or_create_external_version( project, identifier, verbose_name ) - # Build External Version # returns external version verbose_name (pull/merge request number) to_build = build_external_version(project, external_version) @@ -219,17 +227,23 @@ def get_external_version_response(self, project): 'versions': [to_build], } - def delete_external_version(self, project): - try: - # get identifier (commit hash) and verbose name (pull/merge request number) - # from webhook payload - identifier, verbose_name = self.get_external_version_data() - except KeyError: - raise ParseError('Parameters "sha" and "number" are required') - # if external version exists - # Delete external version and return - # verbose name (Pull/Merge Request Number) of the deleted version - # else return None + def get_delete_external_version_response(self, project): + """ + Delete External version on pull/merge request `closed` events and return API response. + + Return a JSON response with the following:: + + { + "version_deleted": true, + "project": "project_name", + "versions": [verbose_name] + } + + :param project: Project instance + :type project: Project + """ + identifier, verbose_name = self.get_external_version_data() + # Delete external version deleted_version = delete_external_version( project, identifier, verbose_name ) @@ -287,19 +301,16 @@ def get_data(self): pass return super().get_data() - def get_action(self): - """Get Pull Request Action Data. ie: opened, closed, synchronize, reopened""" - try: - return self.data['action'] - except KeyError: - return None - def get_external_version_data(self): """Get Commit Sha and pull request number from payload""" - identifier = self.data['pull_request']['head']['sha'] - verbose_name = str(self.data['number']) + try: + identifier = self.data['pull_request']['head']['sha'] + verbose_name = str(self.data['number']) + + return identifier, verbose_name - return identifier, verbose_name + except KeyError: + raise ParseError('Parameters "sha" and "number" are required') def is_payload_valid(self): """ @@ -338,7 +349,7 @@ def get_digest(secret, msg): def handle_webhook(self): # Get event and trigger other webhook events - action = self.get_action() + action = self.data.get('action', None) event = self.request.META.get(GITHUB_EVENT_HEADER, GITHUB_PUSH) webhook_github.send( Project, @@ -370,7 +381,7 @@ def handle_webhook(self): if action == GITHUB_PULL_REQUEST_CLOSED: # Handle closed pull_request event. - return self.delete_external_version(self.project) + return self.get_delete_external_version_response(self.project) return None diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index e567e45f051..1e6ac93e362 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -94,6 +94,17 @@ def sync_versions(project): def get_or_create_external_version(project, identifier, verbose_name): + """ + Get or create external versions using `identifier` and `verbose_name`. + + if external version does not exist create an external version + + :param project: Project instance + :param identifier: Commit Hash + :param verbose_name: pull/merge request number + :returns: External version. + :rtype: Version + """ external_version = project.versions( manager=EXTERNAL ).filter(verbose_name=verbose_name).first() @@ -122,12 +133,22 @@ def get_or_create_external_version(project, identifier, verbose_name): def delete_external_version(project, identifier, verbose_name): + """ + Delete external versions using `identifier` and `verbose_name`. + + if external version does not exist then returns `None`. + + :param project: Project instance + :param identifier: Commit Hash + :param verbose_name: pull/merge request number + :returns: verbose_name (pull/merge request number). + :rtype: str + """ external_version = project.versions(manager=EXTERNAL).filter( verbose_name=verbose_name, identifier=identifier ).first() if external_version: - verbose_name = external_version.verbose_name # Delete External Version external_version.delete() log.info( From 64003052439c840553889dd89b3202469d400610 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 4 Jul 2019 19:11:23 +0600 Subject: [PATCH 119/171] migrations updated --- ...al_version_type.py => 0009_added_external_version_type.py} | 4 ++-- tox.ini | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) rename readthedocs/builds/migrations/{0008_added_external_version_type.py => 0009_added_external_version_type.py} (88%) diff --git a/readthedocs/builds/migrations/0008_added_external_version_type.py b/readthedocs/builds/migrations/0009_added_external_version_type.py similarity index 88% rename from readthedocs/builds/migrations/0008_added_external_version_type.py rename to readthedocs/builds/migrations/0009_added_external_version_type.py index 8de9c4f504f..ece2f2fb25c 100644 --- a/readthedocs/builds/migrations/0008_added_external_version_type.py +++ b/readthedocs/builds/migrations/0009_added_external_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.21 on 2019-06-17 19:43 +# Generated by Django 1.11.21 on 2019-07-04 13:10 from __future__ import unicode_literals from django.db import migrations, models @@ -8,7 +8,7 @@ class Migration(migrations.Migration): dependencies = [ - ('builds', '0007_add-automation-rules'), + ('builds', '0008_remove-version-tags'), ] operations = [ diff --git a/tox.ini b/tox.ini index 8383c86eeba..6e3256ee27c 100644 --- a/tox.ini +++ b/tox.ini @@ -71,3 +71,4 @@ commands = description = upload coverage report deps = codecov commands = codecov +added_external_version_type \ No newline at end of file From 04d4dca42847b26769afa6112ed818942e1d85ae Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 4 Jul 2019 19:13:47 +0600 Subject: [PATCH 120/171] fix typo --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6e3256ee27c..8383c86eeba 100644 --- a/tox.ini +++ b/tox.ini @@ -71,4 +71,3 @@ commands = description = upload coverage report deps = codecov commands = codecov -added_external_version_type \ No newline at end of file From 40fb6eebd73e0dc4eabe231f3f2cec9fc265549d Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 29 Jun 2019 03:08:41 +0600 Subject: [PATCH 121/171] Send Build Status using Github Status API --- readthedocs/builds/constants.py | 31 +++++++++ readthedocs/builds/models.py | 8 +++ readthedocs/core/utils/__init__.py | 11 +++- readthedocs/doc_builder/environments.py | 2 +- readthedocs/oauth/services/base.py | 3 + readthedocs/oauth/services/github.py | 86 +++++++++++++++++++++++++ readthedocs/projects/tasks.py | 49 ++++++++++++++ 7 files changed, 187 insertions(+), 3 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index bdc9a70fe79..2b7c179d2f6 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -61,3 +61,34 @@ LATEST, STABLE, ) + +# GitHub Build Statuses +GITHUB_BUILD_STATE_FAILURE = 'failure' +GITHUB_BUILD_STATE_PENDING = 'pending' +GITHUB_BUILD_STATE_SUCCESS = 'success' + +# GitHub Build Statuses +GITLAB_BUILD_STATE_FAILURE = 'failed' +GITLAB_BUILD_STATE_PENDING = 'pending' +GITLAB_BUILD_STATE_SUCCESS = 'success' + +# General Build Statuses +BUILD_STATUS_FAILURE = 'failed' +BUILD_STATUS_PENDING = 'pending' +BUILD_STATUS_SUCCESS = 'success' + +# Used to select correct Build status to be sent for each service +SELECT_BUILD_STATUS = { + BUILD_STATUS_FAILURE: { + 'github': GITHUB_BUILD_STATE_FAILURE, + 'gitlab': GITLAB_BUILD_STATE_FAILURE, + }, + BUILD_STATUS_PENDING: { + 'github': GITHUB_BUILD_STATE_PENDING, + 'gitlab': GITLAB_BUILD_STATE_PENDING, + }, + BUILD_STATUS_SUCCESS: { + 'github': GITHUB_BUILD_STATE_SUCCESS, + 'gitlab': GITLAB_BUILD_STATE_SUCCESS, + }, +} diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 8e857d5c0e1..1163a078efb 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -734,6 +734,14 @@ def __str__(self): def get_absolute_url(self): return reverse('builds_detail', args=[self.project.slug, self.pk]) + def get_full_url(self): + """Get url including domain and scheme""" + full_url = '{domain}{absolute_url}'.format( + domain=settings.PUBLIC_API_URL, + absolute_url=self.get_absolute_url() + ) + return full_url + @property def finished(self): """Return if build has a finished state.""" diff --git a/readthedocs/core/utils/__init__.py b/readthedocs/core/utils/__init__.py index dbb8245c6fb..9fd917f095c 100644 --- a/readthedocs/core/utils/__init__.py +++ b/readthedocs/core/utils/__init__.py @@ -11,7 +11,10 @@ from django.utils.safestring import SafeText, mark_safe from django.utils.text import slugify as slugify_base -from readthedocs.builds.constants import BUILD_STATE_TRIGGERED +from readthedocs.builds.constants import ( + BUILD_STATE_TRIGGERED, + BUILD_STATUS_PENDING, +) from readthedocs.doc_builder.constants import DOCKER_LIMITS @@ -78,7 +81,7 @@ def prepare_build( # Avoid circular import from readthedocs.builds.models import Build from readthedocs.projects.models import Project - from readthedocs.projects.tasks import update_docs_task + from readthedocs.projects.tasks import update_docs_task, send_build_status build = None @@ -125,6 +128,10 @@ def prepare_build( options['soft_time_limit'] = time_limit options['time_limit'] = int(time_limit * 1.2) + if build: + # Send pending Build Status to Git Status API + send_build_status.delay(build.id, BUILD_STATUS_PENDING) + return ( update_docs_task.signature( args=(version.pk,), diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index 4d9fcb65e50..db47edf41cd 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -658,7 +658,7 @@ def failed(self): def done(self): """Is build in finished state.""" return ( - self.build is not None and + self.build and self.build['state'] == BUILD_STATE_FINISHED ) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 8bf95707b6c..6d9464c9ec9 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -216,6 +216,9 @@ def setup_webhook(self, project): def update_webhook(self, project, integration): raise NotImplementedError + def send_status(self, project, identifier, state, build_url): + raise NotImplementedError + @classmethod def is_project_service(cls, project): """ diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index ce0ed3b9163..3b8b0360b17 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -12,6 +12,11 @@ from readthedocs.api.v2.client import api from readthedocs.builds import utils as build_utils +from readthedocs.builds.constants import ( + GITHUB_BUILD_STATE_FAILURE, + GITHUB_BUILD_STATE_PENDING, + GITHUB_BUILD_STATE_SUCCESS, +) from readthedocs.integrations.models import Integration from ..models import RemoteOrganization, RemoteRepository @@ -311,6 +316,87 @@ def update_webhook(self, project, integration): ) return (False, resp) + def send_status(self, project, identifier, state, build_url): + """ + Create GitHub commit status for project. + + :param project: project to set up commit status for + :type project: Project + :param identifier: commit sha of the version + :type identifier: str + :param state: build state failure, pending, or success. + :type state: str + :param build_url: build url + :type build_url: str + :returns: boolean based on commit status creation was successful or not. + :rtype: Bool + """ + session = self.get_session() + owner, repo = build_utils.get_github_username_repo(url=project.repo) + + description = '' + if state == GITHUB_BUILD_STATE_SUCCESS: + description = 'The build succeeded!' + elif state == GITHUB_BUILD_STATE_FAILURE: + description = 'The build failed!' + elif state == GITHUB_BUILD_STATE_PENDING: + description = 'The build is pending!' + + data = { + "state": state, + "target_url": build_url, + "description": description, + "context": "continuous-documentation/read-the-docs" + } + + resp = None + try: + resp = session.post( + ( + 'https://api.github.com/repos/{owner}/{repo}/statuses/{sha}' + .format(owner=owner, repo=repo, sha=identifier) + ), + data=json.dumps(data), + headers={'content-type': 'application/json'}, + ) + if resp.status_code == 201: + log.info( + 'GitHub commit status for project: %s', + project, + ) + return True + + if resp.status_code in [401, 403, 404]: + log.info( + 'GitHub project does not exist or user does not have ' + 'permissions: project=%s', + project, + ) + return False + + # Catch exceptions with request or deserializing JSON + except (RequestException, ValueError): + log.exception( + 'GitHub commit status creation failed for project: %s', + project, + ) + else: + log.error( + 'GitHub commit status creation failed for project: %s', + project, + ) + # Response data should always be JSON, still try to log if not + # though + try: + debug_data = resp.json() + except ValueError: + debug_data = resp.content + log.debug( + 'GitHub commit status creation failure response: %s', + debug_data, + ) + return False + @classmethod def get_token_for_project(cls, project, force_local=False): """Get access token for project by iterating over project users.""" diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 12fe740c600..c617a583e72 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -34,6 +34,10 @@ LATEST, LATEST_VERBOSE_NAME, STABLE_VERBOSE_NAME, + EXTERNAL, + BUILD_STATUS_SUCCESS, + BUILD_STATUS_FAILURE, + SELECT_BUILD_STATUS, ) from readthedocs.builds.models import APIVersion, Build, Version from readthedocs.builds.signals import build_complete @@ -59,6 +63,7 @@ ) from readthedocs.doc_builder.loader import get_builder_class from readthedocs.doc_builder.python_environments import Conda, Virtualenv +from readthedocs.oauth.services.github import GitHubService from readthedocs.projects.models import APIProject, Feature from readthedocs.search.utils import index_new_files, remove_indexed_files from readthedocs.sphinx_domains.models import SphinxDomain @@ -573,6 +578,16 @@ def run_build(self, docker, record): if self.build_env.failed: self.send_notifications(self.version.pk, self.build['id']) + # send build failure status to git Status API + self.send_build_status( + self.build['id'], BUILD_STATUS_FAILURE + ) + + if self.build_env.successful: + # send build successful status to git Status API + self.send_build_status( + self.build['id'], BUILD_STATUS_SUCCESS + ) build_complete.send(sender=Build, build=self.build_env.build) @@ -1008,6 +1023,10 @@ def is_type_sphinx(self): """Is documentation type Sphinx.""" return 'sphinx' in self.config.doctype + def send_build_status(self, build_pk, state): + """Send github build status for pull/merge requests.""" + send_build_status.delay(build_pk, state) + # Web tasks @app.task(queue='web') @@ -1773,3 +1792,33 @@ def retry_domain_verification(domain_pk): sender=domain.__class__, domain=domain, ) + + +@app.task(queue='web') +def send_build_status(build_pk, state): + """ + Send Build Status to Git Status API for project. + + :param build_pk: Build pk + :param state: build state failed, pending, or success to be sent. + """ + build = Build.objects.get(pk=build_pk) + version = build.version + project = build.project + + # Send status reports for only External (pull/merge request) Versions. + if version.type != EXTERNAL: + return + + if 'github.com' in project.repo: + account = project.remote_repository.account + service = GitHubService(project.users.first(), account) + # select the correct state. + state = SELECT_BUILD_STATUS[state]['github'] + + # send Status report using the API. + service.send_status( + project, version.identifier, state, build.get_full_url() + ) + + # TODO: Send build status for other providers. From c56148673a31fd0c9708b3855c6077123457103c Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 29 Jun 2019 03:28:14 +0600 Subject: [PATCH 122/171] docstring update --- readthedocs/builds/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 1163a078efb..771568039ff 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -735,7 +735,7 @@ def get_absolute_url(self): return reverse('builds_detail', args=[self.project.slug, self.pk]) def get_full_url(self): - """Get url including domain and scheme""" + """Get full url including domain""" full_url = '{domain}{absolute_url}'.format( domain=settings.PUBLIC_API_URL, absolute_url=self.get_absolute_url() From 3cf07f237e63be9d6f11345bb08f6d687b2f2c47 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 29 Jun 2019 14:55:27 +0600 Subject: [PATCH 123/171] Updated Constants and notification sending --- readthedocs/builds/constants.py | 5 ++++- readthedocs/oauth/services/github.py | 20 ++++++-------------- readthedocs/projects/tasks.py | 13 ++++++------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 2b7c179d2f6..ec56e8f8a60 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -77,18 +77,21 @@ BUILD_STATUS_PENDING = 'pending' BUILD_STATUS_SUCCESS = 'success' -# Used to select correct Build status to be sent for each service +# Used to select correct Build status and description to be sent to each service API SELECT_BUILD_STATUS = { BUILD_STATUS_FAILURE: { 'github': GITHUB_BUILD_STATE_FAILURE, 'gitlab': GITLAB_BUILD_STATE_FAILURE, + 'description': 'The build failed!', }, BUILD_STATUS_PENDING: { 'github': GITHUB_BUILD_STATE_PENDING, 'gitlab': GITLAB_BUILD_STATE_PENDING, + 'description': 'The build is pending!', }, BUILD_STATUS_SUCCESS: { 'github': GITHUB_BUILD_STATE_SUCCESS, 'gitlab': GITLAB_BUILD_STATE_SUCCESS, + 'description': 'The build succeeded!', }, } diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 3b8b0360b17..114193677de 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -12,11 +12,7 @@ from readthedocs.api.v2.client import api from readthedocs.builds import utils as build_utils -from readthedocs.builds.constants import ( - GITHUB_BUILD_STATE_FAILURE, - GITHUB_BUILD_STATE_PENDING, - GITHUB_BUILD_STATE_SUCCESS, -) +from readthedocs.builds.constants import SELECT_BUILD_STATUS from readthedocs.integrations.models import Integration from ..models import RemoteOrganization, RemoteRepository @@ -316,7 +312,7 @@ def update_webhook(self, project, integration): ) return (False, resp) - def send_status(self, project, identifier, state, build_url): + def send_build_status(self, project, identifier, state, build_url): """ Create GitHub commit status for project. @@ -334,16 +330,12 @@ def send_status(self, project, identifier, state, build_url): session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) - description = '' - if state == GITHUB_BUILD_STATE_SUCCESS: - description = 'The build succeeded!' - elif state == GITHUB_BUILD_STATE_FAILURE: - description = 'The build failed!' - elif state == GITHUB_BUILD_STATE_PENDING: - description = 'The build is pending!' + # select the correct state and description. + github_build_state = SELECT_BUILD_STATUS[state]['github'] + description = SELECT_BUILD_STATUS[state]['description'] data = { - "state": state, + "state": github_build_state, "target_url": build_url, "description": description, "context": "continuous-documentation/read-the-docs" diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index c617a583e72..4c83ad36ae6 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -37,7 +37,6 @@ EXTERNAL, BUILD_STATUS_SUCCESS, BUILD_STATUS_FAILURE, - SELECT_BUILD_STATUS, ) from readthedocs.builds.models import APIVersion, Build, Version from readthedocs.builds.signals import build_complete @@ -1532,7 +1531,8 @@ def _manage_imported_files(version, path, commit, build): @app.task(queue='web') def send_notifications(version_pk, build_pk): version = Version.objects.get_object_or_log(pk=version_pk) - if not version: + # only send notification for Internal versions + if not version or version.type != EXTERNAL: return build = Build.objects.get(pk=build_pk) @@ -1797,7 +1797,7 @@ def retry_domain_verification(domain_pk): @app.task(queue='web') def send_build_status(build_pk, state): """ - Send Build Status to Git Status API for project. + Send Build Status to Git Status API for project external versions. :param build_pk: Build pk :param state: build state failed, pending, or success to be sent. @@ -1813,12 +1813,11 @@ def send_build_status(build_pk, state): if 'github.com' in project.repo: account = project.remote_repository.account service = GitHubService(project.users.first(), account) - # select the correct state. - state = SELECT_BUILD_STATUS[state]['github'] # send Status report using the API. - service.send_status( - project, version.identifier, state, build.get_full_url() + service.send_build_status( + project, version.identifier, state, + build.get_full_url() ) # TODO: Send build status for other providers. From d9a867bdf997516637fd178fad5a229dbe172bde Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 29 Jun 2019 14:56:18 +0600 Subject: [PATCH 124/171] cleanup --- readthedocs/projects/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 4c83ad36ae6..2e4c63ee196 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -1531,9 +1531,11 @@ def _manage_imported_files(version, path, commit, build): @app.task(queue='web') def send_notifications(version_pk, build_pk): version = Version.objects.get_object_or_log(pk=version_pk) + # only send notification for Internal versions - if not version or version.type != EXTERNAL: + if not version or version.type == EXTERNAL: return + build = Build.objects.get(pk=build_pk) for hook in version.project.webhook_notifications.all(): From ef6707bffc10b35de71e13081f552ddaacecc617 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 2 Jul 2019 19:26:11 +0600 Subject: [PATCH 125/171] task update --- readthedocs/builds/constants.py | 2 +- readthedocs/oauth/services/base.py | 2 +- readthedocs/oauth/services/github.py | 31 ++++++++++------------------ readthedocs/projects/tasks.py | 21 ++++++++----------- 4 files changed, 22 insertions(+), 34 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index ec56e8f8a60..7cb582fa448 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -67,7 +67,7 @@ GITHUB_BUILD_STATE_PENDING = 'pending' GITHUB_BUILD_STATE_SUCCESS = 'success' -# GitHub Build Statuses +# GitLab Build Statuses GITLAB_BUILD_STATE_FAILURE = 'failed' GITLAB_BUILD_STATE_PENDING = 'pending' GITLAB_BUILD_STATE_SUCCESS = 'success' diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 6d9464c9ec9..99098f80487 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -216,7 +216,7 @@ def setup_webhook(self, project): def update_webhook(self, project, integration): raise NotImplementedError - def send_status(self, project, identifier, state, build_url): + def send_build_status(self, build, state): raise NotImplementedError @classmethod diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 114193677de..21201286f89 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -312,42 +312,38 @@ def update_webhook(self, project, integration): ) return (False, resp) - def send_build_status(self, project, identifier, state, build_url): + def send_build_status(self, build, state): """ Create GitHub commit status for project. - :param project: project to set up commit status for - :type project: Project - :param identifier: commit sha of the version - :type identifier: str + :param build: Build to set up commit status for + :type build: Build :param state: build state failure, pending, or success. :type state: str - :param build_url: build url - :type build_url: str :returns: boolean based on commit status creation was successful or not. :rtype: Bool """ session = self.get_session() + project = build.project owner, repo = build_utils.get_github_username_repo(url=project.repo) + build_sha = build.version.identifier # select the correct state and description. github_build_state = SELECT_BUILD_STATUS[state]['github'] description = SELECT_BUILD_STATUS[state]['description'] data = { - "state": github_build_state, - "target_url": build_url, - "description": description, - "context": "continuous-documentation/read-the-docs" + 'state': github_build_state, + 'target_url': build.get_full_url(), + 'description': description, + 'context': 'continuous-documentation/read-the-docs' } resp = None try: + resp = session.post( - ( - 'https://api.github.com/repos/{owner}/{repo}/statuses/{sha}' - .format(owner=owner, repo=repo, sha=identifier) - ), + f'https://api.github.com/repos/{owner}/{repo}/statuses/{build_sha}', data=json.dumps(data), headers={'content-type': 'application/json'}, ) @@ -372,11 +368,6 @@ def send_build_status(self, project, identifier, state, build_url): 'GitHub commit status creation failed for project: %s', project, ) - else: - log.error( - 'GitHub commit status creation failed for project: %s', - project, - ) # Response data should always be JSON, still try to log if not # though try: diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 2e4c63ee196..ffdbaff935f 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -581,8 +581,7 @@ def run_build(self, docker, record): self.send_build_status( self.build['id'], BUILD_STATUS_FAILURE ) - - if self.build_env.successful: + elif self.build_env.successful: # send build successful status to git Status API self.send_build_status( self.build['id'], BUILD_STATUS_SUCCESS @@ -1805,21 +1804,19 @@ def send_build_status(build_pk, state): :param state: build state failed, pending, or success to be sent. """ build = Build.objects.get(pk=build_pk) - version = build.version - project = build.project # Send status reports for only External (pull/merge request) Versions. - if version.type != EXTERNAL: + if build.version.type != EXTERNAL: return - if 'github.com' in project.repo: - account = project.remote_repository.account - service = GitHubService(project.users.first(), account) + if build.project.remote_repository.account.provider == 'github': + account = build.project.remote_repository.account + service = GitHubService( + build.project.remote_repository.users.first(), + account + ) # send Status report using the API. - service.send_build_status( - project, version.identifier, state, - build.get_full_url() - ) + service.send_build_status(build, state) # TODO: Send build status for other providers. From 66526be263149262f9217148809b02ec83224479 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 2 Jul 2019 20:49:27 +0600 Subject: [PATCH 126/171] send_build_status updated --- readthedocs/oauth/services/github.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 21201286f89..dad7592237f 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -340,8 +340,8 @@ def send_build_status(self, build, state): } resp = None - try: + try: resp = session.post( f'https://api.github.com/repos/{owner}/{repo}/statuses/{build_sha}', data=json.dumps(data), @@ -362,6 +362,8 @@ def send_build_status(self, build, state): ) return False + return False + # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.exception( @@ -373,7 +375,10 @@ def send_build_status(self, build, state): try: debug_data = resp.json() except ValueError: - debug_data = resp.content + if resp is not None: + debug_data = resp.content + else: + debug_data = resp log.debug( 'GitHub commit status creation failure response: %s', debug_data, From 8f2505ae3bfc8bd937ce4a36505c861dff738871 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 2 Jul 2019 20:58:19 +0600 Subject: [PATCH 127/171] updated get full url --- readthedocs/builds/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 771568039ff..59c19dee225 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -736,7 +736,7 @@ def get_absolute_url(self): def get_full_url(self): """Get full url including domain""" - full_url = '{domain}{absolute_url}'.format( + full_url = 'https://{domain}{absolute_url}'.format( domain=settings.PUBLIC_API_URL, absolute_url=self.get_absolute_url() ) From ac67805a907a336f488c0705c7f4d0d190de602d Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 2 Jul 2019 21:01:17 +0600 Subject: [PATCH 128/171] changed to PRODUCTION_DOMAIN --- readthedocs/builds/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 59c19dee225..0663f81ae88 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -737,7 +737,7 @@ def get_absolute_url(self): def get_full_url(self): """Get full url including domain""" full_url = 'https://{domain}{absolute_url}'.format( - domain=settings.PUBLIC_API_URL, + domain=settings.PRODUCTION_DOMAIN, absolute_url=self.get_absolute_url() ) return full_url From 95264e0b260c957bbb1726adffa1b41447ee98f4 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 2 Jul 2019 21:47:47 +0600 Subject: [PATCH 129/171] tasks updated --- readthedocs/core/utils/__init__.py | 9 ++++++--- readthedocs/oauth/services/github.py | 13 ++++++------ readthedocs/projects/tasks.py | 30 ++++++++++++++++++---------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/readthedocs/core/utils/__init__.py b/readthedocs/core/utils/__init__.py index 9fd917f095c..8b8834d1f20 100644 --- a/readthedocs/core/utils/__init__.py +++ b/readthedocs/core/utils/__init__.py @@ -81,7 +81,10 @@ def prepare_build( # Avoid circular import from readthedocs.builds.models import Build from readthedocs.projects.models import Project - from readthedocs.projects.tasks import update_docs_task, send_build_status + from readthedocs.projects.tasks import ( + update_docs_task, + send_external_build_status, + ) build = None @@ -129,8 +132,8 @@ def prepare_build( options['time_limit'] = int(time_limit * 1.2) if build: - # Send pending Build Status to Git Status API - send_build_status.delay(build.id, BUILD_STATUS_PENDING) + # Send pending Build Status using Git Status API for External Builds. + send_external_build_status(build.id, BUILD_STATUS_PENDING) return ( update_docs_task.signature( diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index dad7592237f..ce978f99996 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -372,13 +372,14 @@ def send_build_status(self, build, state): ) # Response data should always be JSON, still try to log if not # though - try: - debug_data = resp.json() - except ValueError: - if resp is not None: + if resp is not None: + try: + debug_data = resp.json() + except ValueError: debug_data = resp.content - else: - debug_data = resp + else: + debug_data = resp + log.debug( 'GitHub commit status creation failure response: %s', debug_data, diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index ffdbaff935f..aa9afb26e1e 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -1023,7 +1023,7 @@ def is_type_sphinx(self): def send_build_status(self, build_pk, state): """Send github build status for pull/merge requests.""" - send_build_status.delay(build_pk, state) + send_external_build_status(build_pk, state) # Web tasks @@ -1796,27 +1796,35 @@ def retry_domain_verification(domain_pk): @app.task(queue='web') -def send_build_status(build_pk, state): +def send_build_status(build, state): """ Send Build Status to Git Status API for project external versions. - :param build_pk: Build pk + :param build: Build :param state: build state failed, pending, or success to be sent. """ - build = Build.objects.get(pk=build_pk) - - # Send status reports for only External (pull/merge request) Versions. - if build.version.type != EXTERNAL: - return - if build.project.remote_repository.account.provider == 'github': - account = build.project.remote_repository.account service = GitHubService( build.project.remote_repository.users.first(), - account + build.project.remote_repository.account ) # send Status report using the API. service.send_build_status(build, state) # TODO: Send build status for other providers. + + +def send_external_build_status(build_pk, state): + """ + Check if build is external and Send Build Status for project external versions. + + :param build_pk: Build pk + :param state: build state failed, pending, or success to be sent. + """ + build = Build.objects.get(pk=build_pk) + + # Send status reports for only External (pull/merge request) Versions. + if build.version.type == EXTERNAL: + # call the task that actually send the build status. + send_build_status.delay(build, state) From 1c0dda5febef7aadd30a95d96e3e7c46263f17c2 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 3 Jul 2019 18:39:34 +0600 Subject: [PATCH 130/171] some tests added --- readthedocs/rtd_tests/tests/test_api.py | 20 +++++++----- .../rtd_tests/tests/test_projects_tasks.py | 31 +++++++++++++++++-- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index c1a78d496da..0cfa3de8721 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -906,7 +906,8 @@ def test_github_create_event(self, sync_repository_task, trigger_build): latest_version = self.project.versions.get(slug=LATEST) sync_repository_task.delay.assert_called_with(latest_version.pk) - def test_github_pull_request_opened_event(self, trigger_build): + @mock.patch('readthedocs.core.utils.trigger_build') + def test_github_pull_request_opened_event(self, trigger_build, core_trigger_build): client = APIClient() headers = {GITHUB_EVENT_HEADER: GITHUB_PULL_REQUEST} @@ -925,12 +926,13 @@ def test_github_pull_request_opened_event(self, trigger_build): self.assertTrue(resp.data['build_triggered']) self.assertEqual(resp.data['project'], self.project.slug) self.assertEqual(resp.data['versions'], [external_version.verbose_name]) - trigger_build.assert_called_once_with( + core_trigger_build.assert_called_once_with( force=True, project=self.project, version=external_version ) self.assertTrue(external_version) - def test_github_pull_request_reopened_event(self, trigger_build): + @mock.patch('readthedocs.core.utils.trigger_build') + def test_github_pull_request_reopened_event(self, trigger_build, core_trigger_build): client = APIClient() # Update the payload for `reopened` webhook event @@ -955,12 +957,13 @@ def test_github_pull_request_reopened_event(self, trigger_build): self.assertTrue(resp.data['build_triggered']) self.assertEqual(resp.data['project'], self.project.slug) self.assertEqual(resp.data['versions'], [external_version.verbose_name]) - trigger_build.assert_called_once_with( + core_trigger_build.assert_called_once_with( force=True, project=self.project, version=external_version ) self.assertTrue(external_version) - def test_github_pull_request_synchronize_event(self, trigger_build): + @mock.patch('readthedocs.core.utils.trigger_build') + def test_github_pull_request_synchronize_event(self, trigger_build, core_trigger_build): client = APIClient() pull_request_number = '6' @@ -998,13 +1001,14 @@ def test_github_pull_request_synchronize_event(self, trigger_build): self.assertTrue(resp.data['build_triggered']) self.assertEqual(resp.data['project'], self.project.slug) self.assertEqual(resp.data['versions'], [external_version.verbose_name]) - trigger_build.assert_called_once_with( + core_trigger_build.assert_called_once_with( force=True, project=self.project, version=external_version ) # `synchronize` webhook event updated the identifier (commit hash) self.assertNotEqual(prev_identifier, external_version.identifier) - def test_github_pull_request_closed_event(self, trigger_build): + @mock.patch('readthedocs.core.utils.trigger_build') + def test_github_pull_request_closed_event(self, trigger_build, core_trigger_build): client = APIClient() pull_request_number = '7' @@ -1044,7 +1048,7 @@ def test_github_pull_request_closed_event(self, trigger_build): self.assertTrue(resp.data['version_deleted']) self.assertEqual(resp.data['project'], self.project.slug) self.assertEqual(resp.data['versions'], [version.verbose_name]) - trigger_build.assert_not_called() + core_trigger_build.assert_not_called() def test_github_pull_request_no_action(self, trigger_build): client = APIClient() diff --git a/readthedocs/rtd_tests/tests/test_projects_tasks.py b/readthedocs/rtd_tests/tests/test_projects_tasks.py index c91f9b1534c..1ec1d8d0ae7 100644 --- a/readthedocs/rtd_tests/tests/test_projects_tasks.py +++ b/readthedocs/rtd_tests/tests/test_projects_tasks.py @@ -4,9 +4,14 @@ from django_dynamic_fixture import get from mock import patch -from readthedocs.builds.models import Version +from readthedocs.builds.constants import EXTERNAL, BUILD_STATUS_SUCCESS +from readthedocs.builds.models import Version, Build from readthedocs.projects.models import Project -from readthedocs.projects.tasks import sync_files +from readthedocs.projects.tasks import ( + sync_files, + send_external_build_status, + send_build_status, +) class SyncFilesTests(TestCase): @@ -121,3 +126,25 @@ def test_sync_files_search(self, rmtree, copy): self.assertIn('artifacts', args[0]) self.assertIn('sphinx_search', args[0]) self.assertIn('media/json', args[1]) + + +class SendBuildStatusTests(TestCase): + + def setUp(self): + self.project = get(Project) + self.internal_version = get(Version, project=self.project) + self.external_version = get(Version, project=self.project, type=EXTERNAL) + self.external_build = get(Build, project=self.project, version=self.external_version) + self.internal_build = get(Build, project=self.project, version=self.internal_version) + + @patch('readthedocs.projects.tasks.send_build_status') + def test_send_external_build_status_with_external_version(self, send_build_status): + send_external_build_status(self.external_build.id, BUILD_STATUS_SUCCESS) + + send_build_status.delay.assert_called_once_with(self.external_build, BUILD_STATUS_SUCCESS) + + @patch('readthedocs.projects.tasks.send_build_status') + def test_send_external_build_status_with_internal_version(self, send_build_status): + send_external_build_status(self.internal_build.id, BUILD_STATUS_SUCCESS) + + send_build_status.delay.assert_not_called() From 9b10ea6e5a7f4b45455b1ed2993bb9dfb39db4a6 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 4 Jul 2019 18:54:18 +0600 Subject: [PATCH 131/171] send_build_status tests added --- readthedocs/rtd_tests/tests/test_oauth.py | 53 +++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index 1ef675cc42c..49ac7bf64d4 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -5,6 +5,10 @@ from django.test import TestCase from django.test.utils import override_settings +from django_dynamic_fixture import get + +from readthedocs.builds.constants import EXTERNAL, BUILD_STATUS_SUCCESS +from readthedocs.builds.models import Version, Build from readthedocs.oauth.models import RemoteOrganization, RemoteRepository from readthedocs.oauth.services import ( BitbucketService, @@ -26,6 +30,10 @@ def setUp(self): self.org = RemoteOrganization.objects.create(slug='rtfd', json='') self.privacy = self.project.version_privacy_level self.service = GitHubService(user=self.user, account=None) + self.external_version = get(Version, project=self.project, type=EXTERNAL) + self.external_build = get( + Build, project=self.project, version=self.external_version + ) def test_make_project_pass(self): repo_json = { @@ -142,6 +150,51 @@ def test_multiple_users_same_repo(self): self.assertEqual(github_project, github_project_5) self.assertEqual(github_project_2, github_project_6) + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_send_build_status_successful(self, session, mock_logger): + session().post.return_value.status_code = 201 + success = self.service.send_build_status( + self.external_build, + BUILD_STATUS_SUCCESS + ) + + self.assertTrue(success) + mock_logger.info.assert_called_with( + 'GitHub commit status for project: %s', + self.project + ) + + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_send_build_status_404_error(self, session, mock_logger): + session().post.return_value.status_code = 404 + success = self.service.send_build_status( + self.external_build, + BUILD_STATUS_SUCCESS + ) + + self.assertFalse(success) + mock_logger.info.assert_called_with( + 'GitHub project does not exist or user does not have ' + 'permissions: project=%s', + self.project + ) + + @mock.patch('readthedocs.oauth.services.github.log') + @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session') + def test_send_build_status_value_error(self, session, mock_logger): + session().post.side_effect = ValueError + success = self.service.send_build_status( + self.external_build, BUILD_STATUS_SUCCESS + ) + + self.assertFalse(success) + mock_logger.exception.assert_called_with( + 'GitHub commit status creation failed for project: %s', + self.project, + ) + @override_settings(DEFAULT_PRIVACY_LEVEL='private') def test_make_private_project(self): """ From e8098fbbd160f873db3d9ab4449badf8b3c23ba5 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 4 Jul 2019 20:09:42 +0600 Subject: [PATCH 132/171] tests for tasks added --- readthedocs/rtd_tests/tests/test_celery.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_celery.py b/readthedocs/rtd_tests/tests/test_celery.py index e3036367e57..301cc218dc5 100644 --- a/readthedocs/rtd_tests/tests/test_celery.py +++ b/readthedocs/rtd_tests/tests/test_celery.py @@ -7,11 +7,14 @@ from django_dynamic_fixture import get from mock import MagicMock, patch -from readthedocs.builds.constants import LATEST +from allauth.socialaccount.models import SocialAccount + +from readthedocs.builds.constants import LATEST, BUILD_STATUS_SUCCESS, EXTERNAL from readthedocs.builds.models import Build from readthedocs.doc_builder.exceptions import VersionLockedError from readthedocs.projects import tasks from readthedocs.builds.models import Version +from readthedocs.oauth.models import RemoteRepository from readthedocs.projects.exceptions import RepositoryError from readthedocs.projects.models import Project from readthedocs.rtd_tests.base import RTDTestCase @@ -329,3 +332,17 @@ def test_fileify_logging_when_wrong_version_pk(self, mock_logger): self.assertFalse(Version.objects.filter(pk=345343).exists()) tasks.fileify(version_pk=345343, commit=None, build=1) mock_logger.warning.assert_called_with("Version not found for given kwargs. {'pk': 345343}") + + @patch('readthedocs.projects.tasks.GitHubService.send_build_status') + def test_send_build_status_task(self, send_build_status): + social_account = get(SocialAccount, provider='github') + remote_repo = get(RemoteRepository, account=social_account, project=self.project) + remote_repo.users.add(self.eric) + + external_version = get(Version, project=self.project, type=EXTERNAL) + external_build = get( + Build, project=self.project, version=external_version + ) + tasks.send_build_status(external_build, BUILD_STATUS_SUCCESS) + + send_build_status.assert_called_once_with(external_build, BUILD_STATUS_SUCCESS) From e7a16410749f3a7d3ee8501c23881f60da7a30ab Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 5 Jul 2019 01:33:47 +0600 Subject: [PATCH 133/171] try catch added to task --- readthedocs/projects/tasks.py | 22 +++++++++++++++------- readthedocs/rtd_tests/tests/test_celery.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index aa9afb26e1e..7ac2d17b730 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -62,6 +62,7 @@ ) from readthedocs.doc_builder.loader import get_builder_class from readthedocs.doc_builder.python_environments import Conda, Virtualenv +from readthedocs.oauth.models import RemoteRepository from readthedocs.oauth.services.github import GitHubService from readthedocs.projects.models import APIProject, Feature from readthedocs.search.utils import index_new_files, remove_indexed_files @@ -1803,14 +1804,21 @@ def send_build_status(build, state): :param build: Build :param state: build state failed, pending, or success to be sent. """ - if build.project.remote_repository.account.provider == 'github': - service = GitHubService( - build.project.remote_repository.users.first(), - build.project.remote_repository.account - ) + try: + if build.project.remote_repository.account.provider == 'github': + service = GitHubService( + build.project.remote_repository.users.first(), + build.project.remote_repository.account + ) - # send Status report using the API. - service.send_build_status(build, state) + # send Status report using the API. + service.send_build_status(build, state) + + except RemoteRepository.DoesNotExist: + log.info('Remote repository does not exist for %s', build.project) + + except Exception: + log.exception('Send build status task failed.') # TODO: Send build status for other providers. diff --git a/readthedocs/rtd_tests/tests/test_celery.py b/readthedocs/rtd_tests/tests/test_celery.py index 301cc218dc5..f33c2892b77 100644 --- a/readthedocs/rtd_tests/tests/test_celery.py +++ b/readthedocs/rtd_tests/tests/test_celery.py @@ -346,3 +346,18 @@ def test_send_build_status_task(self, send_build_status): tasks.send_build_status(external_build, BUILD_STATUS_SUCCESS) send_build_status.assert_called_once_with(external_build, BUILD_STATUS_SUCCESS) + + @patch('readthedocs.projects.tasks.log') + @patch('readthedocs.projects.tasks.GitHubService.send_build_status') + def test_send_build_status_task_without_remote_repo(self, send_build_status, mock_logger): + external_version = get(Version, project=self.project, type=EXTERNAL) + external_build = get( + Build, project=self.project, version=external_version + ) + tasks.send_build_status(external_build, BUILD_STATUS_SUCCESS) + + send_build_status.assert_not_called() + mock_logger.info.assert_called_with( + 'Remote repository does not exist for %s', + self.project, + ) From 917b2e56196cf2c974ad5502b77bfa384fe912e8 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 5 Jul 2019 01:47:18 +0600 Subject: [PATCH 134/171] exception log updated --- readthedocs/projects/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 7ac2d17b730..a5aadb0b195 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -1818,7 +1818,7 @@ def send_build_status(build, state): log.info('Remote repository does not exist for %s', build.project) except Exception: - log.exception('Send build status task failed.') + log.exception('Send build status task failed for %s', build.project) # TODO: Send build status for other providers. From 3f500e3a920e8fecea0e9f02a0aba86291f17f81 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 6 Jul 2019 20:25:14 +0600 Subject: [PATCH 135/171] storage updated for external versions --- readthedocs/projects/models.py | 9 +++++++-- readthedocs/rtd_tests/tests/test_project.py | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 688409b8a88..807a4696320 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -19,7 +19,7 @@ from taggit.managers import TaggableManager from readthedocs.api.v2.client import api -from readthedocs.builds.constants import LATEST, STABLE, INTERNAL +from readthedocs.builds.constants import LATEST, STABLE, INTERNAL, EXTERNAL from readthedocs.core.resolver import resolve, resolve_domain from readthedocs.core.utils import broadcast, slugify from readthedocs.projects import constants @@ -543,8 +543,13 @@ def get_storage_path(self, type_, version_slug=LATEST, include_file=True): :return: the path to an item in storage (can be used with ``storage.url`` to get the URL) """ + type_dir = type_ + # Add `external/` prefix for external versions + if self.versions(manager=EXTERNAL).filter(slug=version_slug).exists(): + type_dir = f'{EXTERNAL}/{type_}' + folder_path = '{}/{}/{}'.format( - type_, + type_dir, self.slug, version_slug, ) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index ded11f52c53..05620a43d45 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -34,8 +34,8 @@ def setUp(self): self.external_version = get( Version, identifier='pr-version', - verbose_name='pr-version', - slug='pr-9999', + verbose_name='99', + slug='99', project=self.pip, active=True, type=EXTERNAL @@ -145,6 +145,20 @@ def test_get_storage_path(self): 'htmlzip/pip/latest/pip.zip', ) + def test_get_storage_path_for_external_versions(self): + self.assertEqual( + self.pip.get_storage_path('pdf', self.external_version.slug), + 'external/pdf/pip/99/pip.pdf', + ) + self.assertEqual( + self.pip.get_storage_path('epub', self.external_version.slug), + 'external/epub/pip/99/pip.epub', + ) + self.assertEqual( + self.pip.get_storage_path('htmlzip', self.external_version.slug), + 'external/htmlzip/pip/99/pip.zip', + ) + def test_ordered_active_versions_excludes_external_versions(self): self.assertNotIn(self.external_version, self.pip.ordered_active_versions()) From f9634b66c580b26e9f7e82701f7b5a0a60259fa7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 9 Jul 2019 18:27:09 +0600 Subject: [PATCH 136/171] update send_build_status process --- readthedocs/builds/models.py | 4 ++- readthedocs/oauth/services/base.py | 44 ++++++++++++++++++++++++++++++ readthedocs/projects/tasks.py | 20 ++++++++++---- 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 0663f81ae88..f4cb9310db0 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -736,7 +736,9 @@ def get_absolute_url(self): def get_full_url(self): """Get full url including domain""" - full_url = 'https://{domain}{absolute_url}'.format( + scheme = 'http' if settings.DEBUG else 'https' + full_url = '{scheme}://{domain}{absolute_url}'.format( + scheme=scheme, domain=settings.PRODUCTION_DOMAIN, absolute_url=self.get_absolute_url() ) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 99098f80487..ba21e56c085 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -184,12 +184,28 @@ def paginate(self, url, **kwargs): return [] def sync(self): + """Sync repositories and organizations.""" raise NotImplementedError def create_repository(self, fields, privacy=None, organization=None): + """ + Update or create a repository from API response. + + :param fields: dictionary of response data from API + :param privacy: privacy level to support + :param organization: remote organization to associate with + :type organization: RemoteOrganization + :rtype: RemoteRepository + """ raise NotImplementedError def create_organization(self, fields): + """ + Update or create remote organization from API response. + + :param fields: dictionary response of data from API + :rtype: RemoteOrganization + """ raise NotImplementedError def get_next_url_to_paginate(self, response): @@ -211,12 +227,40 @@ def get_paginated_results(self, response): raise NotImplementedError def setup_webhook(self, project): + """ + Setup webhook for project. + + :param project: project to set up webhook for + :type project: Project + :returns: boolean based on webhook set up success, and requests Response object + :rtype: (Bool, Response) + """ raise NotImplementedError def update_webhook(self, project, integration): + """ + Update webhook integration. + + :param project: project to set up webhook for + :type project: Project + :param integration: Webhook integration to update + :type integration: Integration + :returns: boolean based on webhook update success, and requests Response object + :rtype: (Bool, Response) + """ raise NotImplementedError def send_build_status(self, build, state): + """ + Create commit status for project. + + :param build: Build to set up commit status for + :type build: Build + :param state: build state failure, pending, or success. + :type state: str + :returns: boolean based on commit status creation was successful or not. + :rtype: Bool + """ raise NotImplementedError @classmethod diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index a5aadb0b195..e791f65a0ac 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -579,14 +579,26 @@ def run_build(self, docker, record): if self.build_env.failed: self.send_notifications(self.version.pk, self.build['id']) # send build failure status to git Status API - self.send_build_status( + send_external_build_status( self.build['id'], BUILD_STATUS_FAILURE ) elif self.build_env.successful: # send build successful status to git Status API - self.send_build_status( + send_external_build_status( self.build['id'], BUILD_STATUS_SUCCESS ) + else: + msg = 'Unhandled Build State: Build ID:'.format( + self.build['id'], + ) + log.warning( + LOG_TEMPLATE, + { + 'project': self.project.slug, + 'version': self.version.slug, + 'msg': msg, + } + ) build_complete.send(sender=Build, build=self.build_env.build) @@ -1022,10 +1034,6 @@ def is_type_sphinx(self): """Is documentation type Sphinx.""" return 'sphinx' in self.config.doctype - def send_build_status(self, build_pk, state): - """Send github build status for pull/merge requests.""" - send_external_build_status(build_pk, state) - # Web tasks @app.task(queue='web') From ec1c6a4190071ac7dde0cad591f39d8160aa2207 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 9 Jul 2019 18:29:41 +0600 Subject: [PATCH 137/171] removed gitlab build status constants --- readthedocs/builds/constants.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 7cb582fa448..3606bd7415b 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -67,11 +67,6 @@ GITHUB_BUILD_STATE_PENDING = 'pending' GITHUB_BUILD_STATE_SUCCESS = 'success' -# GitLab Build Statuses -GITLAB_BUILD_STATE_FAILURE = 'failed' -GITLAB_BUILD_STATE_PENDING = 'pending' -GITLAB_BUILD_STATE_SUCCESS = 'success' - # General Build Statuses BUILD_STATUS_FAILURE = 'failed' BUILD_STATUS_PENDING = 'pending' @@ -81,17 +76,14 @@ SELECT_BUILD_STATUS = { BUILD_STATUS_FAILURE: { 'github': GITHUB_BUILD_STATE_FAILURE, - 'gitlab': GITLAB_BUILD_STATE_FAILURE, 'description': 'The build failed!', }, BUILD_STATUS_PENDING: { 'github': GITHUB_BUILD_STATE_PENDING, - 'gitlab': GITLAB_BUILD_STATE_PENDING, 'description': 'The build is pending!', }, BUILD_STATUS_SUCCESS: { 'github': GITHUB_BUILD_STATE_SUCCESS, - 'gitlab': GITLAB_BUILD_STATE_SUCCESS, 'description': 'The build succeeded!', }, } From edab9b37a52c80c068b5a55e1a2cc96272bf9374 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 9 Jul 2019 18:45:20 +0600 Subject: [PATCH 138/171] lint fix --- readthedocs/projects/tasks.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index e791f65a0ac..8a68836e911 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -588,9 +588,7 @@ def run_build(self, docker, record): self.build['id'], BUILD_STATUS_SUCCESS ) else: - msg = 'Unhandled Build State: Build ID:'.format( - self.build['id'], - ) + msg = 'Unhandled Build State' log.warning( LOG_TEMPLATE, { From 1d4e0dbf5431992b4183dc203ccd6d87b036d5ac Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 9 Jul 2019 20:35:07 +0600 Subject: [PATCH 139/171] version type passed to function --- readthedocs/builds/models.py | 7 ++-- readthedocs/projects/models.py | 40 +++++++++++++++------ readthedocs/projects/tasks.py | 2 ++ readthedocs/projects/views/public.py | 3 +- readthedocs/rtd_tests/tests/test_project.py | 13 +++++-- 5 files changed, 48 insertions(+), 17 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 8e857d5c0e1..db9cbf242cb 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -339,17 +339,17 @@ def get_downloads(self, pretty=False): def prettify(k): return k if pretty else k.lower() - if project.has_pdf(self.slug): + if project.has_pdf(self.slug, version_type=self.type): data[prettify('PDF')] = project.get_production_media_url( 'pdf', self.slug, ) - if project.has_htmlzip(self.slug): + if project.has_htmlzip(self.slug, version_type=self.type): data[prettify('HTML')] = project.get_production_media_url( 'htmlzip', self.slug, ) - if project.has_epub(self.slug): + if project.has_epub(self.slug, version_type=self.type): data[prettify('Epub')] = project.get_production_media_url( 'epub', self.slug, @@ -400,6 +400,7 @@ def get_storage_paths(self): type_=type_, version_slug=self.slug, include_file=False, + version_type=self.type, ) ) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 807a4696320..e5f4891a175 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -533,19 +533,26 @@ def get_subproject_urls(self): return [(proj.child.slug, proj.child.get_docs_url()) for proj in self.subprojects.all()] - def get_storage_path(self, type_, version_slug=LATEST, include_file=True): + def get_storage_path( + self, + type_, + version_slug=LATEST, + include_file=True, + version_type=None + ): """ Get a path to a build artifact for use with Django's storage system. :param type_: Media content type, ie - 'pdf', 'htmlzip' :param version_slug: Project version slug for lookup + :param version_type: Project version type :param include_file: Include file name in return :return: the path to an item in storage (can be used with ``storage.url`` to get the URL) """ type_dir = type_ # Add `external/` prefix for external versions - if self.versions(manager=EXTERNAL).filter(slug=version_slug).exists(): + if version_type == EXTERNAL: type_dir = f'{EXTERNAL}/{type_}' folder_path = '{}/{}/{}'.format( @@ -780,7 +787,7 @@ def has_versions(self): def has_aliases(self): return self.aliases.exists() - def has_media(self, type_, version_slug=LATEST): + def has_media(self, type_, version_slug=LATEST, version_type=None): path = self.get_production_media_path( type_=type_, version_slug=version_slug ) @@ -790,20 +797,33 @@ def has_media(self, type_, version_slug=LATEST): if settings.RTD_BUILD_MEDIA_STORAGE: storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)() storage_path = self.get_storage_path( - type_=type_, version_slug=version_slug + type_=type_, version_slug=version_slug, + version_type=version_type ) return storage.exists(storage_path) return False - def has_pdf(self, version_slug=LATEST): - return self.has_media(MEDIA_TYPE_PDF, version_slug=version_slug) + def has_pdf(self, version_slug=LATEST, version_type=None): + return self.has_media( + MEDIA_TYPE_PDF, + version_slug=version_slug, + version_type=version_type + ) - def has_epub(self, version_slug=LATEST): - return self.has_media(MEDIA_TYPE_EPUB, version_slug=version_slug) + def has_epub(self, version_slug=LATEST, version_type=None): + return self.has_media( + MEDIA_TYPE_EPUB, + version_slug=version_slug, + version_type=version_type + ) - def has_htmlzip(self, version_slug=LATEST): - return self.has_media(MEDIA_TYPE_HTMLZIP, version_slug=version_slug) + def has_htmlzip(self, version_slug=LATEST, version_type=None): + return self.has_media( + MEDIA_TYPE_HTMLZIP, + version_slug=version_slug, + version_type=version_type + ) @property def sponsored(self): diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 12fe740c600..dff8a0ff2d0 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -767,6 +767,7 @@ def store_build_artifacts( type_=media_type, version_slug=self.version.slug, include_file=False, + version_type=self.version.type, ) log.info( LOG_TEMPLATE, @@ -795,6 +796,7 @@ def store_build_artifacts( type_=media_type, version_slug=self.version.slug, include_file=False, + version_type=self.version.type, ) log.info( LOG_TEMPLATE, diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index 174e618c9aa..17a3ec173e3 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -220,7 +220,8 @@ def project_download_media(request, project_slug, type_, version_slug): if settings.RTD_BUILD_MEDIA_STORAGE: storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)() storage_path = version.project.get_storage_path( - type_=type_, version_slug=version_slug + type_=type_, version_slug=version_slug, + version_type=version.type, ) if storage.exists(storage_path): return HttpResponseRedirect(storage.url(storage_path)) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 05620a43d45..ae9d3a8da0f 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -147,15 +147,22 @@ def test_get_storage_path(self): def test_get_storage_path_for_external_versions(self): self.assertEqual( - self.pip.get_storage_path('pdf', self.external_version.slug), + self.pip.get_storage_path( + 'pdf', self.external_version.slug, + version_type=self.external_version.type + ), 'external/pdf/pip/99/pip.pdf', ) self.assertEqual( - self.pip.get_storage_path('epub', self.external_version.slug), + self.pip.get_storage_path('epub', self.external_version.slug, + version_type=self.external_version.type + ), 'external/epub/pip/99/pip.epub', ) self.assertEqual( - self.pip.get_storage_path('htmlzip', self.external_version.slug), + self.pip.get_storage_path('htmlzip', self.external_version.slug, + version_type=self.external_version.type + ), 'external/htmlzip/pip/99/pip.zip', ) From 7629360f91a3a3beac7530c1a536fcab6217fac2 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 9 Jul 2019 21:52:39 +0600 Subject: [PATCH 140/171] only build html for external versions --- .gitignore | 1 + readthedocs/projects/tasks.py | 12 ++++--- .../tests/test_config_integration.py | 31 +++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index dbb0fa606cb..dc1401badb6 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ logs/* media/dash media/epub media/export +media/external media/html media/htmlzip media/json diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index dff8a0ff2d0..467a423b6e3 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -34,6 +34,7 @@ LATEST, LATEST_VERBOSE_NAME, STABLE_VERBOSE_NAME, + EXTERNAL, ) from readthedocs.builds.models import APIVersion, Build, Version from readthedocs.builds.signals import build_complete @@ -955,13 +956,16 @@ def build_docs_search(self): """Build search data.""" # Search is always run in sphinx using the rtd-sphinx-extension. # Mkdocs has no search currently. - if self.is_type_sphinx(): + if self.is_type_sphinx() and self.version.type != EXTERNAL: return True return False def build_docs_localmedia(self): """Get local media files with separate build.""" - if 'htmlzip' not in self.config.formats: + if ( + 'htmlzip' not in self.config.formats or + self.version.type == EXTERNAL + ): return False # We don't generate a zip for mkdocs currently. if self.is_type_sphinx(): @@ -970,7 +974,7 @@ def build_docs_localmedia(self): def build_docs_pdf(self): """Build PDF docs.""" - if 'pdf' not in self.config.formats: + if 'pdf' not in self.config.formats or self.version.type == EXTERNAL: return False # Mkdocs has no pdf generation currently. if self.is_type_sphinx(): @@ -979,7 +983,7 @@ def build_docs_pdf(self): def build_docs_epub(self): """Build ePub docs.""" - if 'epub' not in self.config.formats: + if 'epub' not in self.config.formats or self.version.type == EXTERNAL: return False # Mkdocs has no epub generation currently. if self.is_type_sphinx(): diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index 05110a25b96..97a6f448c71 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -8,6 +8,7 @@ from django_dynamic_fixture import get from mock import MagicMock, PropertyMock, patch +from readthedocs.builds.constants import EXTERNAL from readthedocs.builds.models import Version from readthedocs.config import ( ALL, @@ -441,6 +442,36 @@ def test_build_formats_only_pdf( assert not outcomes['localmedia'] assert not outcomes['epub'] + @patch('readthedocs.projects.models.Project.repo_nonblockinglock', new=MagicMock()) + @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs_class') + @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.build') + @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.append_conf') + def test_build_formats_only_html_for_external_versions( + self, append_conf, html_build, build_docs_class, + checkout_path, tmpdir, + ): + # Convert to external Version + self.version.type = EXTERNAL + self.version.save() + + checkout_path.return_value = str(tmpdir) + self.create_config_file(tmpdir, {'formats': ['pdf', 'htmlzip', 'epub']}) + + update_docs = self.get_update_docs_task() + python_env = Virtualenv( + version=self.version, + build_env=update_docs.build_env, + config=update_docs.config, + ) + update_docs.python_env = python_env + + outcomes = update_docs.build_docs() + + assert outcomes['html'] + assert not outcomes['pdf'] + assert not outcomes['localmedia'] + assert not outcomes['epub'] + @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_python_environment', new=MagicMock()) @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs', new=MagicMock()) @patch('readthedocs.doc_builder.environments.BuildEnvironment.failed', new_callable=PropertyMock) From b436ed1ac23a5c307329cdfacfe74a701914ae77 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 10 Jul 2019 12:40:53 +0600 Subject: [PATCH 141/171] doc string update --- readthedocs/projects/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index e5f4891a175..be0f334cc1e 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -545,8 +545,8 @@ def get_storage_path( :param type_: Media content type, ie - 'pdf', 'htmlzip' :param version_slug: Project version slug for lookup - :param version_type: Project version type :param include_file: Include file name in return + :param version_type: Project version type :return: the path to an item in storage (can be used with ``storage.url`` to get the URL) """ From 1117caa917c5f10de583a828855ed1c763fecd6d Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 10 Jul 2019 12:45:20 +0600 Subject: [PATCH 142/171] remove log test --- readthedocs/rtd_tests/tests/test_celery.py | 7 +------ readthedocs/rtd_tests/tests/test_oauth.py | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_celery.py b/readthedocs/rtd_tests/tests/test_celery.py index f33c2892b77..022c76a3294 100644 --- a/readthedocs/rtd_tests/tests/test_celery.py +++ b/readthedocs/rtd_tests/tests/test_celery.py @@ -347,9 +347,8 @@ def test_send_build_status_task(self, send_build_status): send_build_status.assert_called_once_with(external_build, BUILD_STATUS_SUCCESS) - @patch('readthedocs.projects.tasks.log') @patch('readthedocs.projects.tasks.GitHubService.send_build_status') - def test_send_build_status_task_without_remote_repo(self, send_build_status, mock_logger): + def test_send_build_status_task_without_remote_repo(self, send_build_status): external_version = get(Version, project=self.project, type=EXTERNAL) external_build = get( Build, project=self.project, version=external_version @@ -357,7 +356,3 @@ def test_send_build_status_task_without_remote_repo(self, send_build_status, moc tasks.send_build_status(external_build, BUILD_STATUS_SUCCESS) send_build_status.assert_not_called() - mock_logger.info.assert_called_with( - 'Remote repository does not exist for %s', - self.project, - ) diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index 49ac7bf64d4..b928e6b0aa5 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -191,8 +191,8 @@ def test_send_build_status_value_error(self, session, mock_logger): self.assertFalse(success) mock_logger.exception.assert_called_with( - 'GitHub commit status creation failed for project: %s', - self.project, + 'GitHub commit status creation failed for project: %s', + self.project, ) @override_settings(DEFAULT_PRIVACY_LEVEL='private') From 21bc2278e39529e92ddcc06cb8d45c8b9c34a71d Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 10 Jul 2019 23:02:07 +0600 Subject: [PATCH 143/171] add check for external version in conf.py.tmpl --- readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl index fca9d18d1f2..d5bcf740c96 100644 --- a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl +++ b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl @@ -141,6 +141,10 @@ if 'extensions' in globals(): else: extensions = ["readthedocs_ext.readthedocs"] +# Add External version warning banner to the external version documentation +if '{{ version.type }}' == 'external': + extensions.insert(1, "readthedocs_ext.external_version_warning") + project_language = '{{ project.language }}' # User's Sphinx configurations From d7b1d0c219879323480a0fc546e328cbd043cf82 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 10 Jul 2019 23:50:28 +0600 Subject: [PATCH 144/171] remove external versions from get_latest_build --- readthedocs/projects/models.py | 2 +- readthedocs/rtd_tests/tests/test_project.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 688409b8a88..8195fdde612 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -898,7 +898,7 @@ def get_latest_build(self, finished=True): kwargs = {'type': 'html'} if finished: kwargs['state'] = 'finished' - return self.builds.filter(**kwargs).first() + return self.builds(manager=INTERNAL).filter(**kwargs).first() def api_versions(self): from readthedocs.builds.models import APIVersion diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index ded11f52c53..a1f577e9c55 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -155,17 +155,23 @@ def test_all_active_versions_excludes_external_versions(self): self.assertNotIn(self.external_version, self.pip.all_active_versions()) def test_update_stable_version_excludes_external_versions(self): - # Delete all versions excluding PR Versions. + # Delete all versions excluding External Versions. self.pip.versions.exclude(type=EXTERNAL).delete() - # Test that PR Version is not considered for stable. + # Test that External Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) def test_has_good_build_excludes_external_versions(self): - # Delete all versions excluding PR Versions. + # Delete all versions excluding External Versions. self.pip.versions.exclude(type=EXTERNAL).delete() - # Test that PR Version is not considered for has_good_build. + # Test that External Version is not considered for has_good_build. self.assertFalse(self.pip.has_good_build) + def test_get_latest_build_excludes_external_versions(self): + # Delete all versions excluding External Versions. + self.pip.versions.exclude(type=EXTERNAL).delete() + # Test that External Version is not considered for get_latest_build. + self.assertEqual(self.pip.get_latest_build(), None) + class TestProjectTranslations(ProjectMixin, TestCase): From 9bc0e338084ed2185ee05b55be817ab8ee4bcff7 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 10 Jul 2019 16:03:04 -0700 Subject: [PATCH 145/171] Pass the build_pk to the task instead of the build object itself This breaks in celery when we will ship to prod. --- readthedocs/core/utils/__init__.py | 2 +- readthedocs/projects/tasks.py | 16 ++++++++-------- .../rtd_tests/tests/test_projects_tasks.py | 7 ++++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/readthedocs/core/utils/__init__.py b/readthedocs/core/utils/__init__.py index 8b8834d1f20..ab132ec7bc9 100644 --- a/readthedocs/core/utils/__init__.py +++ b/readthedocs/core/utils/__init__.py @@ -133,7 +133,7 @@ def prepare_build( if build: # Send pending Build Status using Git Status API for External Builds. - send_external_build_status(build.id, BUILD_STATUS_PENDING) + send_external_build_status(version=version, build_pk=build.id, status=BUILD_STATUS_PENDING) return ( update_docs_task.signature( diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 2a6a60cb8d2..3d2f55f3a4b 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -580,12 +580,12 @@ def run_build(self, docker, record): self.send_notifications(self.version.pk, self.build['id']) # send build failure status to git Status API send_external_build_status( - self.build['id'], BUILD_STATUS_FAILURE + version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_FAILURE ) elif self.build_env.successful: # send build successful status to git Status API send_external_build_status( - self.build['id'], BUILD_STATUS_SUCCESS + version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_SUCCESS ) else: msg = 'Unhandled Build State' @@ -1808,13 +1808,14 @@ def retry_domain_verification(domain_pk): @app.task(queue='web') -def send_build_status(build, state): +def send_build_status(build_pk, state): """ Send Build Status to Git Status API for project external versions. - :param build: Build + :param build_pk: Buil primary key :param state: build state failed, pending, or success to be sent. """ + build = Build.objects.get(pk=build_pk) try: if build.project.remote_repository.account.provider == 'github': service = GitHubService( @@ -1834,16 +1835,15 @@ def send_build_status(build, state): # TODO: Send build status for other providers. -def send_external_build_status(build_pk, state): +def send_external_build_status(version, build_pk, status): """ Check if build is external and Send Build Status for project external versions. :param build_pk: Build pk :param state: build state failed, pending, or success to be sent. """ - build = Build.objects.get(pk=build_pk) # Send status reports for only External (pull/merge request) Versions. - if build.version.type == EXTERNAL: + if version.type == EXTERNAL: # call the task that actually send the build status. - send_build_status.delay(build, state) + send_build_status.delay(build_pk, status) diff --git a/readthedocs/rtd_tests/tests/test_projects_tasks.py b/readthedocs/rtd_tests/tests/test_projects_tasks.py index 1ec1d8d0ae7..f8a5d4560bc 100644 --- a/readthedocs/rtd_tests/tests/test_projects_tasks.py +++ b/readthedocs/rtd_tests/tests/test_projects_tasks.py @@ -10,7 +10,6 @@ from readthedocs.projects.tasks import ( sync_files, send_external_build_status, - send_build_status, ) @@ -139,12 +138,14 @@ def setUp(self): @patch('readthedocs.projects.tasks.send_build_status') def test_send_external_build_status_with_external_version(self, send_build_status): - send_external_build_status(self.external_build.id, BUILD_STATUS_SUCCESS) + send_external_build_status(self.external_version, + self.external_build.id, BUILD_STATUS_SUCCESS) send_build_status.delay.assert_called_once_with(self.external_build, BUILD_STATUS_SUCCESS) @patch('readthedocs.projects.tasks.send_build_status') def test_send_external_build_status_with_internal_version(self, send_build_status): - send_external_build_status(self.internal_build.id, BUILD_STATUS_SUCCESS) + send_external_build_status(self.internal_version, + self.internal_build.id, BUILD_STATUS_SUCCESS) send_build_status.delay.assert_not_called() From 23a215ee7ae0f3e61e87143e24780a4d665a301b Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 11 Jul 2019 18:10:47 +0600 Subject: [PATCH 146/171] added Feature flag to enable External version building --- docs/guides/feature-flags.rst | 4 ++- readthedocs/api/v2/views/integrations.py | 33 ++++++++++++------------ readthedocs/projects/models.py | 5 ++++ readthedocs/rtd_tests/tests/test_api.py | 29 +++++++++++++++++++++ 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/docs/guides/feature-flags.rst b/docs/guides/feature-flags.rst index 2a61ff03ac7..6b6e6b6fe13 100644 --- a/docs/guides/feature-flags.rst +++ b/docs/guides/feature-flags.rst @@ -22,4 +22,6 @@ The ``DONT_SHALLOW_CLONE`` flag is useful if your code accesses old commits duri e.g. python-reno release notes manager is known to do that (error message line would probably include one of old Git commit id's). -``USE_TESTING_BUILD_IMAGE``: :featureflags:`USE_TESTING_BUILD_IMAGE` \ No newline at end of file +``USE_TESTING_BUILD_IMAGE``: :featureflags:`USE_TESTING_BUILD_IMAGE` + +``ENABLE_EXTERNAL_VERSION_BUILD``: :featureflags:`ENABLE_EXTERNAL_VERSION_BUILD` diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 2a9d715531e..e56441b13ef 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -27,7 +27,7 @@ build_external_version, ) from readthedocs.integrations.models import HttpExchange, Integration -from readthedocs.projects.models import Project +from readthedocs.projects.models import Project, Feature log = logging.getLogger(__name__) @@ -367,21 +367,22 @@ def handle_webhook(self): if event in (GITHUB_CREATE, GITHUB_DELETE): return self.sync_versions(self.project) - if event == GITHUB_PULL_REQUEST and action: - if ( - action in - [ - GITHUB_PULL_REQUEST_OPENED, - GITHUB_PULL_REQUEST_REOPENED, - GITHUB_PULL_REQUEST_SYNC - ] - ): - # Handle opened, synchronize, reopened pull_request event. - return self.get_external_version_response(self.project) - - if action == GITHUB_PULL_REQUEST_CLOSED: - # Handle closed pull_request event. - return self.get_delete_external_version_response(self.project) + if self.project.has_feature(Feature.ENABLE_EXTERNAL_VERSION_BUILD): + if event == GITHUB_PULL_REQUEST and action: + if ( + action in + [ + GITHUB_PULL_REQUEST_OPENED, + GITHUB_PULL_REQUEST_REOPENED, + GITHUB_PULL_REQUEST_SYNC + ] + ): + # Handle opened, synchronize, reopened pull_request event. + return self.get_external_version_response(self.project) + + if action == GITHUB_PULL_REQUEST_CLOSED: + # Handle closed pull_request event. + return self.get_delete_external_version_response(self.project) return None diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index c3b1a4bc7b0..68d12287551 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1419,6 +1419,7 @@ def add_features(sender, **kwargs): SHARE_SPHINX_DOCTREE = 'share_sphinx_doctree' DEFAULT_TO_MKDOCS_0_17_3 = 'default_to_mkdocs_0_17_3' CLEAN_AFTER_BUILD = 'clean_after_build' + ENABLE_EXTERNAL_VERSION_BUILD = 'enable_external_version_build' FEATURES = ( (USE_SPHINX_LATEST, _('Use latest version of Sphinx')), @@ -1459,6 +1460,10 @@ def add_features(sender, **kwargs): CLEAN_AFTER_BUILD, _('Clean all files used in the build process'), ), + ( + ENABLE_EXTERNAL_VERSION_BUILD, + _('Enable project to build on pull/merge requests'), + ), ) projects = models.ManyToManyField( diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index 0cfa3de8721..d37ae60fd56 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -768,6 +768,11 @@ class IntegrationsTests(TestCase): def setUp(self): self.project = get(Project) + self.feature_flag = get( + Feature, + projects=[self.project], + feature_id=Feature.ENABLE_EXTERNAL_VERSION_BUILD, + ) self.version = get( Version, slug='master', verbose_name='master', active=True, project=self.project, @@ -1105,6 +1110,30 @@ def test_github_pull_request_closed_event_invalid_payload(self, trigger_build): self.assertEqual(resp.status_code, 400) + @mock.patch('readthedocs.core.utils.trigger_build') + def test_github_pull_request_event_no_feature_flag(self, trigger_build, core_trigger_build): + # delete feature flag + self.feature_flag.delete() + + client = APIClient() + + headers = {GITHUB_EVENT_HEADER: GITHUB_PULL_REQUEST} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + self.github_pull_request_payload, + format='json', + **headers + ) + # get external version + external_version = self.project.versions( + manager=EXTERNAL + ).filter(verbose_name='2').first() + + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.data['detail'], 'Unhandled webhook event') + core_trigger_build.assert_not_called() + self.assertFalse(external_version) + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') def test_github_delete_event(self, sync_repository_task, trigger_build): client = APIClient() From 219e0dfe8bd0df209f558ac3d23206ad25f19928 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 11 Jul 2019 18:24:47 +0600 Subject: [PATCH 147/171] lint fix --- readthedocs/api/v2/views/integrations.py | 2 +- readthedocs/builds/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 2a9d715531e..b4550ac332f 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -302,7 +302,7 @@ def get_data(self): return super().get_data() def get_external_version_data(self): - """Get Commit Sha and pull request number from payload""" + """Get Commit Sha and pull request number from payload.""" try: identifier = self.data['pull_request']['head']['sha'] verbose_name = str(self.data['number']) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 677aa3fc51e..5ec7ca9313a 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -736,7 +736,7 @@ def get_absolute_url(self): return reverse('builds_detail', args=[self.project.slug, self.pk]) def get_full_url(self): - """Get full url including domain""" + """Get full url including domain.""" scheme = 'http' if settings.DEBUG else 'https' full_url = '{scheme}://{domain}{absolute_url}'.format( scheme=scheme, From 64174f46282859debd43937b2c94f87bb598c453 Mon Sep 17 00:00:00 2001 From: Eric Holscher <25510+ericholscher@users.noreply.github.com> Date: Thu, 11 Jul 2019 08:49:55 -0700 Subject: [PATCH 148/171] Apply suggestions from code review Co-Authored-By: Maksudul Haque --- readthedocs/projects/tasks.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 3d2f55f3a4b..8af28c0c1f5 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -588,7 +588,7 @@ def run_build(self, docker, record): version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_SUCCESS ) else: - msg = 'Unhandled Build State' + msg = 'Unhandled Build Status' log.warning( LOG_TEMPLATE, { @@ -1808,12 +1808,12 @@ def retry_domain_verification(domain_pk): @app.task(queue='web') -def send_build_status(build_pk, state): +def send_build_status(build_pk, status): """ Send Build Status to Git Status API for project external versions. - :param build_pk: Buil primary key - :param state: build state failed, pending, or success to be sent. + :param build_pk: Build primary key + :param status: build status failed, pending, or success to be sent. """ build = Build.objects.get(pk=build_pk) try: @@ -1839,8 +1839,9 @@ def send_external_build_status(version, build_pk, status): """ Check if build is external and Send Build Status for project external versions. - :param build_pk: Build pk - :param state: build state failed, pending, or success to be sent. + :param version: Version instance + :param build_pk: Build pk + :param status: build status failed, pending, or success to be sent. """ # Send status reports for only External (pull/merge request) Versions. From b4a39aa043301f2b6622e23ace435ab3834b78a7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 11 Jul 2019 22:08:09 +0600 Subject: [PATCH 149/171] update conditions --- readthedocs/api/v2/views/integrations.py | 34 +++++++++++++----------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index e56441b13ef..7fb9d172ecb 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -367,22 +367,24 @@ def handle_webhook(self): if event in (GITHUB_CREATE, GITHUB_DELETE): return self.sync_versions(self.project) - if self.project.has_feature(Feature.ENABLE_EXTERNAL_VERSION_BUILD): - if event == GITHUB_PULL_REQUEST and action: - if ( - action in - [ - GITHUB_PULL_REQUEST_OPENED, - GITHUB_PULL_REQUEST_REOPENED, - GITHUB_PULL_REQUEST_SYNC - ] - ): - # Handle opened, synchronize, reopened pull_request event. - return self.get_external_version_response(self.project) - - if action == GITHUB_PULL_REQUEST_CLOSED: - # Handle closed pull_request event. - return self.get_delete_external_version_response(self.project) + if ( + self.project.has_feature(Feature.ENABLE_EXTERNAL_VERSION_BUILD) and + event == GITHUB_PULL_REQUEST and action + ): + if ( + action in + [ + GITHUB_PULL_REQUEST_OPENED, + GITHUB_PULL_REQUEST_REOPENED, + GITHUB_PULL_REQUEST_SYNC + ] + ): + # Handle opened, synchronize, reopened pull_request event. + return self.get_external_version_response(self.project) + + if action == GITHUB_PULL_REQUEST_CLOSED: + # Handle closed pull_request event. + return self.get_delete_external_version_response(self.project) return None From 5466b14a0de2e05f1fb08a452ad28ce49a60e904 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 11 Jul 2019 22:47:49 +0600 Subject: [PATCH 150/171] tests fixed --- readthedocs/projects/tasks.py | 6 +++--- readthedocs/rtd_tests/tests/test_celery.py | 4 ++-- readthedocs/rtd_tests/tests/test_projects_tasks.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 8af28c0c1f5..005eddf7223 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -588,7 +588,7 @@ def run_build(self, docker, record): version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_SUCCESS ) else: - msg = 'Unhandled Build Status' + msg = 'Unhandled Build Status' log.warning( LOG_TEMPLATE, { @@ -1813,7 +1813,7 @@ def send_build_status(build_pk, status): Send Build Status to Git Status API for project external versions. :param build_pk: Build primary key - :param status: build status failed, pending, or success to be sent. + :param status: build status failed, pending, or success to be sent. """ build = Build.objects.get(pk=build_pk) try: @@ -1824,7 +1824,7 @@ def send_build_status(build_pk, status): ) # send Status report using the API. - service.send_build_status(build, state) + service.send_build_status(build, status) except RemoteRepository.DoesNotExist: log.info('Remote repository does not exist for %s', build.project) diff --git a/readthedocs/rtd_tests/tests/test_celery.py b/readthedocs/rtd_tests/tests/test_celery.py index 022c76a3294..441d5873583 100644 --- a/readthedocs/rtd_tests/tests/test_celery.py +++ b/readthedocs/rtd_tests/tests/test_celery.py @@ -343,7 +343,7 @@ def test_send_build_status_task(self, send_build_status): external_build = get( Build, project=self.project, version=external_version ) - tasks.send_build_status(external_build, BUILD_STATUS_SUCCESS) + tasks.send_build_status(external_build.id, BUILD_STATUS_SUCCESS) send_build_status.assert_called_once_with(external_build, BUILD_STATUS_SUCCESS) @@ -353,6 +353,6 @@ def test_send_build_status_task_without_remote_repo(self, send_build_status): external_build = get( Build, project=self.project, version=external_version ) - tasks.send_build_status(external_build, BUILD_STATUS_SUCCESS) + tasks.send_build_status(external_build.id, BUILD_STATUS_SUCCESS) send_build_status.assert_not_called() diff --git a/readthedocs/rtd_tests/tests/test_projects_tasks.py b/readthedocs/rtd_tests/tests/test_projects_tasks.py index f8a5d4560bc..1794cb17d54 100644 --- a/readthedocs/rtd_tests/tests/test_projects_tasks.py +++ b/readthedocs/rtd_tests/tests/test_projects_tasks.py @@ -141,7 +141,7 @@ def test_send_external_build_status_with_external_version(self, send_build_statu send_external_build_status(self.external_version, self.external_build.id, BUILD_STATUS_SUCCESS) - send_build_status.delay.assert_called_once_with(self.external_build, BUILD_STATUS_SUCCESS) + send_build_status.delay.assert_called_once_with(self.external_build.id, BUILD_STATUS_SUCCESS) @patch('readthedocs.projects.tasks.send_build_status') def test_send_external_build_status_with_internal_version(self, send_build_status): From 1167c40dad75cb327b50230033ae90c894bce0ba Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 12 Jul 2019 02:04:54 +0600 Subject: [PATCH 151/171] Update build list and detail page --- readthedocs/builds/constants.py | 2 ++ readthedocs/builds/models.py | 33 +++++++++++++++++++ readthedocs/builds/views.py | 15 --------- readthedocs/projects/constants.py | 4 +++ readthedocs/projects/views/public.py | 4 +-- .../templates/builds/build_detail.html | 11 +++++-- .../templates/core/build_list_detailed.html | 2 +- 7 files changed, 51 insertions(+), 20 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 3606bd7415b..b46afdb21a5 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -87,3 +87,5 @@ 'description': 'The build succeeded!', }, } + +GITHUB_EXTERNAL_VERSION_NAME = 'Pull Request' diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 5ec7ca9313a..4c862f2d4d5 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -22,6 +22,7 @@ from readthedocs.projects.constants import ( BITBUCKET_URL, GITHUB_URL, + GITHUB_PULL_REQEST_URL, GITLAB_URL, PRIVACY_CHOICES, PRIVATE, @@ -36,6 +37,7 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, BUILD_TYPES, + GITHUB_EXTERNAL_VERSION_NAME, INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, @@ -539,6 +541,23 @@ def get_bitbucket_url(self, docroot, filename, source_suffix='.rst'): source_suffix=source_suffix, ) + def get_external_version_url(self): + """Return a Pull/Merge Request URL.""" + repo_url = self.project.repo + user, repo = get_github_username_repo(repo_url) + + if not user and not repo: + return '' + + if 'github' in repo_url: + repo = repo.rstrip('/') + return GITHUB_PULL_REQEST_URL.format( + user=user, + repo=repo, + number=self.verbose_name + ) + return '' + class APIVersion(Version): @@ -756,6 +775,20 @@ def is_stale(self): mins_ago = timezone.now() - datetime.timedelta(minutes=5) return self.state == BUILD_STATE_TRIGGERED and self.date < mins_ago + @property + def is_external(self): + return self.version.type == EXTERNAL + + @property + def external_version_name(self): + if self.is_external: + try: + if self.project.remote_repository.account.provider == 'github': + return GITHUB_EXTERNAL_VERSION_NAME + except Exception: + return None + return None + def using_latest_config(self): return int(self.config.get('version', '1')) == LATEST_CONFIGURATION_VERSION diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index ab1c7dd5b5b..3333495f6c8 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -84,21 +84,6 @@ def post(self, request, project_slug): class BuildList(BuildBase, BuildTriggerMixin, ListView): - def get_queryset(self): - # this is used to include only internal version - # builds in the build list page - self.project_slug = self.kwargs.get('project_slug', None) - self.project = get_object_or_404( - Project.objects.protected(self.request.user), - slug=self.project_slug, - ) - queryset = Build.objects.public( - user=self.request.user, - project=self.project, - ).select_related('project', 'version') - - return queryset - def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 8b770830f79..dc27de3b08a 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -325,6 +325,10 @@ 'https://github.com/{user}/{repo}/' '{action}/{version}{docroot}{path}{source_suffix}' ) +GITHUB_PULL_REQEST_URL = ( + 'https://github.com/{user}/{repo}/' + 'pull/{number}' +) BITBUCKET_URL = ( 'https://bitbucket.org/{user}/{repo}/' 'src/{version}{docroot}{path}{source_suffix}' diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index 17c81867c2e..b41fcc49a1d 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -98,7 +98,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) project = self.get_object() - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=project, ) @@ -274,7 +274,7 @@ def project_versions(request, project_slug): slug=project_slug, ) - versions = Version.objects.public( + versions = Version.internal.public( user=request.user, project=project, only_active=False, diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index 1608f1bfdbc..fb1993bf4ef 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -79,10 +79,17 @@
- {% if request.user|is_admin:project %} + {% if request.user|is_admin:project and not build.is_external %} {{ build.version.slug }} {% else %} - {{ build.version.slug }} + {% if build.is_external %} + {% blocktrans with build.external_version_name as external_version_name %} + {{ external_version_name }} # + {% endblocktrans %} + {{ build.version.verbose_name }} + {% else %} + {{ build.version.slug }} + {% endif %} {% endif %} {% if build.is_stale %}{% endif %}{% if build.state != 'finished' %}{{ build.get_state_display }} {% else %} {% if build.success %}{% trans "Passed" %}{% else %}{% trans "Failed" %}{% endif %}{% endif %} - {% if build.version %}{% blocktrans with build.version.slug as slug and build.type as type %}version {{ slug }} ({{ type }}){% endblocktrans %}{% endif %}{% blocktrans with build.date|timesince as date %}{{ date }} ago{% endblocktrans %} + {% if build.version %}{% if build.is_external %}{% blocktrans %}External {% endblocktrans %}{% endif %}{% blocktrans with build.version.slug as slug and build.type as type %}version {{ slug }} ({{ type }}){% endblocktrans %}{% endif %}{% blocktrans with build.date|timesince as date %}{{ date }} ago{% endblocktrans %}
From d5af035122dbd10487e89b7a42b1e08a3add7f3a Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 12 Jul 2019 03:15:27 +0600 Subject: [PATCH 152/171] added commit url --- readthedocs/api/v2/serializers.py | 1 + readthedocs/builds/constants.py | 1 + readthedocs/builds/models.py | 23 ++++++++++++------- .../builds/static-src/builds/js/detail.js | 2 ++ readthedocs/builds/static/builds/js/detail.js | 2 +- readthedocs/projects/constants.py | 4 ++++ .../templates/builds/build_detail.html | 21 ++++++++--------- .../templates/core/build_list_detailed.html | 2 +- 8 files changed, 34 insertions(+), 22 deletions(-) diff --git a/readthedocs/api/v2/serializers.py b/readthedocs/api/v2/serializers.py index 6d9a1cd48a6..21bbb1040b2 100644 --- a/readthedocs/api/v2/serializers.py +++ b/readthedocs/api/v2/serializers.py @@ -118,6 +118,7 @@ class BuildSerializer(serializers.ModelSerializer): version_slug = serializers.ReadOnlyField(source='version.slug') docs_url = serializers.ReadOnlyField(source='version.get_absolute_url') state_display = serializers.ReadOnlyField(source='get_state_display') + vcs_url = serializers.ReadOnlyField(source='version.vcs_url') # Jsonfield needs an explicit serializer # https://github.com/dmkoch/django-jsonfield/issues/188#issuecomment-300439829 config = serializers.JSONField(required=False) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index b46afdb21a5..a80e5dbd155 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -89,3 +89,4 @@ } GITHUB_EXTERNAL_VERSION_NAME = 'Pull Request' +GENERIC_EXTERNAL_VERSION_NAME = 'External Version' diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 4c862f2d4d5..0f0c56de9bc 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -23,6 +23,7 @@ BITBUCKET_URL, GITHUB_URL, GITHUB_PULL_REQEST_URL, + GITHUB_PULL_REQEST_COMMIT_URL, GITLAB_URL, PRIVACY_CHOICES, PRIVATE, @@ -37,6 +38,7 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, BUILD_TYPES, + GENERIC_EXTERNAL_VERSION_NAME, GITHUB_EXTERNAL_VERSION_NAME, INTERNAL, LATEST, @@ -161,7 +163,7 @@ def vcs_url(self): Generate VCS (github, gitlab, bitbucket) URL for this version. Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. - Pull/merge Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. + Pull/merge Request Example: https://github.com/rtfd/readthedocs.org/pull/99/commits/b630b630 """ url = '' if self.slug == STABLE: @@ -173,9 +175,14 @@ def vcs_url(self): if self.type == EXTERNAL: if 'github' in self.project.repo: - slug_url = self.verbose_name - url = f'/pull/{slug_url}/' + user, repo = get_github_username_repo(self.project.repo) + return GITHUB_PULL_REQEST_COMMIT_URL.format( + user=user, + repo=repo, + number=self.verbose_name, + commit= self.commit_name + ) if 'gitlab' in self.project.repo: slug_url = self.identifier url = f'/merge_requests/{slug_url}/' @@ -544,12 +551,12 @@ def get_bitbucket_url(self, docroot, filename, source_suffix='.rst'): def get_external_version_url(self): """Return a Pull/Merge Request URL.""" repo_url = self.project.repo - user, repo = get_github_username_repo(repo_url) - - if not user and not repo: - return '' if 'github' in repo_url: + user, repo = get_github_username_repo(repo_url) + if not user and not repo: + return '' + repo = repo.rstrip('/') return GITHUB_PULL_REQEST_URL.format( user=user, @@ -786,7 +793,7 @@ def external_version_name(self): if self.project.remote_repository.account.provider == 'github': return GITHUB_EXTERNAL_VERSION_NAME except Exception: - return None + return GENERIC_EXTERNAL_VERSION_NAME return None def using_latest_config(self): diff --git a/readthedocs/builds/static-src/builds/js/detail.js b/readthedocs/builds/static-src/builds/js/detail.js index 58637d6b1ee..93f5911013e 100644 --- a/readthedocs/builds/static-src/builds/js/detail.js +++ b/readthedocs/builds/static-src/builds/js/detail.js @@ -52,6 +52,7 @@ function BuildDetailView(instance) { }); self.commit = ko.observable(instance.commit); self.docs_url = ko.observable(instance.docs_url); + self.vcs_url = ko.observable(instance.vcs_url); /* Others */ self.legacy_output = ko.observable(false); @@ -72,6 +73,7 @@ function BuildDetailView(instance) { self.length(data.length); self.commit(data.commit); self.docs_url(data.docs_url); + self.vcs_url(data.vcs_url); var n; for (n in data.commands) { var command = data.commands[n]; diff --git a/readthedocs/builds/static/builds/js/detail.js b/readthedocs/builds/static/builds/js/detail.js index 2b1cf9490fb..edc5a89ebdb 100644 --- a/readthedocs/builds/static/builds/js/detail.js +++ b/readthedocs/builds/static/builds/js/detail.js @@ -1 +1 @@ -require=function n(i,u,a){function c(t,e){if(!u[t]){if(!i[t]){var o="function"==typeof require&&require;if(!e&&o)return o(t,!0);if(d)return d(t,!0);var r=new Error("Cannot find module '"+t+"'");throw r.code="MODULE_NOT_FOUND",r}var s=u[t]={exports:{}};i[t][0].call(s.exports,function(e){return c(i[t][1][e]||e)},s,s.exports,n,i,u,a)}return u[t].exports}for(var d="function"==typeof require&&require,e=0;e {% if request.user|is_admin:project and not build.is_external %} {{ build.version.slug }} - {% else %} - {% if build.is_external %} - {% blocktrans with build.external_version_name as external_version_name %} - {{ external_version_name }} # - {% endblocktrans %} - {{ build.version.verbose_name }} - {% else %} - {{ build.version.slug }} - {% endif %} + {% elif build.is_external %} + {% blocktrans with build.external_version_name as external_version_name %} + {{ external_version_name }} + {% endblocktrans %} + #{{ build.version.verbose_name }} + {% else %} + {{ build.version.slug }} {% endif %} - - ({{ build.commit }}) + + ({{ build.commit }}) diff --git a/readthedocs/templates/core/build_list_detailed.html b/readthedocs/templates/core/build_list_detailed.html index 23991d14426..f8b0ef9e2b5 100644 --- a/readthedocs/templates/core/build_list_detailed.html +++ b/readthedocs/templates/core/build_list_detailed.html @@ -6,7 +6,7 @@ From 1dd6bf2ffb534045d89c848af945a26d277d0b78 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 12 Jul 2019 20:35:54 +0600 Subject: [PATCH 153/171] Added commit url for build details page --- readthedocs/builds/models.py | 48 ++++++++++++++----------------- readthedocs/projects/constants.py | 4 +-- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 0f0c56de9bc..f184960f73a 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -22,8 +22,8 @@ from readthedocs.projects.constants import ( BITBUCKET_URL, GITHUB_URL, - GITHUB_PULL_REQEST_URL, - GITHUB_PULL_REQEST_COMMIT_URL, + GITHUB_PULL_REQUEST_URL, + GITHUB_PULL_REQUEST_COMMIT_URL, GITLAB_URL, PRIVACY_CHOICES, PRIVATE, @@ -162,41 +162,36 @@ def vcs_url(self): """ Generate VCS (github, gitlab, bitbucket) URL for this version. - Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. - Pull/merge Request Example: https://github.com/rtfd/readthedocs.org/pull/99/commits/b630b630 + Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + External Version Example: https://github.com/rtfd/readthedocs.org/pull/99/commits/b630b630/. """ - url = '' - if self.slug == STABLE: - slug_url = self.ref - elif self.slug == LATEST: - slug_url = self.project.default_branch or self.project.vcs_repo().fallback_branch - else: - slug_url = self.slug - if self.type == EXTERNAL: if 'github' in self.project.repo: user, repo = get_github_username_repo(self.project.repo) - return GITHUB_PULL_REQEST_COMMIT_URL.format( + # Get Github Pull Request Commit URL. + return GITHUB_PULL_REQUEST_COMMIT_URL.format( user=user, repo=repo, number=self.verbose_name, commit= self.commit_name - ) - if 'gitlab' in self.project.repo: - slug_url = self.identifier - url = f'/merge_requests/{slug_url}/' + # TODO: Add VCS ULR for other Git Providers + return '' - if 'bitbucket' in self.project.repo: - slug_url = self.identifier - url = f'/pull-requests/{slug_url}' + url = '' + if self.slug == STABLE: + slug_url = self.ref + elif self.slug == LATEST: + slug_url = self.project.default_branch or self.project.vcs_repo().fallback_branch else: - if ('github' in self.project.repo) or ('gitlab' in self.project.repo): - url = f'/tree/{slug_url}/' + slug_url = self.slug + + if ('github' in self.project.repo) or ('gitlab' in self.project.repo): + url = f'/tree/{slug_url}/' - if 'bitbucket' in self.project.repo: - slug_url = self.identifier - url = f'/src/{slug_url}' + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/src/{slug_url}' # TODO: improve this replacing return self.project.repo.replace('git://', 'https://').replace('.git', '') + url @@ -558,11 +553,12 @@ def get_external_version_url(self): return '' repo = repo.rstrip('/') - return GITHUB_PULL_REQEST_URL.format( + return GITHUB_PULL_REQUEST_URL.format( user=user, repo=repo, number=self.verbose_name ) + # TODO: Add External Version ULR for other Git Providers return '' diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 16f3af28981..ea71e7d2395 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -325,11 +325,11 @@ 'https://github.com/{user}/{repo}/' '{action}/{version}{docroot}{path}{source_suffix}' ) -GITHUB_PULL_REQEST_URL = ( +GITHUB_PULL_REQUEST_URL = ( 'https://github.com/{user}/{repo}/' 'pull/{number}' ) -GITHUB_PULL_REQEST_COMMIT_URL = ( +GITHUB_PULL_REQUEST_COMMIT_URL = ( 'https://github.com/{user}/{repo}/' 'pull/{number}/commits/{commit}' ) From 2166770988364884c48f2ed7a5c9981e7d244a2d Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 12 Jul 2019 21:10:57 +0600 Subject: [PATCH 154/171] Added tests --- readthedocs/builds/models.py | 2 +- readthedocs/rtd_tests/tests/test_builds.py | 85 +++++++++++++++++++ .../rtd_tests/tests/test_project_views.py | 11 ++- readthedocs/rtd_tests/tests/test_version.py | 11 ++- 4 files changed, 105 insertions(+), 4 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index f184960f73a..7a892734300 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -173,7 +173,7 @@ def vcs_url(self): user=user, repo=repo, number=self.verbose_name, - commit= self.commit_name + commit=self.commit_name ) # TODO: Add VCS ULR for other Git Providers return '' diff --git a/readthedocs/rtd_tests/tests/test_builds.py b/readthedocs/rtd_tests/tests/test_builds.py index 76c2c09a75b..6a55262f10b 100644 --- a/readthedocs/rtd_tests/tests/test_builds.py +++ b/readthedocs/rtd_tests/tests/test_builds.py @@ -3,14 +3,23 @@ import os import mock +from django.contrib.auth.models import User from django.test import TestCase from django_dynamic_fixture import fixture, get from django.utils import timezone +from allauth.socialaccount.models import SocialAccount + +from readthedocs.builds.constants import ( + EXTERNAL, + GITHUB_EXTERNAL_VERSION_NAME, + GENERIC_EXTERNAL_VERSION_NAME +) from readthedocs.builds.models import Build, Version from readthedocs.doc_builder.config import load_yaml_config from readthedocs.doc_builder.environments import LocalBuildEnvironment from readthedocs.doc_builder.python_environments import Virtualenv +from readthedocs.oauth.models import RemoteRepository from readthedocs.projects.models import EnvironmentVariable, Project from readthedocs.projects.tasks import UpdateDocsTaskStep from readthedocs.rtd_tests.tests.test_config_integration import create_load @@ -369,7 +378,13 @@ def test_get_env_vars(self): class BuildModelTests(TestCase): def setUp(self): + self.eric = User(username='eric') + self.eric.set_password('test') + self.eric.save() + self.project = get(Project) + self.project.users.add(self.eric) + self.version = get(Version, project=self.project) def test_get_previous_build(self): @@ -582,3 +597,73 @@ def test_using_latest_config(self): build.save() self.assertTrue(build.using_latest_config()) + + def test_build_is_external(self): + # Turn the build version to EXTERNAL type. + self.version.type = EXTERNAL + self.version.save() + + external_build = get( + Build, + project=self.project, + version=self.version, + config={'version': 1}, + ) + + self.assertTrue(external_build.is_external) + + def test_build_is_not_external(self): + build = get( + Build, + project=self.project, + version=self.version, + config={'version': 1}, + ) + + self.assertFalse(build.is_external) + + def test_no_external_version_name(self): + build = get( + Build, + project=self.project, + version=self.version, + config={'version': 1}, + ) + + self.assertEqual(build.external_version_name, None) + + def test_external_version_name_github(self): + social_account = get(SocialAccount, provider='github') + remote_repo = get( + RemoteRepository, + account=social_account, + project=self.project + ) + remote_repo.users.add(self.eric) + + external_version = get(Version, project=self.project, type=EXTERNAL) + external_build = get( + Build, project=self.project, version=external_version + ) + + self.assertEqual( + external_build.external_version_name, + GITHUB_EXTERNAL_VERSION_NAME + ) + + def test_external_version_name_generic(self): + # Turn the build version to EXTERNAL type. + self.version.type = EXTERNAL + self.version.save() + + external_build = get( + Build, + project=self.project, + version=self.version, + config={'version': 1}, + ) + + self.assertEqual( + external_build.external_version_name, + GENERIC_EXTERNAL_VERSION_NAME + ) diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index a94e4faa9db..212c9d11732 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -406,11 +406,11 @@ def test_project_download_media(self): response = self.client.get(url) self.assertEqual(response.status_code, 302) - def test_project_detail_view_shows_external_versons(self): + def test_project_detail_view_only_shows_internal_versons(self): url = reverse('projects_detail', args=[self.pip.slug]) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertIn(self.external_version, response.context['versions']) + self.assertNotIn(self.external_version, response.context['versions']) def test_project_downloads_only_shows_internal_versons(self): url = reverse('project_downloads', args=[self.pip.slug]) @@ -418,6 +418,13 @@ def test_project_downloads_only_shows_internal_versons(self): self.assertEqual(response.status_code, 200) self.assertNotIn(self.external_version, response.context['versions']) + def test_project_versions_only_shows_internal_versons(self): + url = reverse('project_version_list', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.external_version, response.context['active_versions']) + self.assertNotIn(self.external_version, response.context['inactive_versions']) + class TestPrivateViews(MockBuildTestCase): def setUp(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py index 6ae29c3e241..b6a33be0e97 100644 --- a/readthedocs/rtd_tests/tests/test_version.py +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -46,7 +46,10 @@ def setUp(self): class TestVersionModel(VersionMixin, TestCase): def test_vcs_url_for_external_version(self): - expected_url = f'https://github.com/pypa/pip/pull/{self.external_version.verbose_name}/' + expected_url = 'https://github.com/pypa/pip/pull/{number}/commits/{sha}'.format( + number=self.external_version.verbose_name, + sha=self.external_version.identifier + ) self.assertEqual(self.external_version.vcs_url, expected_url) def test_vcs_url_for_latest_version(self): @@ -66,3 +69,9 @@ def test_commit_name_for_latest_version(self): def test_commit_name_for_external_version(self): self.assertEqual(self.external_version.commit_name, self.external_version.identifier) + + def test_get_external_version_url(self): + expected_url = 'https://github.com/pypa/pip/pull/{number}'.format( + number=self.external_version.verbose_name, + ) + self.assertEqual(self.external_version.get_external_version_url(), expected_url) From 9af5521d9736f33b92e11b06ace1b5fda968009c Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 12 Jul 2019 23:04:06 +0600 Subject: [PATCH 155/171] setting commit url updated --- readthedocs/api/v2/serializers.py | 2 +- readthedocs/builds/models.py | 93 ++++++++++++++----- .../builds/static-src/builds/js/detail.js | 4 +- readthedocs/builds/static/builds/js/detail.js | 2 +- readthedocs/projects/constants.py | 12 +++ readthedocs/rtd_tests/tests/test_builds.py | 50 +++++++++- readthedocs/rtd_tests/tests/test_version.py | 11 +-- .../templates/builds/build_detail.html | 4 +- 8 files changed, 138 insertions(+), 40 deletions(-) diff --git a/readthedocs/api/v2/serializers.py b/readthedocs/api/v2/serializers.py index 21bbb1040b2..d153166e612 100644 --- a/readthedocs/api/v2/serializers.py +++ b/readthedocs/api/v2/serializers.py @@ -118,7 +118,7 @@ class BuildSerializer(serializers.ModelSerializer): version_slug = serializers.ReadOnlyField(source='version.slug') docs_url = serializers.ReadOnlyField(source='version.get_absolute_url') state_display = serializers.ReadOnlyField(source='get_state_display') - vcs_url = serializers.ReadOnlyField(source='version.vcs_url') + commit_url = serializers.ReadOnlyField(source='get_commit_url') # Jsonfield needs an explicit serializer # https://github.com/dmkoch/django-jsonfield/issues/188#issuecomment-300439829 config = serializers.JSONField(required=False) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 7a892734300..e36b5a1744c 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -20,10 +20,13 @@ from readthedocs.config import LATEST_CONFIGURATION_VERSION from readthedocs.core.utils import broadcast from readthedocs.projects.constants import ( + BITBUCKET_COMMIT_URL, BITBUCKET_URL, + GITHUB_COMMIT_URL, GITHUB_URL, GITHUB_PULL_REQUEST_URL, GITHUB_PULL_REQUEST_COMMIT_URL, + GITLAB_COMMIT_URL, GITLAB_URL, PRIVACY_CHOICES, PRIVATE, @@ -163,19 +166,17 @@ def vcs_url(self): Generate VCS (github, gitlab, bitbucket) URL for this version. Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. - External Version Example: https://github.com/rtfd/readthedocs.org/pull/99/commits/b630b630/. + External Version Example: https://github.com/rtfd/readthedocs.org/pull/99/. """ if self.type == EXTERNAL: if 'github' in self.project.repo: user, repo = get_github_username_repo(self.project.repo) - # Get Github Pull Request Commit URL. - return GITHUB_PULL_REQUEST_COMMIT_URL.format( + return GITHUB_PULL_REQUEST_URL.format( user=user, repo=repo, number=self.verbose_name, - commit=self.commit_name ) - # TODO: Add VCS ULR for other Git Providers + # TODO: Add VCS URL for other Git Providers return '' url = '' @@ -543,24 +544,6 @@ def get_bitbucket_url(self, docroot, filename, source_suffix='.rst'): source_suffix=source_suffix, ) - def get_external_version_url(self): - """Return a Pull/Merge Request URL.""" - repo_url = self.project.repo - - if 'github' in repo_url: - user, repo = get_github_username_repo(repo_url) - if not user and not repo: - return '' - - repo = repo.rstrip('/') - return GITHUB_PULL_REQUEST_URL.format( - user=user, - repo=repo, - number=self.verbose_name - ) - # TODO: Add External Version ULR for other Git Providers - return '' - class APIVersion(Version): @@ -767,6 +750,62 @@ def get_full_url(self): ) return full_url + def get_commit_url(self): + """Return the commit URL.""" + repo_url = self.project.repo + if self.is_external: + if 'github' in repo_url: + user, repo = get_github_username_repo(repo_url) + if not user and not repo: + return '' + + repo = repo.rstrip('/') + return GITHUB_PULL_REQUEST_COMMIT_URL.format( + user=user, + repo=repo, + number=self.version.verbose_name, + commit=self.commit + ) + # TODO: Add External Version Commit URL for other Git Providers + else: + if 'github' in repo_url: + user, repo = get_github_username_repo(repo_url) + if not user and not repo: + return '' + + repo = repo.rstrip('/') + return GITHUB_COMMIT_URL.format( + user=user, + repo=repo, + commit=self.commit + ) + elif 'gitlab' in repo_url: + user, repo = get_gitlab_username_repo(repo_url) + if not user and not repo: + return '' + + repo = repo.rstrip('/') + return GITLAB_COMMIT_URL.format( + user=user, + repo=repo, + commit=self.commit + ) + elif 'bitbucket' in repo_url: + user, repo = get_bitbucket_username_repo(repo_url) + if not user and not repo: + return '' + + repo = repo.rstrip('/') + return BITBUCKET_COMMIT_URL.format( + user=user, + repo=repo, + commit=self.commit + ) + else: + log.info('Unknown Git provider for %s', self.project) + + return '' + @property def finished(self): """Return if build has a finished state.""" @@ -788,7 +827,15 @@ def external_version_name(self): try: if self.project.remote_repository.account.provider == 'github': return GITHUB_EXTERNAL_VERSION_NAME + # TODO: Add External Version Name for other Git Providers + except RemoteRepository.DoesNotExist: + log.info('Remote repository does not exist for %s', self.project) + return GENERIC_EXTERNAL_VERSION_NAME except Exception: + log.exception( + 'Unhandled exception raised for %s while getting external_version_name', + self.project + ) return GENERIC_EXTERNAL_VERSION_NAME return None diff --git a/readthedocs/builds/static-src/builds/js/detail.js b/readthedocs/builds/static-src/builds/js/detail.js index 93f5911013e..047c1ecc713 100644 --- a/readthedocs/builds/static-src/builds/js/detail.js +++ b/readthedocs/builds/static-src/builds/js/detail.js @@ -52,7 +52,7 @@ function BuildDetailView(instance) { }); self.commit = ko.observable(instance.commit); self.docs_url = ko.observable(instance.docs_url); - self.vcs_url = ko.observable(instance.vcs_url); + self.commit_url = ko.observable(instance.commit_url); /* Others */ self.legacy_output = ko.observable(false); @@ -73,7 +73,7 @@ function BuildDetailView(instance) { self.length(data.length); self.commit(data.commit); self.docs_url(data.docs_url); - self.vcs_url(data.vcs_url); + self.commit_url(data.commit_url); var n; for (n in data.commands) { var command = data.commands[n]; diff --git a/readthedocs/builds/static/builds/js/detail.js b/readthedocs/builds/static/builds/js/detail.js index edc5a89ebdb..e541a417d4c 100644 --- a/readthedocs/builds/static/builds/js/detail.js +++ b/readthedocs/builds/static/builds/js/detail.js @@ -1 +1 @@ -require=function n(u,i,a){function c(r,e){if(!i[r]){if(!u[r]){var t="function"==typeof require&&require;if(!e&&t)return t(r,!0);if(l)return l(r,!0);var o=new Error("Cannot find module '"+r+"'");throw o.code="MODULE_NOT_FOUND",o}var s=i[r]={exports:{}};u[r][0].call(s.exports,function(e){return c(u[r][1][e]||e)},s,s.exports,n,u,i,a)}return i[r].exports}for(var l="function"==typeof require&&require,e=0;e{{ external_version_name }} {% endblocktrans %} - #{{ build.version.verbose_name }} + #{{ build.version.verbose_name }} {% else %} {{ build.version.slug }} {% endif %} - ({{ build.commit }}) + ({{ build.commit }}) From 8310e588b94015425b83c8fa50dac56192def31f Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 12 Jul 2019 23:19:13 +0600 Subject: [PATCH 156/171] lint fix --- readthedocs/builds/models.py | 7 +++---- readthedocs/rtd_tests/tests/test_version.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index e36b5a1744c..6d7007924b0 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -70,6 +70,7 @@ get_gitlab_username_repo, ) from readthedocs.builds.version_slug import VersionSlugField +from readthedocs.oauth.models import RemoteRepository log = logging.getLogger(__name__) @@ -779,7 +780,7 @@ def get_commit_url(self): repo=repo, commit=self.commit ) - elif 'gitlab' in repo_url: + if 'gitlab' in repo_url: user, repo = get_gitlab_username_repo(repo_url) if not user and not repo: return '' @@ -790,7 +791,7 @@ def get_commit_url(self): repo=repo, commit=self.commit ) - elif 'bitbucket' in repo_url: + if 'bitbucket' in repo_url: user, repo = get_bitbucket_username_repo(repo_url) if not user and not repo: return '' @@ -801,8 +802,6 @@ def get_commit_url(self): repo=repo, commit=self.commit ) - else: - log.info('Unknown Git provider for %s', self.project) return '' diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py index 6ae29c3e241..b5033c3e581 100644 --- a/readthedocs/rtd_tests/tests/test_version.py +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -46,7 +46,7 @@ def setUp(self): class TestVersionModel(VersionMixin, TestCase): def test_vcs_url_for_external_version(self): - expected_url = f'https://github.com/pypa/pip/pull/{self.external_version.verbose_name}/' + expected_url = f'https://github.com/pypa/pip/pull/{self.external_version.verbose_name}' self.assertEqual(self.external_version.vcs_url, expected_url) def test_vcs_url_for_latest_version(self): From 763b857626004deb987878d80f6e4513ea59922f Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 15:10:15 +0600 Subject: [PATCH 157/171] commit url return updated --- readthedocs/builds/models.py | 2 +- readthedocs/templates/builds/build_detail.html | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 6d7007924b0..fc1334f4c56 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -803,7 +803,7 @@ def get_commit_url(self): commit=self.commit ) - return '' + return None @property def finished(self): diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index 5e04dd05795..ae54a116d44 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -91,7 +91,17 @@ {% endif %} - ({{ build.commit }}) + {% if build.get_commit_url %} + ( + + {{ build.commit }} + + ) + {% else %} + ( + {{ build.commit }} + ) + {% endif %} From 2537fc367896e795ff55d62d2e914607bd1822b5 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 18:46:18 +0600 Subject: [PATCH 158/171] small updates according to suggestions --- readthedocs/api/v2/views/integrations.py | 4 ++-- readthedocs/builds/constants.py | 2 ++ readthedocs/builds/models.py | 8 ++++++-- readthedocs/builds/views.py | 2 +- readthedocs/oauth/services/github.py | 10 +++++++--- readthedocs/projects/constants.py | 2 +- readthedocs/projects/models.py | 4 ++-- readthedocs/rtd_tests/tests/test_oauth.py | 3 ++- readthedocs/vcs_support/backends/git.py | 11 +++++------ 9 files changed, 28 insertions(+), 18 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index d974d7ed93d..0897da89b20 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -200,7 +200,7 @@ def sync_versions(self, project): def get_external_version_response(self, project): """ - Build External version on pull/merge request events and return API response. + Trigger builds for External versions on pull/merge request events and return API response. Return a JSON response with the following:: @@ -211,7 +211,7 @@ def get_external_version_response(self, project): } :param project: Project instance - :type project: Project + :type project: readthedocs.projects.models.Project """ identifier, verbose_name = self.get_external_version_data() # create or get external version object using `verbose_name`. diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index a80e5dbd155..d2a375363b4 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -90,3 +90,5 @@ GITHUB_EXTERNAL_VERSION_NAME = 'Pull Request' GENERIC_EXTERNAL_VERSION_NAME = 'External Version' + +RTD_BUILD_STATUS_API_NAME = 'continuous-documentation/read-the-docs' diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index fc1334f4c56..54dd6848e03 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -212,7 +212,7 @@ def config(self): """ last_build = ( self.builds(manager=INTERNAL).filter( - state='finished', + state=BUILD_STATE_FINISHED, success=True, ).order_by('-date') .only('_config') @@ -742,7 +742,11 @@ def get_absolute_url(self): return reverse('builds_detail', args=[self.project.slug, self.pk]) def get_full_url(self): - """Get full url including domain.""" + """ + Get full url of the build including domain. + + Example: https://readthedocs.org/projects/pip/builds/99999999/ + """ scheme = 'http' if settings.DEBUG else 'https' full_url = '{scheme}://{domain}{absolute_url}'.format( scheme=scheme, diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index 3333495f6c8..65b3d43c9da 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -35,7 +35,7 @@ class BuildBase: def get_queryset(self): self.project_slug = self.kwargs.get('project_slug', None) self.project = get_object_or_404( - Project.objects.protected(self.request.user), + Project.objects.public(self.request.user), slug=self.project_slug, ) queryset = Build.objects.public( diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index ce978f99996..c45c2ac4e42 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -12,7 +12,10 @@ from readthedocs.api.v2.client import api from readthedocs.builds import utils as build_utils -from readthedocs.builds.constants import SELECT_BUILD_STATUS +from readthedocs.builds.constants import ( + SELECT_BUILD_STATUS, + RTD_BUILD_STATUS_API_NAME +) from readthedocs.integrations.models import Integration from ..models import RemoteOrganization, RemoteRepository @@ -336,7 +339,7 @@ def send_build_status(self, build, state): 'state': github_build_state, 'target_url': build.get_full_url(), 'description': description, - 'context': 'continuous-documentation/read-the-docs' + 'context': RTD_BUILD_STATUS_API_NAME } resp = None @@ -349,7 +352,8 @@ def send_build_status(self, build, state): ) if resp.status_code == 201: log.info( - 'GitHub commit status for project: %s', + "GitHub '%s' commit status created for project: %s", + github_build_state, project, ) return True diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 8d16db1e66f..78a793bb391 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -354,4 +354,4 @@ 'commit/{commit}' ) -GITHUB_GIT_PATTERN = 'pull/{id}/head:external-{id}' +GITHUB_PR_PULL_PATTERN = 'pull/{id}/head:external-{id}' diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index b38ac579afd..6e647afe284 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1407,7 +1407,7 @@ def add_features(sender, **kwargs): SHARE_SPHINX_DOCTREE = 'share_sphinx_doctree' DEFAULT_TO_MKDOCS_0_17_3 = 'default_to_mkdocs_0_17_3' CLEAN_AFTER_BUILD = 'clean_after_build' - ENABLE_EXTERNAL_VERSION_BUILD = 'enable_external_version_build' + EXTERNAL_VERSION_BUILD = 'enable_external_version_build' UPDATE_CONDA_STARTUP = 'update_conda_startup' FEATURES = ( @@ -1450,7 +1450,7 @@ def add_features(sender, **kwargs): _('Clean all files used in the build process'), ), ( - ENABLE_EXTERNAL_VERSION_BUILD, + EXTERNAL_VERSION_BUILD, _('Enable project to build on pull/merge requests'), ), ( diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index b928e6b0aa5..fdabb75206a 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -161,7 +161,8 @@ def test_send_build_status_successful(self, session, mock_logger): self.assertTrue(success) mock_logger.info.assert_called_with( - 'GitHub commit status for project: %s', + "GitHub '%s' commit status created for project: %s", + BUILD_STATUS_SUCCESS, self.project ) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 34556bf78c0..49642751dc7 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -12,7 +12,7 @@ from readthedocs.builds.constants import EXTERNAL from readthedocs.config import ALL -from readthedocs.projects.constants import GITHUB_GIT_PATTERN +from readthedocs.projects.constants import GITHUB_PR_PULL_PATTERN from readthedocs.projects.exceptions import RepositoryError from readthedocs.projects.validators import validate_submodule_url from readthedocs.vcs_support.base import BaseVCS, VCSVersion @@ -57,7 +57,6 @@ def update(self): super().update() if self.repo_exists(): self.set_remote_url(self.repo_url) - # A fetch is always required to get external versions properly return self.fetch() self.make_clean_working_dir() # A fetch is always required to get external versions properly @@ -154,18 +153,18 @@ def fetch(self): cmd = ['git', 'fetch', 'origin', '--tags', '--prune', '--prune-tags'] + if self.use_shallow_clone(): + cmd.extend(['--depth', str(self.repo_depth)]) + if ( self.verbose_name and self.version_type == EXTERNAL and 'github.com' in self.repo_url ): cmd.append( - GITHUB_GIT_PATTERN.format(id=self.verbose_name) + GITHUB_PR_PULL_PATTERN.format(id=self.verbose_name) ) - if self.use_shallow_clone(): - cmd.extend(['--depth', str(self.repo_depth)]) - code, stdout, stderr = self.run(*cmd) if code != 0: raise RepositoryError From 9e643dcef93e60cae8758f850ad0e9866bac1685 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 18:54:17 +0600 Subject: [PATCH 159/171] Send failure status when build state in unknown --- readthedocs/projects/tasks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 49c87cfb459..c5c33999963 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -589,6 +589,10 @@ def run_build(self, docker, record): ) else: msg = 'Unhandled Build Status' + # send build failure status to git Status API when Unhandled Build Status Found. + send_external_build_status( + version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_FAILURE + ) log.warning( LOG_TEMPLATE, { From 999070b17ad5ad1b8ff326fd58243952bb8152c3 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 18:55:37 +0600 Subject: [PATCH 160/171] Test Fix --- readthedocs/api/v2/views/integrations.py | 2 +- readthedocs/rtd_tests/tests/test_api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 0897da89b20..d0d5f5cc18f 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -369,7 +369,7 @@ def handle_webhook(self): return self.sync_versions(self.project) if ( - self.project.has_feature(Feature.ENABLE_EXTERNAL_VERSION_BUILD) and + self.project.has_feature(Feature.EXTERNAL_VERSION_BUILD) and event == GITHUB_PULL_REQUEST and action ): if ( diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index d37ae60fd56..ea44b9527a7 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -771,7 +771,7 @@ def setUp(self): self.feature_flag = get( Feature, projects=[self.project], - feature_id=Feature.ENABLE_EXTERNAL_VERSION_BUILD, + feature_id=Feature.EXTERNAL_VERSION_BUILD, ) self.version = get( Version, slug='master', verbose_name='master', From 4f1709e6e7e3cc41d4b11801a2f793089b7dda59 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 19:10:41 +0600 Subject: [PATCH 161/171] build details page UI fix --- readthedocs/templates/builds/build_detail.html | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index ae54a116d44..26fd120394c 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -92,15 +92,9 @@ {% if build.get_commit_url %} - ( - - {{ build.commit }} - - ) + ({{ build.commit }}) {% else %} - ( - {{ build.commit }} - ) + ({{ build.commit }}) {% endif %} From ce1353e594772502d9933c0b303cdb47d1df9049 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 19:37:39 +0600 Subject: [PATCH 162/171] Added get_or_create for external version creation --- readthedocs/core/views/hooks.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 1e6ac93e362..4ca623d0825 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -105,30 +105,23 @@ def get_or_create_external_version(project, identifier, verbose_name): :returns: External version. :rtype: Version """ - external_version = project.versions( - manager=EXTERNAL - ).filter(verbose_name=verbose_name).first() + external_version, created = project.versions.get_or_create( + verbose_name=verbose_name, + type=EXTERNAL, + defaults={'identifier': identifier, 'active': True}, + ) - if external_version: + if not created: # identifier will change if there is a new commit to the Pull/Merge Request if external_version.identifier != identifier: external_version.identifier = identifier external_version.save() else: - # create an external version if the version does not exist. - created_external_version = Version.objects.create( - project=project, - type=EXTERNAL, - identifier=identifier, - verbose_name=verbose_name, - active=True - ) log.info( '(Create External Version) Added Version: [%s] ', ' '.join( - created_external_version.slug + external_version.slug ) ) - return created_external_version return external_version From f84ee698b18027e0923a319f13e8859a3cdb882d Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 21:16:10 +0600 Subject: [PATCH 163/171] create external version updated --- readthedocs/core/views/hooks.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 4ca623d0825..91717310c19 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -111,17 +111,23 @@ def get_or_create_external_version(project, identifier, verbose_name): defaults={'identifier': identifier, 'active': True}, ) - if not created: - # identifier will change if there is a new commit to the Pull/Merge Request - if external_version.identifier != identifier: - external_version.identifier = identifier - external_version.save() - else: + if created: log.info( '(Create External Version) Added Version: [%s] ', ' '.join( external_version.slug ) ) + else: + # identifier will change if there is a new commit to the Pull/Merge Request + if external_version.identifier != identifier: + external_version.identifier = identifier + external_version.save() + + log.info( + '(Update External Version) Updated Version: [%s] ', ' '.join( + external_version.slug + ) + ) return external_version From b1fe82f7e588f16dc0bafb46687107df71962d6c Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 21:23:31 +0600 Subject: [PATCH 164/171] comments updated --- readthedocs/projects/tasks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index c5c33999963..80c482b1f63 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -578,18 +578,21 @@ def run_build(self, docker, record): if self.build_env.failed: self.send_notifications(self.version.pk, self.build['id']) - # send build failure status to git Status API + # if the build failed, send build failure status to git Status API + # to show status report on the providers pull/merge request UI. send_external_build_status( version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_FAILURE ) elif self.build_env.successful: - # send build successful status to git Status API + # if the build succeed, send build successful status to git Status API + # to show status report on the providers pull/merge request UI. send_external_build_status( version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_SUCCESS ) else: msg = 'Unhandled Build Status' - # send build failure status to git Status API when Unhandled Build Status Found. + # if unhandled build status found, send build failure status to git Status API + # to show status report on the providers pull/merge request UI. send_external_build_status( version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_FAILURE ) From 8ed034512da76e27cf9bc11e76f20226fcc3fc6f Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 21:41:45 +0600 Subject: [PATCH 165/171] log message updated --- readthedocs/oauth/services/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index c45c2ac4e42..d7701c54293 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -352,9 +352,9 @@ def send_build_status(self, build, state): ) if resp.status_code == 201: log.info( - "GitHub '%s' commit status created for project: %s", - github_build_state, + "GitHub commit status created for project: %s, commit status: %s", project, + github_build_state, ) return True From 290bab8ab4362326dd64bb48a2eddff90b5e7c75 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 21:53:05 +0600 Subject: [PATCH 166/171] log mistake fix --- readthedocs/core/views/hooks.py | 15 ++++++--------- readthedocs/projects/tasks.py | 6 ------ 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 91717310c19..55cfbb4c624 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -113,9 +113,8 @@ def get_or_create_external_version(project, identifier, verbose_name): if created: log.info( - '(Create External Version) Added Version: [%s] ', ' '.join( - external_version.slug - ) + '(Create External Version) Added Version: [%s] ', + external_version.slug ) else: # identifier will change if there is a new commit to the Pull/Merge Request @@ -124,9 +123,8 @@ def get_or_create_external_version(project, identifier, verbose_name): external_version.save() log.info( - '(Update External Version) Updated Version: [%s] ', ' '.join( - external_version.slug - ) + '(Update External Version) Updated Version: [%s] ', + external_version.slug ) return external_version @@ -151,9 +149,8 @@ def delete_external_version(project, identifier, verbose_name): # Delete External Version external_version.delete() log.info( - '(Delete External Version) Deleted Version: [%s]', ' '.join( - external_version.slug - ) + '(Delete External Version) Deleted Version: [%s]', + external_version.slug ) return external_version.verbose_name diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 80c482b1f63..05b268da4a0 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -578,21 +578,15 @@ def run_build(self, docker, record): if self.build_env.failed: self.send_notifications(self.version.pk, self.build['id']) - # if the build failed, send build failure status to git Status API - # to show status report on the providers pull/merge request UI. send_external_build_status( version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_FAILURE ) elif self.build_env.successful: - # if the build succeed, send build successful status to git Status API - # to show status report on the providers pull/merge request UI. send_external_build_status( version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_SUCCESS ) else: msg = 'Unhandled Build Status' - # if unhandled build status found, send build failure status to git Status API - # to show status report on the providers pull/merge request UI. send_external_build_status( version=self.version, build_pk=self.build['id'], status=BUILD_STATUS_FAILURE ) From bf8247fff31d76375deeaa81881e25ae47f0fefe Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 16 Jul 2019 23:30:25 +0600 Subject: [PATCH 167/171] test fix --- readthedocs/rtd_tests/tests/test_config_integration.py | 3 ++- readthedocs/rtd_tests/tests/test_oauth.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index 97a6f448c71..394df0c53e8 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -8,7 +8,7 @@ from django_dynamic_fixture import get from mock import MagicMock, PropertyMock, patch -from readthedocs.builds.constants import EXTERNAL +from readthedocs.builds.constants import EXTERNAL, BUILD_STATE_TRIGGERED from readthedocs.builds.models import Version from readthedocs.config import ( ALL, @@ -368,6 +368,7 @@ def get_update_docs_task(self): config=load_yaml_config(self.version), project=self.project, version=self.version, + build={'id': 99, 'state': BUILD_STATE_TRIGGERED} ) return update_docs diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index fdabb75206a..3cf88f2d40d 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -161,9 +161,9 @@ def test_send_build_status_successful(self, session, mock_logger): self.assertTrue(success) mock_logger.info.assert_called_with( - "GitHub '%s' commit status created for project: %s", - BUILD_STATUS_SUCCESS, - self.project + "GitHub commit status created for project: %s, commit status: %s", + self.project, + BUILD_STATUS_SUCCESS ) @mock.patch('readthedocs.oauth.services.github.log') From 249e393fcf44775476d8e8c1dad000080c3991d7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 17 Jul 2019 14:03:53 +0600 Subject: [PATCH 168/171] update build status message --- readthedocs/builds/constants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index d2a375363b4..d24d84f13d9 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -76,15 +76,15 @@ SELECT_BUILD_STATUS = { BUILD_STATUS_FAILURE: { 'github': GITHUB_BUILD_STATE_FAILURE, - 'description': 'The build failed!', + 'description': 'Read the Docs build failed!', }, BUILD_STATUS_PENDING: { 'github': GITHUB_BUILD_STATE_PENDING, - 'description': 'The build is pending!', + 'description': 'Read the Docs build is in progress!', }, BUILD_STATUS_SUCCESS: { 'github': GITHUB_BUILD_STATE_SUCCESS, - 'description': 'The build succeeded!', + 'description': 'Read the Docs build succeeded!', }, } From 008051a99c0dbb21463915621d9a8bb1d5398594 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 17 Jul 2019 15:11:03 +0600 Subject: [PATCH 169/171] small updates --- docs/guides/feature-flags.rst | 2 +- readthedocs/api/v2/views/footer_views.py | 2 +- readthedocs/builds/constants.py | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/guides/feature-flags.rst b/docs/guides/feature-flags.rst index 5c539e8427b..884b16f5b8c 100644 --- a/docs/guides/feature-flags.rst +++ b/docs/guides/feature-flags.rst @@ -30,4 +30,4 @@ e.g. python-reno release notes manager is known to do that ``USE_TESTING_BUILD_IMAGE``: :featureflags:`USE_TESTING_BUILD_IMAGE` -``ENABLE_EXTERNAL_VERSION_BUILD``: :featureflags:`ENABLE_EXTERNAL_VERSION_BUILD` +``EXTERNAL_VERSION_BUILD``: :featureflags:`EXTERNAL_VERSION_BUILD` diff --git a/readthedocs/api/v2/views/footer_views.py b/readthedocs/api/v2/views/footer_views.py index 04fedeadd3f..1036ea961b1 100644 --- a/readthedocs/api/v2/views/footer_views.py +++ b/readthedocs/api/v2/views/footer_views.py @@ -9,7 +9,7 @@ from rest_framework_jsonp.renderers import JSONPRenderer from readthedocs.api.v2.signals import footer_response -from readthedocs.builds.constants import LATEST, TAG, INTERNAL +from readthedocs.builds.constants import LATEST, TAG from readthedocs.builds.models import Version from readthedocs.projects.models import Project from readthedocs.projects.version_handling import ( diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index d2a375363b4..262aee18474 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -62,33 +62,33 @@ STABLE, ) -# GitHub Build Statuses -GITHUB_BUILD_STATE_FAILURE = 'failure' -GITHUB_BUILD_STATE_PENDING = 'pending' -GITHUB_BUILD_STATE_SUCCESS = 'success' - # General Build Statuses BUILD_STATUS_FAILURE = 'failed' BUILD_STATUS_PENDING = 'pending' BUILD_STATUS_SUCCESS = 'success' +# GitHub Build Statuses +GITHUB_BUILD_STATUS_FAILURE = 'failure' +GITHUB_BUILD_STATUS_PENDING = 'pending' +GITHUB_BUILD_STATUS_SUCCESS = 'success' + # Used to select correct Build status and description to be sent to each service API SELECT_BUILD_STATUS = { BUILD_STATUS_FAILURE: { - 'github': GITHUB_BUILD_STATE_FAILURE, + 'github': GITHUB_BUILD_STATUS_FAILURE, 'description': 'The build failed!', }, BUILD_STATUS_PENDING: { - 'github': GITHUB_BUILD_STATE_PENDING, + 'github': GITHUB_BUILD_STATUS_PENDING, 'description': 'The build is pending!', }, BUILD_STATUS_SUCCESS: { - 'github': GITHUB_BUILD_STATE_SUCCESS, + 'github': GITHUB_BUILD_STATUS_SUCCESS, 'description': 'The build succeeded!', }, } +RTD_BUILD_STATUS_API_NAME = 'continuous-documentation/read-the-docs' + GITHUB_EXTERNAL_VERSION_NAME = 'Pull Request' GENERIC_EXTERNAL_VERSION_NAME = 'External Version' - -RTD_BUILD_STATUS_API_NAME = 'continuous-documentation/read-the-docs' From b3128ae2f9a9d1d3f8758e484366ef6f8ea3d602 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 17 Jul 2019 15:32:26 +0600 Subject: [PATCH 170/171] external version feature flag fix --- readthedocs/projects/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 6e647afe284..07c7224fcc6 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1407,7 +1407,7 @@ def add_features(sender, **kwargs): SHARE_SPHINX_DOCTREE = 'share_sphinx_doctree' DEFAULT_TO_MKDOCS_0_17_3 = 'default_to_mkdocs_0_17_3' CLEAN_AFTER_BUILD = 'clean_after_build' - EXTERNAL_VERSION_BUILD = 'enable_external_version_build' + EXTERNAL_VERSION_BUILD = 'external_version_build' UPDATE_CONDA_STARTUP = 'update_conda_startup' FEATURES = ( From 49e3fe495592cc47feacc1a8441ea40411ba6a95 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 18 Jul 2019 16:46:22 +0600 Subject: [PATCH 171/171] update status msg --- readthedocs/builds/constants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 262aee18474..d122c9065de 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -76,15 +76,15 @@ SELECT_BUILD_STATUS = { BUILD_STATUS_FAILURE: { 'github': GITHUB_BUILD_STATUS_FAILURE, - 'description': 'The build failed!', + 'description': 'Read the Docs build failed!', }, BUILD_STATUS_PENDING: { 'github': GITHUB_BUILD_STATUS_PENDING, - 'description': 'The build is pending!', + 'description': 'Read the Docs build is in progress!', }, BUILD_STATUS_SUCCESS: { 'github': GITHUB_BUILD_STATUS_SUCCESS, - 'description': 'The build succeeded!', + 'description': 'Read the Docs build succeeded!', }, }