POC: Feat/make indexing more resiliant

Directory.Build.props

@@ -35,7 +35,7 @@
     <EnableStrictModeForCompatibleTfms>true</EnableStrictModeForCompatibleTfms>
   </PropertyGroup>
 
-  <!-- Calculate version only once for the whole repository -->
+  <!-- Calculate version only once for the whole repository --> 
   <PropertyGroup>
     <GitVersionBaseDirectory>$(MSBuildThisFileDirectory)</GitVersionBaseDirectory>
   </PropertyGroup>

Directory.Packages.props

@@ -45,8 +45,8 @@
     <PackageVersion Include="Asp.Versioning.Mvc" Version="7.1.1" />
     <PackageVersion Include="Asp.Versioning.Mvc.ApiExplorer" Version="7.1.0" />
     <PackageVersion Include="Dazinator.Extensions.FileProviders" Version="2.0.0" />
-    <PackageVersion Include="Examine" Version="3.7.0" />
-    <PackageVersion Include="Examine.Core" Version="3.7.0" />
+    <PackageVersion Include="Examine" Version="3.7.1" />
+    <PackageVersion Include="Examine.Core" Version="3.7.1" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.11.71" />
     <PackageVersion Include="K4os.Compression.LZ4" Version="1.3.8" />
     <PackageVersion Include="MailKit" Version="4.8.0" />

build/azure-pipelines.yml

@@ -338,7 +338,9 @@ stages:
       # Integration Tests (SQL Server)
       - job:
         timeoutInMinutes: 120
-        condition: or(eq(stageDependencies.Build.A.outputs['build.NBGV_PublicRelease'], 'True'), ${{parameters.sqlServerIntegrationTests}})
+        # We are currently encountering issues when running SQL Server Linux tests Microsoft.Data.SqlClient.SqlException (0x80131904)
+        # condition: or(eq(stageDependencies.Build.A.outputs['build.NBGV_PublicRelease'], 'True'), ${{parameters.sqlServerIntegrationTests}})
+        condition: eq(${{parameters.sqlServerIntegrationTests}}, True)
         displayName: Integration Tests (SQL Server)
         strategy:
           matrix:
@@ -529,8 +531,8 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Install Playwright and dependencies
-          - pwsh: npx playwright install --with-deps
-            displayName: Install Playwright
+          - pwsh: npx playwright install chromium
+            displayName: Install Playwright only with Chromium browser
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
@@ -560,13 +562,24 @@ stages:
             displayName: Copy Playwright results
             condition: succeededOrFailed()
 
-          # Publish
+          # Publish test artifacts
           - task: PublishPipelineArtifact@1
             displayName: Publish test artifacts
             condition: succeededOrFailed()
             inputs:
               targetPath: $(Build.ArtifactStagingDirectory)
               artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
+
+          # Publish test results
+          - task: PublishTestResults@2
+            displayName: "Publish test results"
+            condition: succeededOrFailed()
+            inputs:
+              testResultsFormat: 'JUnit'
+              testResultsFiles: '*.xml'
+              searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
+              testRunTitle: "$(Agent.JobName)"
+
       - job:
         displayName: E2E Tests (SQL Server)
         condition: or(eq(stageDependencies.Build.A.outputs['build.NBGV_PublicRelease'], 'True'), ${{parameters.sqlServerAcceptanceTests}})
@@ -685,14 +698,13 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Install Playwright and dependencies
-          - pwsh: npx playwright install --with-deps
-            displayName: Install Playwright
+          - pwsh: npx playwright install chromium
+            displayName: Install Playwright only with Chromium browser
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
           - pwsh: $(testCommand)
             displayName: Run Playwright tests
-            continueOnError: true
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
             env:
               CI: true
@@ -725,14 +737,24 @@ stages:
             displayName: Copy Playwright results
             condition: succeededOrFailed()
 
-          # Publish
+          # Publish test artifacts
           - task: PublishPipelineArtifact@1
             displayName: Publish test artifacts
             condition: succeededOrFailed()
             inputs:
               targetPath: $(Build.ArtifactStagingDirectory)
               artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
 
+          # Publish test results
+          - task: PublishTestResults@2
+            displayName: "Publish test results"
+            condition: succeededOrFailed()
+            inputs:
+              testResultsFormat: 'JUnit'
+              testResultsFiles: '*.xml'
+              searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
+              testRunTitle: "$(Agent.JobName)"
+
   ###############################################
   ## Release
   ###############################################
@@ -745,6 +767,8 @@ stages:
     condition: and(succeeded(), or(eq(dependencies.Build.outputs['A.build.NBGV_PublicRelease'], 'True'), ${{parameters.myGetDeploy}}))
     jobs:
       - job:
+        pool:
+          vmImage: "windows-latest" # NuGetCommand@2 is no longer supported on Ubuntu 24.04 so we'll use windows until an alternative is available.
         displayName: Push to pre-release feed
         steps:
           - checkout: none
@@ -771,6 +795,8 @@ stages:
     condition: and(succeeded(), or(eq(dependencies.Build.outputs['A.build.NBGV_PublicRelease'], 'True'), ${{parameters.nuGetDeploy}}))
     jobs:
       - job:
+        pool:
+          vmImage: "windows-latest" # NuGetCommand@2 is no longer supported on Ubuntu 24.04 so we'll use windows until an alternative is available.
         displayName: Push to NuGet
         steps:
           - checkout: none

src/Umbraco.Cms.Api.Delivery/Caching/DeliveryApiOutputCachePolicy.cs

@@ -1,15 +1,20 @@
-using Microsoft.AspNetCore.OutputCaching;
+using Microsoft.AspNetCore.OutputCaching;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Primitives;
 using Umbraco.Cms.Core.DeliveryApi;
 
 namespace Umbraco.Cms.Api.Delivery.Caching;
 
 internal sealed class DeliveryApiOutputCachePolicy : IOutputCachePolicy
 {
     private readonly TimeSpan _duration;
+    private readonly StringValues _varyByHeaderNames;
 
-    public DeliveryApiOutputCachePolicy(TimeSpan duration)
-        => _duration = duration;
+    public DeliveryApiOutputCachePolicy(TimeSpan duration, StringValues varyByHeaderNames)
+    {
+        _duration = duration;
+        _varyByHeaderNames = varyByHeaderNames;
+    }
 
     ValueTask IOutputCachePolicy.CacheRequestAsync(OutputCacheContext context, CancellationToken cancellationToken)
     {
@@ -18,8 +23,14 @@ ValueTask IOutputCachePolicy.CacheRequestAsync(OutputCacheContext context, Cance
             .RequestServices
             .GetRequiredService<IRequestPreviewService>();
 
-        context.EnableOutputCaching = requestPreviewService.IsPreview() is false;
+        IApiAccessService apiAccessService = context
+            .HttpContext
+            .RequestServices
+            .GetRequiredService<IApiAccessService>();
+
+        context.EnableOutputCaching = requestPreviewService.IsPreview() is false && apiAccessService.HasPublicAccess();
         context.ResponseExpirationTimeSpan = _duration;
+        context.CacheVaryByRules.HeaderNames = _varyByHeaderNames;
 
         return ValueTask.CompletedTask;
     }

src/Umbraco.Cms.Api.Delivery/Controllers/Content/ByRouteContentApiController.cs

@@ -19,7 +19,7 @@ public class ByRouteContentApiController : ContentApiItemControllerBase
     private readonly IRequestRedirectService _requestRedirectService;
     private readonly IRequestPreviewService _requestPreviewService;
     private readonly IRequestMemberAccessService _requestMemberAccessService;
-    private const string PreviewContentRequestPathPrefix = $"/{Constants.DeliveryApi.Routing.PreviewContentPathPrefix}";
+    private const string PreviewContentRequestPathPrefix = $"/{Umbraco.Cms.Core.Constants.DeliveryApi.Routing.PreviewContentPathPrefix}";
 
     [Obsolete($"Please use the constructor that does not accept {nameof(IPublicAccessService)}. Will be removed in V14.")]
     public ByRouteContentApiController(
@@ -145,6 +145,11 @@ private async Task<IActionResult> HandleRequest(string path)
         path = DecodePath(path);
         path = path.Length == 0 ? "/" : path;
 
+        if (_apiContentPathResolver.IsResolvablePath(path) is false)
+        {
+            return NotFound();
+        }
+
         IPublishedContent? contentItem = GetContent(path);
         if (contentItem is not null)
         {

src/Umbraco.Cms.Api.Delivery/DependencyInjection/UmbracoBuilderExtensions.cs

@@ -5,6 +5,7 @@
 using Microsoft.AspNetCore.Routing;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Primitives;
 using Umbraco.Cms.Api.Common.DependencyInjection;
 using Umbraco.Cms.Api.Delivery.Accessors;
 using Umbraco.Cms.Api.Delivery.Caching;
@@ -108,12 +109,20 @@ private static IUmbracoBuilder AddOutputCache(this IUmbracoBuilder builder)
 
             if (outputCacheSettings.ContentDuration.TotalSeconds > 0)
             {
-                options.AddPolicy(Constants.DeliveryApi.OutputCache.ContentCachePolicy, new DeliveryApiOutputCachePolicy(outputCacheSettings.ContentDuration));
+                options.AddPolicy(
+                    Constants.DeliveryApi.OutputCache.ContentCachePolicy,
+                    new DeliveryApiOutputCachePolicy(
+                        outputCacheSettings.ContentDuration,
+                        new StringValues([Constants.DeliveryApi.HeaderNames.AcceptLanguage, Constants.DeliveryApi.HeaderNames.StartItem])));
             }
 
             if (outputCacheSettings.MediaDuration.TotalSeconds > 0)
             {
-                options.AddPolicy(Constants.DeliveryApi.OutputCache.MediaCachePolicy, new DeliveryApiOutputCachePolicy(outputCacheSettings.MediaDuration));
+                options.AddPolicy(
+                    Constants.DeliveryApi.OutputCache.MediaCachePolicy,
+                    new DeliveryApiOutputCachePolicy(
+                        outputCacheSettings.MediaDuration,
+                        Constants.DeliveryApi.HeaderNames.StartItem));
             }
         });
 

src/Umbraco.Cms.Api.Delivery/Filters/SwaggerContentDocumentationFilter.cs

@@ -1,4 +1,4 @@
-using Microsoft.OpenApi.Any;
+using Microsoft.OpenApi.Any;
 using Microsoft.OpenApi.Models;
 using Swashbuckle.AspNetCore.SwaggerGen;
 using Umbraco.Cms.Api.Delivery.Configuration;
@@ -21,7 +21,7 @@ protected override void ApplyOperation(OpenApiOperation operation, OperationFilt
 
         operation.Parameters.Add(new OpenApiParameter
         {
-            Name = "Accept-Language",
+            Name = Core.Constants.DeliveryApi.HeaderNames.AcceptLanguage,
             In = ParameterLocation.Header,
             Required = false,
             Description = "Defines the language to return. Use this when querying language variant content items.",
@@ -37,7 +37,7 @@ protected override void ApplyOperation(OpenApiOperation operation, OperationFilt
 
         operation.Parameters.Add(new OpenApiParameter
         {
-            Name = "Preview",
+            Name = Core.Constants.DeliveryApi.HeaderNames.Preview,
             In = ParameterLocation.Header,
             Required = false,
             Description = "Whether to request draft content.",
@@ -46,7 +46,7 @@ protected override void ApplyOperation(OpenApiOperation operation, OperationFilt
 
         operation.Parameters.Add(new OpenApiParameter
         {
-            Name = "Start-Item",
+            Name = Core.Constants.DeliveryApi.HeaderNames.StartItem,
             In = ParameterLocation.Header,
             Required = false,
             Description = "URL segment or GUID of a root content item.",

src/Umbraco.Cms.Api.Delivery/Filters/SwaggerDocumentationFilterBase.cs

@@ -1,4 +1,4 @@
-using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc;
 using Microsoft.OpenApi.Any;
 using Microsoft.OpenApi.Models;
 using Swashbuckle.AspNetCore.SwaggerGen;
@@ -63,7 +63,7 @@ protected void AddFields(OpenApiOperation operation, OperationFilterContext cont
     protected void AddApiKey(OpenApiOperation operation) =>
         operation.Parameters.Add(new OpenApiParameter
         {
-            Name = "Api-Key",
+            Name = Core.Constants.DeliveryApi.HeaderNames.ApiKey,
             In = ParameterLocation.Header,
             Required = false,
             Description = "API key specified through configuration to authorize access to the API.",

src/Umbraco.Cms.Api.Delivery/Handlers/InitializeMemberApplicationNotificationHandler.cs

@@ -6,6 +6,7 @@
 using Umbraco.Cms.Core.Events;
 using Umbraco.Cms.Core.Notifications;
 using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Core.Sync;
 using Umbraco.Cms.Infrastructure.Security;
 
 namespace Umbraco.Cms.Api.Delivery.Handlers;
@@ -16,16 +17,21 @@ internal sealed class InitializeMemberApplicationNotificationHandler : INotifica
     private readonly ILogger<InitializeMemberApplicationNotificationHandler> _logger;
     private readonly DeliveryApiSettings _deliveryApiSettings;
     private readonly IServiceScopeFactory _serviceScopeFactory;
+    private readonly IServerRoleAccessor _serverRoleAccessor;
+    private static readonly SemaphoreSlim _locker = new(1);
+    private static bool _isInitialized = false;
 
     public InitializeMemberApplicationNotificationHandler(
         IRuntimeState runtimeState,
         IOptions<DeliveryApiSettings> deliveryApiSettings,
         ILogger<InitializeMemberApplicationNotificationHandler> logger,
-        IServiceScopeFactory serviceScopeFactory)
+        IServiceScopeFactory serviceScopeFactory,
+        IServerRoleAccessor serverRoleAccessor)
     {
         _runtimeState = runtimeState;
         _logger = logger;
         _serviceScopeFactory = serviceScopeFactory;
+        _serverRoleAccessor = serverRoleAccessor;
         _deliveryApiSettings = deliveryApiSettings.Value;
     }
 
@@ -36,34 +42,55 @@ public async Task HandleAsync(UmbracoApplicationStartingNotification notificatio
             return;
         }
 
-        // we cannot inject the IMemberApplicationManager because it ultimately takes a dependency on the DbContext ... and during
-        // install that is not allowed (no connection string means no DbContext)
-        using IServiceScope scope = _serviceScopeFactory.CreateScope();
-        IMemberApplicationManager memberApplicationManager = scope.ServiceProvider.GetRequiredService<IMemberApplicationManager>();
-
-        if (_deliveryApiSettings.MemberAuthorization?.AuthorizationCodeFlow?.Enabled is not true)
+        if (_serverRoleAccessor.CurrentServerRole is ServerRole.Subscriber)
         {
-            await memberApplicationManager.DeleteMemberApplicationAsync(cancellationToken);
+            // subscriber instances should not alter the member application
             return;
         }
 
-        if (ValidateRedirectUrls(_deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LoginRedirectUrls) is false)
+        try
         {
-            await memberApplicationManager.DeleteMemberApplicationAsync(cancellationToken);
-            return;
-        }
+            await _locker.WaitAsync(cancellationToken);
+            if (_isInitialized)
+            {
+                return;
+            }
+
+            _isInitialized = true;
+
+            // we cannot inject the IMemberApplicationManager because it ultimately takes a dependency on the DbContext ... and during
+            // install that is not allowed (no connection string means no DbContext)
+            using IServiceScope scope = _serviceScopeFactory.CreateScope();
+            IMemberApplicationManager memberApplicationManager = scope.ServiceProvider.GetRequiredService<IMemberApplicationManager>();
+
+            if (_deliveryApiSettings.MemberAuthorization?.AuthorizationCodeFlow?.Enabled is not true)
+            {
+                await memberApplicationManager.DeleteMemberApplicationAsync(cancellationToken);
+                return;
+            }
 
-        if (_deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LogoutRedirectUrls.Any()
-            && ValidateRedirectUrls(_deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LogoutRedirectUrls) is false)
+            if (ValidateRedirectUrls(_deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LoginRedirectUrls) is false)
+            {
+                await memberApplicationManager.DeleteMemberApplicationAsync(cancellationToken);
+                return;
+            }
+
+            if (_deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LogoutRedirectUrls.Any()
+                && ValidateRedirectUrls(_deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LogoutRedirectUrls) is false)
+            {
+                await memberApplicationManager.DeleteMemberApplicationAsync(cancellationToken);
+                return;
+            }
+
+            await memberApplicationManager.EnsureMemberApplicationAsync(
+                _deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LoginRedirectUrls,
+                _deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LogoutRedirectUrls,
+                cancellationToken);
+        }
+        finally
         {
-            await memberApplicationManager.DeleteMemberApplicationAsync(cancellationToken);
-            return;
+            _locker.Release();
         }
-
-        await memberApplicationManager.EnsureMemberApplicationAsync(
-            _deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LoginRedirectUrls,
-            _deliveryApiSettings.MemberAuthorization.AuthorizationCodeFlow.LogoutRedirectUrls,
-            cancellationToken);
     }
 
     private bool ValidateRedirectUrls(Uri[] redirectUrls)

src/Umbraco.Cms.Api.Delivery/Services/ApiAccessService.cs

@@ -1,4 +1,4 @@
-using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Options;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.DeliveryApi;
@@ -29,7 +29,7 @@ public ApiAccessService(IHttpContextAccessor httpContextAccessor, IOptionsMonito
     private bool IfEnabled(Func<bool> condition) => _deliveryApiSettings.Enabled && condition();
 
     private bool HasValidApiKey() => _deliveryApiSettings.ApiKey.IsNullOrWhiteSpace() == false
-                                     && _deliveryApiSettings.ApiKey.Equals(GetHeaderValue("Api-Key"));
+                                     && _deliveryApiSettings.ApiKey.Equals(GetHeaderValue(Core.Constants.DeliveryApi.HeaderNames.ApiKey));
 
     private bool IfMediaEnabled(Func<bool> condition) => _deliveryApiSettings is { Enabled: true, Media.Enabled: true } && condition();
 }