Distributed Background Jobs: Catch exceptions in job loop to improve application resilience (#21099)

* Set exit code when exception is thrown

* Catch any error instead so we don't stop the application

* Handle exception when ensuring jobs

Author: nikolajlauridsen

src/Umbraco.Infrastructure/BackgroundJobs/DistributedBackgroundJobHostedService.cs

@@ -47,16 +47,38 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             await Task.Delay(_distributedJobSettings.Delay, stoppingToken);
         }
 
-        // Update all jobs, periods might have changed when restarting.
-        await _distributedJobService.EnsureJobsAsync();
+        try
+        {
+            // Update all jobs, periods might have changed when restarting.
+            await _distributedJobService.EnsureJobsAsync();
+        }
+        catch (Exception exception)
+        {
+            // We swallow exception here, don't want the app to crash if something goes wrong
+            _logger.LogError(exception, "An exception occurred while attempting to ensure distributed background jobs on startup.");
+        }
+
 
         using PeriodicTimer timer = new(_distributedJobSettings.Period);
 
         try
         {
             while (await timer.WaitForNextTickAsync(stoppingToken))
             {
-                await RunRunnableJob();
+                try
+                {
+                    await RunRunnableJob();
+                }
+                catch (Exception exception)
+                {
+                    if (exception is OperationCanceledException)
+                    {
+                        // If the operation was canceled, just re-throw to stop the service
+                        throw;
+                    }
+
+                    _logger.LogError(exception, "An exception occurred while attempting to run a distributed background job.");
+                }
             }
         }
         catch (OperationCanceledException)