Merge pull request #449 from WikipediaLibrary/kgraessle/refactor-crons-and-close-db-connections

Match try/except block structure that we're using for linkevents_archive

Author: suecarmol

extlinks/aggregates/management/helpers/aggregate_archive_command.py

@@ -11,7 +11,7 @@
 from django.core import serializers
 from django.core.management import call_command
 from django.core.management.base import BaseCommand, CommandError, CommandParser
-from django.db import models
+from django.db import models, close_old_connections
 
 from extlinks.common import swift
 
@@ -153,6 +153,8 @@ def handle(self, *args, **options):
         elif subcommand == "upload":
             self.upload(container=options["container"], filenames=options["filenames"])
 
+        close_old_connections()
+
     def dump(
         self,
         start: Optional[datetime.date] = None,
@@ -202,7 +204,7 @@ def dump(
                 end = (datetime.date.today() - relativedelta(years=1)).replace(day=1)
 
         if not container:
-            container = os.environ.get("SWIFT_CONTAINER_AGGREGATES")
+            container = os.environ.get("SWIFT_CONTAINER_AGGREGATES", "archive-aggregates")
 
         if end:
             cursor = start
@@ -286,25 +288,32 @@ def upload(self, container: str, filenames: List[str]):
             return False
 
         try:
-            was_created = swift.ensure_container_exists(conn, container)
-            if was_created:
-                self.log_msg(f"Created new container: {container}")
-        except RuntimeError as e:
-            self.log_msg(str(e), level="error")
-            return False
+            # Ensure the container exists before uploading.
+            try:
+                was_created = swift.ensure_container_exists(conn, container)
+                if was_created:
+                    self.log_msg(f"Created new container: {container}")
+            except RuntimeError as e:
+                self.log_msg(str(e), level="error")
+                return False
 
-        successful, failed = swift.batch_upload_files(conn, container, filenames)
+            successful, failed = swift.batch_upload_files(conn, container, filenames)
 
-        self.log_msg(
-            "Uploaded %d/%d archives to object storage",
-            len(successful),
-            len(filenames),
-        )
+            self.log_msg(
+                "Uploaded %d/%d archives to object storage",
+                len(successful),
+                len(filenames),
+            )
 
-        if len(failed) > 0:
-            raise CommandError(
-                f"The following {failed} archives failed to upload: {','.join(failed)}"
+            if len(failed) > 0:
+                raise CommandError(
+                    f"The following {failed} archives failed to upload: {','.join(failed)}"
+                )
+        except Exception as e:
+            self.log_msg(
+                f"Failed to upload to Swift: {e}", level="error"
             )
+            return False
 
     def archive(
         self,