delete source widgets for current sync update

securedrop_client/gui/widgets.py

@@ -586,7 +586,7 @@ def show_sources(self, sources: List[Source]):
             self.empty_conversation_view.show_no_sources_message()
             self.empty_conversation_view.show()
 
-        if self.source_list.source_widgets:
+        if self.source_list.source_items:
             # The source list already contains sources.
             deleted_sources = self.source_list.update(sources)
             for source_uuid in deleted_sources:
@@ -747,8 +747,8 @@ def __lt__(self, other):
         me = lw.itemWidget(self)
         them = lw.itemWidget(other)
         if me and them:
-            my_ts = arrow.get(me.source.last_updated)
-            other_ts = arrow.get(them.source.last_updated)
+            my_ts = arrow.get(me.last_updated)
+            other_ts = arrow.get(them.last_updated)
             return my_ts < other_ts
         return True
 
@@ -758,6 +758,8 @@ class SourceList(QListWidget):
     Displays the list of sources.
     """
 
+    NUM_SOURCES_TO_ADD_AT_A_TIME = 32
+
     def __init__(self):
         super().__init__()
 
@@ -772,8 +774,8 @@ def __init__(self):
         # Enable ordering.
         self.setSortingEnabled(True)
 
-        # To hold references to SourceWidget instances indexed by source UUID.
-        self.source_widgets = {}
+        # To hold references to SourceListWidgetItem instances indexed by source UUID.
+        self.source_items = {}
 
     def setup(self, controller):
         self.controller = controller
@@ -789,52 +791,57 @@ def update(self, sources: List[Source]) -> List[str]:
         """
         Update the list with the passed in list of sources.
         """
-        # Delete widgets that no longer exist in source list
-        source_uuids = [source.uuid for source in sources]
+        sources_to_update = []
+        sources_to_add = {}
+        for source in sources:
+            try:
+                if source.uuid in self.source_items:
+                    sources_to_update.append(source.uuid)
+                else:
+                    sources_to_add[source.uuid] = source
+            except sqlalchemy.exc.InvalidRequestError as e:
+                logger.debug(e)
+                continue
+
+        # Delete widgets for sources not in the supplied sourcelist
         deleted_uuids = []
-        for i in range(self.count()):
-            list_item = self.item(i)
-            list_widget = self.itemWidget(list_item)
+        sources_to_delete = [self.source_items[uuid] for uuid in self.source_items
+                             if uuid not in sources_to_update]
+        for source_item in sources_to_delete:
+            if source_item.isSelected():
+                self.setCurrentItem(None)
 
-            if list_widget and list_widget.source_uuid not in source_uuids:
-                if list_item.isSelected():
-                    self.setCurrentItem(None)
+            source_widget = self.itemWidget(source_item)
+            self.takeItem(self.row(source_item))
+            if source_widget.source_uuid in self.source_items:
+                del self.source_items[source_widget.source_uuid]
 
-                try:
-                    del self.source_widgets[list_widget.source_uuid]
-                except KeyError:
-                    pass
+            deleted_uuids.append(source_widget.source_uuid)
+            source_widget.deleteLater()
 
-                self.takeItem(i)
-                deleted_uuids.append(list_widget.source_uuid)
-                list_widget.deleteLater()
+        # Update the remaining widgets
+        for i in range(self.count()):
+            source_widget = self.itemWidget(self.item(i))
 
-        # Create new widgets for new sources
-        widget_uuids = [self.itemWidget(self.item(i)).source_uuid for i in range(self.count())]
-        for source in sources:
-            if source.uuid in widget_uuids:
-                try:
-                    self.source_widgets[source.uuid].update()
-                except sqlalchemy.exc.InvalidRequestError as e:
-                    logger.error(
-                        "Could not update SourceWidget for source %s; deleting it. Error was: %s",
-                        source.uuid,
-                        e
-                    )
-                    deleted_uuids.append(source.uuid)
-                    self.source_widgets[source.uuid].deleteLater()
-                    del self.source_widgets[list_widget.source_uuid]
-            else:
-                new_source = SourceWidget(self.controller, source)
-                self.source_widgets[source.uuid] = new_source
+            if not source_widget:
+                continue
 
-                list_item = SourceListWidgetItem()
-                self.insertItem(0, list_item)
-                list_item.setSizeHint(new_source.sizeHint())
-                self.setItemWidget(list_item, new_source)
+            source_widget.update()
 
-        # Sort..!
+        # Add widgets for new sources
+        for uuid in sources_to_add:
+            source_widget = SourceWidget(self.controller, sources_to_add[uuid])
+            source_item = SourceListWidgetItem(self)
+            source_item.setSizeHint(source_widget.sizeHint())
+            self.insertItem(0, source_item)
+            self.setItemWidget(source_item, source_widget)
+            self.source_items[uuid] = source_item
+
+        # Re-sort SourceList to make sure the most recently-updated sources appear at the top
         self.sortItems(Qt.DescendingOrder)
+
+        # Return uuids of source widgets that were deleted so we can later delete the corresponding
+        # conversation widgets
         return deleted_uuids
 
     def initial_update(self, sources: List[Source]):
@@ -851,22 +858,29 @@ def add_source(self, sources, slice_size=1):
 
         def schedule_source_management(slice_size=slice_size):
             if not sources:
-                # Nothing more to do.
                 return
+
             # Process the remaining "slice_size" number of sources.
             sources_slice = sources[:slice_size]
             for source in sources_slice:
-                new_source = SourceWidget(self.controller, source)
-                self.source_widgets[source.uuid] = new_source
-                list_item = SourceListWidgetItem(self)
-                list_item.setSizeHint(new_source.sizeHint())
+                try:
+                    source_uuid = source.uuid
+                    source_widget = SourceWidget(self.controller, source)
+                    source_item = SourceListWidgetItem(self)
+                    source_item.setSizeHint(source_widget.sizeHint())
+                    self.insertItem(0, source_item)
+                    self.setItemWidget(source_item, source_widget)
+                    self.source_items[source_uuid] = source_item
+                except sqlalchemy.exc.InvalidRequestError as e:
+                    logger.debug(e)
+
+            # Re-sort SourceList to make sure the most recently-updated sources appear at the top
+            self.sortItems(Qt.DescendingOrder)
 
-                self.insertItem(0, list_item)
-                self.setItemWidget(list_item, new_source)
             # ATTENTION! 32 is an arbitrary number arrived at via
             # experimentation. It adds plenty of sources, but doesn't block
             # for a noticable amount of time.
-            new_slice_size = min(32, slice_size * 2)
+            new_slice_size = min(self.NUM_SOURCES_TO_ADD_AT_A_TIME, slice_size * 2)
             # Call add_source again for the remaining sources.
             self.add_source(sources[slice_size:], new_slice_size)
 
@@ -888,8 +902,8 @@ def get_source_widget(self, source_uuid: str) -> Optional[QListWidget]:
         First try to get the source widget from the cache, then look for it in the SourceList.
         '''
         try:
-            source_widget = self.source_widgets[source_uuid]
-            return source_widget
+            source_item = self.source_items[source_uuid]
+            return self.itemWidget(source_item)
         except KeyError:
             pass
 
@@ -954,6 +968,7 @@ def __init__(self, controller: Controller, source: Source):
 
         # Store source
         self.source_uuid = source.uuid
+        self.last_updated = source.last_updated
         self.source = source
 
         # Set layout
@@ -1039,6 +1054,7 @@ def update(self):
         """
         try:
             self.controller.session.refresh(self.source)
+            self.last_updated = self.source.last_updated
             self.timestamp.setText(_(arrow.get(self.source.last_updated).format('DD MMM')))
             self.name.setText(self.source.journalist_designation)
 
@@ -1052,8 +1068,7 @@ def update(self):
                 self.paperclip.hide()
             self.star.update(self.source.is_starred)
         except sqlalchemy.exc.InvalidRequestError as e:
-            logger.error(f"Could not update SourceWidget for source {self.source_uuid}: {e}")
-            raise
+            logger.debug(f"Could not update SourceWidget for source {self.source_uuid}: {e}")
 
     def set_snippet(self, source_uuid: str, content: str):
         """

tests/functional/test_delete_source.py

@@ -21,13 +21,14 @@ def test_delete_source_and_their_docs(functional_test_logged_in_context, qtbot,
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     assert len(source_ids) == 2
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
     qtbot.wait(TIME_RENDER_CONV_VIEW)
 

tests/functional/test_download_file.py

@@ -24,12 +24,13 @@ def test_download_file(functional_test_logged_in_context, qtbot, mocker):
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
 
     qtbot.wait(TIME_SYNC)

tests/functional/test_export_dialog.py

@@ -24,12 +24,13 @@ def test_export_dialog(functional_test_logged_in_context, qtbot, mocker):
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
 
     qtbot.wait(TIME_SYNC)

tests/functional/test_offline_delete_source.py

@@ -22,13 +22,14 @@ def test_offline_delete_source_and_their_docs(functional_test_logged_in_context,
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     assert len(source_ids) == 2
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
     qtbot.wait(TIME_CLICK_ACTION)
 

tests/functional/test_offline_read_conversations.py

@@ -22,12 +22,13 @@ def test_offline_read_conversations(functional_test_logged_in_context, qtbot, mo
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
 
     # Otherwise our test is running too fast to create all files/directories
@@ -44,7 +45,8 @@ def check_login_button():
     # Ensure that clicking on a source shows a conversation that contains
     # activity.
     second_source_id = source_ids[1]
-    second_source_widget = gui.main_view.source_list.source_widgets[second_source_id]
+    second_source_item = gui.main_view.source_list.source_items[second_source_id]
+    second_source_widget = gui.main_view.source_list.itemWidget(second_source_item)
     qtbot.mouseClick(second_source_widget, Qt.LeftButton)
     conversation = gui.main_view.view_layout.itemAt(0).widget()
     assert len(list(conversation.conversation_view.current_messages.keys())) > 0

tests/functional/test_offline_send_reply.py

@@ -22,12 +22,13 @@ def test_offline_send_reply_to_source(functional_test_logged_in_context, qtbot,
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
 
     # Now logout.

tests/functional/test_offline_star_source.py

@@ -23,12 +23,13 @@ def test_offline_star_source(functional_test_logged_in_context, qtbot):
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
 
     # Now logout.

tests/functional/test_receive_message.py

@@ -24,12 +24,13 @@ def test_receive_message_from_source(functional_test_logged_in_context, qtbot, m
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
 
     qtbot.wait(TIME_SYNC)

tests/functional/test_send_reply.py

@@ -22,12 +22,13 @@ def test_send_reply_to_source(functional_test_logged_in_context, qtbot, mocker):
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
     # Type something into the reply box and click the send button.
     message = "Hello, world!"

tests/functional/test_star_source.py

@@ -22,12 +22,13 @@ def test_star_source(functional_test_logged_in_context, qtbot, mocker):
     qtbot.wait(TIME_APP_START)
 
     def check_for_sources():
-        assert len(list(gui.main_view.source_list.source_widgets.keys()))
+        assert len(list(gui.main_view.source_list.source_items.keys()))
 
     qtbot.waitUntil(check_for_sources, timeout=TIME_RENDER_SOURCE_LIST)
-    source_ids = list(gui.main_view.source_list.source_widgets.keys())
+    source_ids = list(gui.main_view.source_list.source_items.keys())
     first_source_id = source_ids[0]
-    first_source_widget = gui.main_view.source_list.source_widgets[first_source_id]
+    first_source_item = gui.main_view.source_list.source_items[first_source_id]
+    first_source_widget = gui.main_view.source_list.itemWidget(first_source_item)
     qtbot.mouseClick(first_source_widget, Qt.LeftButton)
 
     # Check the source isn't checked.