diff --git a/scannet/load_scannet_data.py b/scannet/load_scannet_data.py
index 873257bb0..dd441a8e6 100644
--- a/scannet/load_scannet_data.py
+++ b/scannet/load_scannet_data.py
@@ -38,7 +38,7 @@ def read_aggregation(filename):
             if label in label_to_segs:
                 label_to_segs[label].extend(segs)
             else:
-                label_to_segs[label] = segs
+                label_to_segs[label] = segs.copy()
     return object_id_to_segs, label_to_segs