fileio from petrel (#2182)

Author: LareinaM

mmpose/datasets/datasets/base/base_coco_style_dataset.py

@@ -7,8 +7,8 @@
 
 import numpy as np
 from mmengine.dataset import BaseDataset, force_full_init
-from mmengine.fileio import load
-from mmengine.utils import check_file_exist, is_list_of
+from mmengine.fileio import exists, get_local_path, load
+from mmengine.utils import is_list_of
 from xtcocotools.coco import COCO
 
 from mmpose.registry import DATASETS
@@ -195,18 +195,19 @@ def load_data_list(self) -> List[dict]:
     def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         """Load data from annotations in COCO format."""
 
-        check_file_exist(self.ann_file)
+        assert exists(self.ann_file), 'Annotation file does not exist'
 
-        coco = COCO(self.ann_file)
+        with get_local_path(self.ann_file) as local_path:
+            self.coco = COCO(local_path)
         # set the metainfo about categories, which is a list of dict
         # and each dict contains the 'id', 'name', etc. about this category
-        self._metainfo['CLASSES'] = coco.loadCats(coco.getCatIds())
+        self._metainfo['CLASSES'] = self.coco.loadCats(self.coco.getCatIds())
 
         instance_list = []
         image_list = []
 
-        for img_id in coco.getImgIds():
-            img = coco.loadImgs(img_id)[0]
+        for img_id in self.coco.getImgIds():
+            img = self.coco.loadImgs(img_id)[0]
             img.update({
                 'img_id':
                 img_id,
@@ -215,8 +216,8 @@ def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
             })
             image_list.append(img)
 
-            ann_ids = coco.getAnnIds(imgIds=img_id)
-            for ann in coco.loadAnns(ann_ids):
+            ann_ids = self.coco.getAnnIds(imgIds=img_id)
+            for ann in self.coco.loadAnns(ann_ids):
 
                 instance_info = self.parse_data_info(
                     dict(raw_ann_info=ann, raw_img_info=img))
@@ -380,18 +381,18 @@ def _get_bottomup_data_infos(self, instance_list: List[Dict],
     def _load_detection_results(self) -> List[dict]:
         """Load data from detection results with dummy keypoint annotations."""
 
-        check_file_exist(self.ann_file)
-        check_file_exist(self.bbox_file)
-
+        assert exists(self.ann_file), 'Annotation file does not exist'
+        assert exists(self.bbox_file), 'Bbox file does not exist'
         # load detection results
         det_results = load(self.bbox_file)
         assert is_list_of(det_results, dict)
 
         # load coco annotations to build image id-to-name index
-        coco = COCO(self.ann_file)
+        with get_local_path(self.ann_file) as local_path:
+            self.coco = COCO(local_path)
         # set the metainfo about categories, which is a list of dict
         # and each dict contains the 'id', 'name', etc. about this category
-        self._metainfo['CLASSES'] = coco.loadCats(coco.getCatIds())
+        self._metainfo['CLASSES'] = self.coco.loadCats(self.coco.getCatIds())
 
         num_keypoints = self.metainfo['num_keypoints']
         data_list = []
@@ -401,7 +402,7 @@ def _load_detection_results(self) -> List[dict]:
             if det['category_id'] != 1:
                 continue
 
-            img = coco.loadImgs(det['image_id'])[0]
+            img = self.coco.loadImgs(det['image_id'])[0]
 
             img_path = osp.join(self.data_prefix['img'], img['file_name'])
             bbox_xywh = np.array(

mmpose/datasets/datasets/body/mpii_dataset.py

@@ -4,7 +4,7 @@
 from typing import Callable, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
-from mmengine.utils import check_file_exist
+from mmengine.fileio import exists, get_local_path
 from scipy.io import loadmat
 
 from mmpose.registry import DATASETS
@@ -137,14 +137,16 @@ def __init__(self,
     def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         """Load data from annotations in MPII format."""
 
-        check_file_exist(self.ann_file)
-        with open(self.ann_file) as anno_file:
-            anns = json.load(anno_file)
+        assert exists(self.ann_file), 'Annotation file does not exist'
+        with get_local_path(self.ann_file) as local_path:
+            with open(local_path) as anno_file:
+                self.anns = json.load(anno_file)
 
         if self.headbox_file:
-            check_file_exist(self.headbox_file)
-            headbox_dict = loadmat(self.headbox_file)
-            headboxes_src = np.transpose(headbox_dict['headboxes_src'],
+            assert exists(self.headbox_file), 'Headbox file does not exist'
+            with get_local_path(self.headbox_file) as local_path:
+                self.headbox_dict = loadmat(local_path)
+            headboxes_src = np.transpose(self.headbox_dict['headboxes_src'],
                                          [2, 0, 1])
             SC_BIAS = 0.6
 
@@ -156,7 +158,7 @@ def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         # mpii bbox scales are normalized with factor 200.
         pixel_std = 200.
 
-        for idx, ann in enumerate(anns):
+        for idx, ann in enumerate(self.anns):
             center = np.array(ann['center'], dtype=np.float32)
             scale = np.array([ann['scale'], ann['scale']],
                              dtype=np.float32) * pixel_std

mmpose/datasets/datasets/body/mpii_trb_dataset.py

@@ -4,7 +4,7 @@
 from typing import List, Tuple
 
 import numpy as np
-from mmengine.utils import check_file_exist
+from mmengine.fileio import exists, get_local_path
 
 from mmpose.registry import DATASETS
 from mmpose.structures.bbox import bbox_cs2xyxy
@@ -106,11 +106,12 @@ class MpiiTrbDataset(BaseCocoStyleDataset):
     def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         """Load data from annotations in MPII-TRB format."""
 
-        check_file_exist(self.ann_file)
-        with open(self.ann_file) as anno_file:
-            data = json.load(anno_file)
+        assert exists(self.ann_file), 'Annotation file does not exist'
+        with get_local_path(self.ann_file) as local_path:
+            with open(local_path) as anno_file:
+                self.data = json.load(anno_file)
 
-        imgid2info = {img['id']: img for img in data['images']}
+        imgid2info = {img['id']: img for img in self.data['images']}
 
         instance_list = []
         image_list = []
@@ -119,7 +120,7 @@ def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         # mpii-trb bbox scales are normalized with factor 200.
         pixel_std = 200.
 
-        for ann in data['annotations']:
+        for ann in self.data['annotations']:
             img_id = ann['image_id']
 
             # center, scale in shape [1, 2] and bbox in [1, 4]

mmpose/datasets/datasets/body/posetrack18_video_dataset.py

@@ -3,8 +3,8 @@
 from typing import Callable, List, Optional, Sequence, Union
 
 import numpy as np
-from mmengine.fileio import load
-from mmengine.utils import check_file_exist, is_list_of
+from mmengine.fileio import exists, get_local_path, load
+from mmengine.utils import is_list_of
 from xtcocotools.coco import COCO
 
 from mmpose.registry import DATASETS
@@ -287,22 +287,22 @@ def parse_data_info(self, raw_data_info: dict) -> Optional[dict]:
 
     def _load_detection_results(self) -> List[dict]:
         """Load data from detection results with dummy keypoint annotations."""
-
-        check_file_exist(self.ann_file)
-        check_file_exist(self.bbox_file)
+        assert exists(self.ann_file), 'Annotation file does not exist'
+        assert exists(self.bbox_file), 'Bbox file does not exist'
 
         # load detection results
         det_results = load(self.bbox_file)
         assert is_list_of(det_results, dict)
 
         # load coco annotations to build image id-to-name index
-        coco = COCO(self.ann_file)
+        with get_local_path(self.ann_file) as local_path:
+            self.coco = COCO(local_path)
 
         # mapping image name to id
         name2id = {}
         # mapping image id to name
         id2name = {}
-        for img_id, image in coco.imgs.items():
+        for img_id, image in self.coco.imgs.items():
             file_name = image['file_name']
             id2name[img_id] = file_name
             name2id[file_name] = img_id
@@ -333,7 +333,7 @@ def _load_detection_results(self) -> List[dict]:
                     img_id = name2id[det['image_name']]
                 else:
                     img_id = det['image_id']
-                img_ann = coco.loadImgs(img_id)[0]
+                img_ann = self.coco.loadImgs(img_id)[0]
                 nframes = int(img_ann['nframes'])
 
             # deal with multiple image paths

mmpose/datasets/datasets/hand/coco_wholebody_hand_dataset.py

@@ -3,7 +3,7 @@
 from typing import List, Tuple
 
 import numpy as np
-from mmengine.utils import check_file_exist
+from mmengine.fileio import exists, get_local_path
 from xtcocotools.coco import COCO
 
 from mmpose.registry import DATASETS
@@ -87,15 +87,16 @@ class CocoWholeBodyHandDataset(BaseCocoStyleDataset):
     def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
         """Load data from annotations in COCO format."""
 
-        check_file_exist(self.ann_file)
+        assert exists(self.ann_file), 'Annotation file does not exist'
 
-        coco = COCO(self.ann_file)
+        with get_local_path(self.ann_file) as local_path:
+            self.coco = COCO(local_path)
         instance_list = []
         image_list = []
         id = 0
 
-        for img_id in coco.getImgIds():
-            img = coco.loadImgs(img_id)[0]
+        for img_id in self.coco.getImgIds():
+            img = self.coco.loadImgs(img_id)[0]
 
             img.update({
                 'img_id':
@@ -105,8 +106,8 @@ def _load_annotations(self) -> Tuple[List[dict], List[dict]]:
             })
             image_list.append(img)
 
-            ann_ids = coco.getAnnIds(imgIds=img_id, iscrowd=False)
-            anns = coco.loadAnns(ann_ids)
+            ann_ids = self.coco.getAnnIds(imgIds=img_id, iscrowd=False)
+            anns = self.coco.loadAnns(ann_ids)
             for ann in anns:
                 for type in ['left', 'right']:
                     # filter invalid hand annotations, there might be two

mmpose/evaluation/metrics/coco_metric.py

@@ -7,7 +7,7 @@
 
 import numpy as np
 from mmengine.evaluator import BaseMetric
-from mmengine.fileio import dump, load
+from mmengine.fileio import dump, get_local_path, load
 from mmengine.logging import MMLogger
 from xtcocotools.coco import COCO
 from xtcocotools.cocoeval import COCOeval
@@ -102,7 +102,8 @@ def __init__(self,
         # initialize coco helper with the annotation json file
         # if ann_file is not specified, initialize with the converted dataset
         if ann_file is not None:
-            self.coco = COCO(ann_file)
+            with get_local_path(ann_file) as local_path:
+                self.coco = COCO(local_path)
         else:
             self.coco = None