Add loss, save_inference_model and logger

examples/bert/bert_classifier.py

@@ -18,7 +18,8 @@
 from hapi.configure import Config
 from hapi.text.bert import BertEncoder
 from paddle.fluid.dygraph import Linear, Layer
-from hapi.model import set_device, Model, SoftmaxWithCrossEntropy, Input
+from hapi.model import set_device, Model, Input
+from hapi.loss import SoftmaxWithCrossEntropy
 import hapi.text.tokenizer.tokenization as tokenization
 from hapi.text.bert import Optimizer, BertConfig, BertDataLoader, BertInputExample
 

examples/bert_leveldb/bert_classifier.py

@@ -18,7 +18,8 @@
 from hapi.configure import Config
 from hapi.text.bert import BertEncoder
 from paddle.fluid.dygraph import Linear, Layer
-from hapi.model import set_device, Model, SoftmaxWithCrossEntropy, Input
+from hapi.model import set_device, Model, Input
+from hapi.loss import SoftmaxWithCrossEntropy
 import hapi.text.tokenizer.tokenization as tokenization
 from hapi.text.bert import Optimizer, BertConfig, BertDataLoader, BertInputExample
 

examples/bmn/modeling.py

@@ -17,8 +17,9 @@
 import numpy as np
 import math
 
-from hapi.model import Model, Loss
-from hapi.download import get_weights_path
+from hapi.model import Model
+from hapi.loss import Loss
+from hapi.download import get_weights_path_from_url
 
 __all__ = ["BMN", "BmnLoss", "bmn"]
 
@@ -459,7 +460,7 @@ def bmn(tscale,
     model = BMN(tscale, dscale, prop_boundary_ratio, num_sample,
                 num_sample_perbin)
     if pretrained:
-        weight_path = get_weights_path(*(pretrain_infos['bmn']))
+        weight_path = get_weights_path_from_url(*(pretrain_infos['bmn']))
         assert weight_path.endswith('.pdparams'), \
                 "suffix of weight must be .pdparams"
         model.load(weight_path)

examples/cyclegan/cyclegan.py

@@ -19,7 +19,8 @@
 import numpy as np
 
 import paddle.fluid as fluid
-from hapi.model import Model, Loss
+from hapi.model import Model
+from hapi.loss import Loss
 
 from layers import ConvBN, DeConvBN
 

examples/image_classification/imagenet_dataset.py

@@ -50,7 +50,7 @@ def __init__(self,
     def __getitem__(self, idx):
         img_path, label = self.samples[idx]
         img = cv2.imread(img_path).astype(np.float32)
-        label = np.array([label])
+        label = np.array([label]).astype(np.int64)
         return self.transform(img), label
 
     def __len__(self):

examples/image_classification/main.py

@@ -27,7 +27,8 @@
 from paddle.fluid.dygraph.parallel import ParallelEnv
 from paddle.io import BatchSampler, DataLoader
 
-from hapi.model import CrossEntropy, Input, set_device
+from hapi.model import Input, set_device
+from hapi.loss import CrossEntropy
 from hapi.distributed import DistributedBatchSampler
 from hapi.metrics import Accuracy
 import hapi.vision.models as models

examples/ocr/seq2seq_attn.py

@@ -20,7 +20,8 @@
 from paddle.fluid.layers import BeamSearchDecoder
 
 from hapi.text import RNNCell, RNN, DynamicDecode
-from hapi.model import Model, Loss
+from hapi.model import Model
+from hapi.loss import Loss
 
 
 class ConvBNPool(fluid.dygraph.Layer):

examples/sequence_tagging/train.py

@@ -29,7 +29,8 @@
 sys.path.append(os.path.join(work_dir, "../"))
 
 from hapi.metrics import Metric
-from hapi.model import Model, Input, Loss, set_device
+from hapi.model import Model, Input, set_device
+from hapi.loss import Loss
 from hapi.text.text import SequenceTagging
 
 from utils.check import check_gpu, check_version

examples/tsm/main.py

@@ -22,7 +22,8 @@
 from paddle import fluid
 from paddle.fluid.dygraph.parallel import ParallelEnv
 
-from hapi.model import Model, CrossEntropy, Input, set_device
+from hapi.model import Model, Input, set_device
+from hapi.loss import CrossEntropy
 from hapi.metrics import Accuracy
 
 from modeling import tsm_resnet50
@@ -33,11 +34,10 @@
 
 def make_optimizer(step_per_epoch, parameter_list=None):
     boundaries = [e * step_per_epoch for e in [40, 60]]
-    values = [FLAGS.lr * (0.1 ** i) for i in range(len(boundaries) + 1)]
+    values = [FLAGS.lr * (0.1**i) for i in range(len(boundaries) + 1)]
 
     learning_rate = fluid.layers.piecewise_decay(
-        boundaries=boundaries,
-        values=values)
+        boundaries=boundaries, values=values)
     optimizer = fluid.optimizer.Momentum(
         learning_rate=learning_rate,
         regularization=fluid.regularizer.L2Decay(1e-4),
@@ -51,29 +51,27 @@ def main():
     device = set_device(FLAGS.device)
     fluid.enable_dygraph(device) if FLAGS.dynamic else None
 
-    train_transform = Compose([GroupScale(),
-                               GroupMultiScaleCrop(),
-                               GroupRandomCrop(),
-                               GroupRandomFlip(),
-                               NormalizeImage()])
+    train_transform = Compose([
+        GroupScale(), GroupMultiScaleCrop(), GroupRandomCrop(),
+        GroupRandomFlip(), NormalizeImage()
+    ])
     train_dataset = KineticsDataset(
-            file_list=os.path.join(FLAGS.data, 'train_10.list'),
-            pickle_dir=os.path.join(FLAGS.data, 'train_10'),
-            label_list=os.path.join(FLAGS.data, 'label_list'),
-            transform=train_transform)
-    val_transform = Compose([GroupScale(),
-                             GroupCenterCrop(),
-                             NormalizeImage()])
+        file_list=os.path.join(FLAGS.data, 'train_10.list'),
+        pickle_dir=os.path.join(FLAGS.data, 'train_10'),
+        label_list=os.path.join(FLAGS.data, 'label_list'),
+        transform=train_transform)
+    val_transform = Compose(
+        [GroupScale(), GroupCenterCrop(), NormalizeImage()])
     val_dataset = KineticsDataset(
-            file_list=os.path.join(FLAGS.data, 'val_10.list'),
-            pickle_dir=os.path.join(FLAGS.data, 'val_10'),
-            label_list=os.path.join(FLAGS.data, 'label_list'),
-            mode='val',
-            transform=val_transform)
+        file_list=os.path.join(FLAGS.data, 'val_10.list'),
+        pickle_dir=os.path.join(FLAGS.data, 'val_10'),
+        label_list=os.path.join(FLAGS.data, 'label_list'),
+        mode='val',
+        transform=val_transform)
 
     pretrained = FLAGS.eval_only and FLAGS.weights is None
-    model = tsm_resnet50(num_classes=train_dataset.num_classes,
-                         pretrained=pretrained)
+    model = tsm_resnet50(
+        num_classes=train_dataset.num_classes, pretrained=pretrained)
 
     step_per_epoch = int(len(train_dataset) / FLAGS.batch_size \
                          / ParallelEnv().nranks)
@@ -116,7 +114,9 @@ def main():
 if __name__ == '__main__':
     parser = argparse.ArgumentParser("CNN training on TSM")
     parser.add_argument(
-        "--data", type=str, default='dataset/kinetics',
+        "--data",
+        type=str,
+        default='dataset/kinetics',
         help="path to dataset root directory")
     parser.add_argument(
         "--device", type=str, default='gpu', help="device to use, gpu or cpu")

examples/tsm/modeling.py

@@ -18,7 +18,7 @@
 from paddle.fluid.dygraph.nn import Conv2D, Pool2D, BatchNorm, Linear
 
 from hapi.model import Model
-from hapi.download import get_weights_path
+from hapi.download import get_weights_path_from_url
 
 __all__ = ["TSM_ResNet", "tsm_resnet50"]
 
@@ -122,6 +122,7 @@ class TSM_ResNet(Model):
         seg_num (int): segment number of each video sample. Default 8.
         num_classes (int): video class number. Default 400.
     """
+
     def __init__(self, num_layers=50, seg_num=8, num_classes=400):
         super(TSM_ResNet, self).__init__()
 
@@ -136,7 +137,11 @@ def __init__(self, num_layers=50, seg_num=8, num_classes=400):
         num_filters = [64, 128, 256, 512]
 
         self.conv = ConvBNLayer(
-            num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu')
+            num_channels=3,
+            num_filters=64,
+            filter_size=7,
+            stride=2,
+            act='relu')
         self.pool2d_max = Pool2D(
             pool_size=3, pool_stride=2, pool_padding=1, pool_type='max')
 
@@ -193,7 +198,7 @@ def _tsm_resnet(num_layers, seg_num=8, num_classes=400, pretrained=True):
         assert num_layers in pretrain_infos.keys(), \
                 "TSM-ResNet{} do not have pretrained weights now, " \
                 "pretrained should be set as False".format(num_layers)
-        weight_path = get_weights_path(*(pretrain_infos[num_layers]))
+        weight_path = get_weights_path_from_url(*(pretrain_infos[num_layers]))
         assert weight_path.endswith('.pdparams'), \
                 "suffix of weight must be .pdparams"
         model.load(weight_path)

examples/yolov3/modeling.py

@@ -20,8 +20,9 @@
 from paddle.fluid.param_attr import ParamAttr
 from paddle.fluid.regularizer import L2Decay
 
-from hapi.model import Model, Loss
-from hapi.download import get_weights_path
+from hapi.model import Model
+from hapi.loss import Loss
+from hapi.download import get_weights_path_from_url
 from hapi.vision.models import darknet53
 
 __all__ = ['YoloLoss', 'YOLOv3', 'yolov3_darknet53']
@@ -315,7 +316,7 @@ def _yolov3_darknet(num_layers=53,
         assert num_layers in pretrain_infos.keys(), \
                 "YOLOv3-DarkNet{} do not have pretrained weights now, " \
                 "pretrained should be set as False".format(num_layers)
-        weight_path = get_weights_path(*(pretrain_infos[num_layers]))
+        weight_path = get_weights_path_from_url(*(pretrain_infos[num_layers]))
         assert weight_path.endswith('.pdparams'), \
                 "suffix of weight must be .pdparams"
         model.load(weight_path)

hapi/__init__.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from hapi import logger
 from hapi.configure import Config
 from hapi import callbacks
 from hapi import datasets
@@ -22,16 +22,11 @@
 from hapi import progressbar
 from hapi import text
 from hapi import vision
+from hapi import loss
+
+logger.setup_logger()
 
 __all__ = [
-    'Config',
-    'callbacks',
-    'datasets',
-    'distributed',
-    'download',
-    'metrics',
-    'model',
-    'progressbar',
-    'text',
-    'vision',
+    'Config', 'callbacks', 'datasets', 'distributed', 'download', 'metrics',
+    'model', 'progressbar', 'text', 'vision', 'loss'
 ]

hapi/datasets/folder.py

@@ -150,7 +150,7 @@ def __getitem__(self, index):
         path, target = self.samples[index]
         sample = self.loader(path)
         if self.transform is not None:
-            sample, target = self.transform(sample)
+            sample = self.transform(sample)
 
         return sample, target
 

hapi/download.py

@@ -29,7 +29,7 @@
 import logging
 logger = logging.getLogger(__name__)
 
-__all__ = ['get_weights_path', 'is_url']
+__all__ = ['get_weights_path_from_url', 'is_url']
 
 WEIGHTS_HOME = osp.expanduser("~/.cache/paddle/hapi/weights")
 
@@ -45,48 +45,56 @@ def is_url(path):
     return path.startswith('http://') or path.startswith('https://')
 
 
-def get_weights_path(url, md5sum=None):
+def get_weights_path_from_url(url, md5sum=None):
     """Get weights path from WEIGHT_HOME, if not exists,
     download it from url.
+
+    Args:
+        url (str): download url
+        md5sum (str): md5 sum of download package
+
+    Returns:
+        str: a local path to save downloaded weights.
     """
-    path, _ = get_path(url, WEIGHTS_HOME, md5sum)
+    path = get_path_from_url(url, WEIGHTS_HOME, md5sum)
     return path
 
 
-def map_path(url, root_dir):
+def _map_path(url, root_dir):
     # parse path after download under root_dir
     fname = osp.split(url)[-1]
     fpath = fname
     return osp.join(root_dir, fpath)
 
 
-def get_path(url, root_dir, md5sum=None, check_exist=True):
+def get_path_from_url(url, root_dir, md5sum=None, check_exist=True):
     """ Download from given url to root_dir.
     if file or directory specified by url is exists under
     root_dir, return the path directly, otherwise download
     from url and decompress it, return the path.
 
-    url (str): download url
-    root_dir (str): root dir for downloading, it should be
-                    WEIGHTS_HOME or DATASET_HOME
-    md5sum (str): md5 sum of download package
+    Args:
+        url (str): download url
+        root_dir (str): root dir for downloading, it should be
+                        WEIGHTS_HOME or DATASET_HOME
+        md5sum (str): md5 sum of download package
+
+    Returns:
+        str: a local path to save downloaded models & weights & datasets.
     """
     assert is_url(url), "downloading from {} not a url".format(url)
     # parse path after download to decompress under root_dir
-    fullpath = map_path(url, root_dir)
+    fullpath = _map_path(url, root_dir)
 
-    exist_flag = False
     if osp.exists(fullpath) and check_exist and _md5check(fullpath, md5sum):
-        exist_flag = True
-        if ParallelEnv().local_rank == 0:
-            logger.info("Found {}".format(fullpath))
+        logger.info("Found {}".format(fullpath))
     else:
         if ParallelEnv().local_rank == 0:
             fullpath = _download(url, root_dir, md5sum)
         else:
             while not os.path.exists(fullpath):
                 time.sleep(1)
-    return fullpath, exist_flag
+    return fullpath
 
 
 def _download(url, path, md5sum=None):
@@ -109,8 +117,8 @@ def _download(url, path, md5sum=None):
         else:
             raise RuntimeError("Download from {} failed. "
                                "Retry limit reached".format(url))
-        if ParallelEnv().local_rank == 0:
-            logger.info("Downloading {} from {}".format(fname, url))
+
+        logger.info("Downloading {} from {}".format(fname, url))
 
         req = requests.get(url, stream=True)
         if req.status_code != 200:
@@ -141,17 +149,16 @@ def _download(url, path, md5sum=None):
 def _md5check(fullname, md5sum=None):
     if md5sum is None:
         return True
-    if ParallelEnv().local_rank == 0:
-        logger.info("File {} md5 checking...".format(fullname))
+
+    logger.info("File {} md5 checking...".format(fullname))
     md5 = hashlib.md5()
     with open(fullname, 'rb') as f:
         for chunk in iter(lambda: f.read(4096), b""):
             md5.update(chunk)
     calc_md5sum = md5.hexdigest()
 
     if calc_md5sum != md5sum:
-        if ParallelEnv().local_rank == 0:
-            logger.info("File {} md5 check failed, {}(calc) != "
-                        "{}(base)".format(fullname, calc_md5sum, md5sum))
+        logger.info("File {} md5 check failed, {}(calc) != "
+                    "{}(base)".format(fullname, calc_md5sum, md5sum))
         return False
     return True