Merge pull request #1049 from will-jl944/deploy_jf

add python deployment

PaddleDetection

@@ -1 +1 @@
-Subproject commit 6ec36068ee450df33c0038fb0cdc575b21510acd
+Subproject commit 3bdf2671f3188de3c4158c9056a46e949cf02eb8

paddlex/__init__.py

@@ -22,6 +22,7 @@ from . import seg
 from . import cls
 from . import det
 from . import tools
+from . import deploy
 
 from .cv.models.utils.visualize import visualize_detection as visualize_det
 from .cv.models.utils.visualize import visualize_segmentation as visualize_seg

paddlex/cv/models/base.py

@@ -34,6 +34,7 @@ from paddlex.utils import (seconds_to_hms, get_single_card_bs, dict2str,
                            _get_shared_memory_size_in_M, EarlyStop)
 import paddlex.utils.logging as logging
 from .slim.prune import _pruner_eval_fn, _pruner_template_input, sensitive_prune
+from .utils.infer_nets import InferNet
 
 
 class BaseModel:
@@ -504,6 +505,10 @@ class BaseModel:
             logging.info("Pruned model is saved at {}".format(save_dir))
 
     def _prepare_qat(self, quant_config):
+        if self.status == 'Infer':
+            logging.error(
+                "Exported inference model does not support quantization aware training.",
+                exit=True)
         if quant_config is None:
             # default quantization configuration
             quant_config = {
@@ -578,13 +583,19 @@ class BaseModel:
         pipeline_info["version"] = "1.0.0"
         return pipeline_info
 
+    def _build_inference_net(self):
+        infer_net = self.net if self.model_type == 'detector' else InferNet(
+            self.net, self.model_type)
+        infer_net.eval()
+        return infer_net
+
     def _export_inference_model(self, save_dir, image_shape=None):
         save_dir = osp.join(save_dir, 'inference_model')
-        self.net.eval()
         self.test_inputs = self._get_test_inputs(image_shape)
+        infer_net = self._build_inference_net()
 
         if self.status == 'Quantized':
-            self.quantizer.save_quantized_model(self.net,
+            self.quantizer.save_quantized_model(infer_net,
                                                 osp.join(save_dir, 'model'),
                                                 self.test_inputs)
             quant_info = self.get_quant_info()
@@ -594,7 +605,7 @@ class BaseModel:
                 yaml.dump(quant_info, f)
         else:
             static_net = paddle.jit.to_static(
-                self.net, input_spec=self.test_inputs)
+                infer_net, input_spec=self.test_inputs)
             paddle.jit.save(static_net, osp.join(save_dir, 'model'))
 
         if self.status == 'Pruned':

paddlex/cv/models/classifier.py

@@ -18,7 +18,6 @@ import os.path as osp
 from collections import OrderedDict
 import numpy as np
 import paddle
-from paddle import to_tensor
 import paddle.nn.functional as F
 from paddle.static import InputSpec
 from paddlex.utils import logging, TrainingStats, DisablePrint
@@ -55,6 +54,8 @@ class BaseClassifier(BaseModel):
         self.init_params.update(params)
         if 'lr_mult_list' in self.init_params:
             del self.init_params['lr_mult_list']
+        if 'with_net' in self.init_params:
+            del self.init_params['with_net']
         super(BaseClassifier, self).__init__('classifier')
         if not hasattr(architectures, model_name):
             raise Exception("ERROR: There's no model named {}.".format(
@@ -65,7 +66,9 @@ class BaseClassifier(BaseModel):
         self.num_classes = num_classes
         for k, v in params.items():
             setattr(self, k, v)
-        self.net = self.build_net(**params)
+        if params.get('with_net', True):
+            params.pop('with_net', None)
+            self.net = self.build_net(**params)
 
     def build_net(self, **params):
         with paddle.utils.unique_name.guard():
@@ -95,7 +98,7 @@ class BaseClassifier(BaseModel):
 
     def run(self, net, inputs, mode):
         net_out = net(inputs[0])
-        softmax_out = F.softmax(net_out)
+        softmax_out = net_out if self.status == 'Infer' else F.softmax(net_out)
         if mode == 'test':
             outputs = OrderedDict([('prediction', softmax_out)])
 
@@ -225,6 +228,10 @@ class BaseClassifier(BaseModel):
                 `pretrain_weights` can be set simultaneously. Defaults to None.
 
         """
+        if self.status == 'Infer':
+            logging.error(
+                "Exported inference model does not support training.",
+                exit=True)
         if pretrain_weights is not None and resume_checkpoint is not None:
             logging.error(
                 "pretrain_weights and resume_checkpoint cannot be set simultaneously.",
@@ -407,7 +414,7 @@ class BaseClassifier(BaseModel):
         """
         Do inference.
         Args:
-            img_file(List[np.ndarray or str], str or np.ndarray): img_file(list or str or np.array)：
+            img_file(List[np.ndarray or str], str or np.ndarray):
                 Image path or decoded image data in a BGR format, which also could constitute a list,
                 meaning all images to be predicted as a mini-batch.
             transforms(paddlex.transforms.Compose or None, optional):
@@ -432,36 +439,39 @@ class BaseClassifier(BaseModel):
             images = [img_file]
         else:
             images = img_file
-        im = self._preprocess(images, transforms, self.model_type)
+        im = self._preprocess(images, transforms)
         self.net.eval()
         with paddle.no_grad():
             outputs = self.run(self.net, im, mode='test')
         prediction = outputs['prediction'].numpy()
-        prediction = self._postprocess(prediction, true_topk, self.labels)
+        prediction = self._postprocess(prediction, true_topk)
         if isinstance(img_file, (str, np.ndarray)):
             prediction = prediction[0]
 
         return prediction
 
-    def _preprocess(self, images, transforms, model_type):
+    def _preprocess(self, images, transforms, to_tensor=True):
         arrange_transforms(
-            model_type=model_type, transforms=transforms, mode='test')
+            model_type=self.model_type, transforms=transforms, mode='test')
         batch_im = list()
         for im in images:
             sample = {'image': im}
             batch_im.append(transforms(sample))
 
-        batch_im = to_tensor(batch_im)
+        if to_tensor:
+            batch_im = paddle.to_tensor(batch_im)
+        else:
+            batch_im = np.asarray(batch_im)
 
         return batch_im,
 
-    def _postprocess(self, results, true_topk, labels):
+    def _postprocess(self, results, true_topk):
         preds = list()
         for i, pred in enumerate(results):
             pred_label = np.argsort(pred)[::-1][:true_topk]
             preds.append([{
                 'category_id': l,
-                'category': labels[l],
+                'category': self.labels[l],
                 'score': results[i][l]
             } for l in pred_label])
 
@@ -469,93 +479,95 @@ class BaseClassifier(BaseModel):
 
 
 class ResNet18(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet18, self).__init__(
-            model_name='ResNet18', num_classes=num_classes)
+            model_name='ResNet18', num_classes=num_classes, **params)
 
 
 class ResNet34(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet34, self).__init__(
-            model_name='ResNet34', num_classes=num_classes)
+            model_name='ResNet34', num_classes=num_classes, **params)
 
 
 class ResNet50(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet50, self).__init__(
-            model_name='ResNet50', num_classes=num_classes)
+            model_name='ResNet50', num_classes=num_classes, **params)
 
 
 class ResNet101(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet101, self).__init__(
-            model_name='ResNet101', num_classes=num_classes)
+            model_name='ResNet101', num_classes=num_classes, **params)
 
 
 class ResNet152(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet152, self).__init__(
-            model_name='ResNet152', num_classes=num_classes)
+            model_name='ResNet152', num_classes=num_classes, **params)
 
 
 class ResNet18_vd(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet18_vd, self).__init__(
-            model_name='ResNet18_vd', num_classes=num_classes)
+            model_name='ResNet18_vd', num_classes=num_classes, **params)
 
 
 class ResNet34_vd(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet34_vd, self).__init__(
-            model_name='ResNet34_vd', num_classes=num_classes)
+            model_name='ResNet34_vd', num_classes=num_classes, **params)
 
 
 class ResNet50_vd(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet50_vd, self).__init__(
-            model_name='ResNet50_vd', num_classes=num_classes)
+            model_name='ResNet50_vd', num_classes=num_classes, **params)
 
 
 class ResNet50_vd_ssld(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet50_vd_ssld, self).__init__(
             model_name='ResNet50_vd',
             num_classes=num_classes,
-            lr_mult_list=[.1, .1, .2, .2, .3])
+            lr_mult_list=[.1, .1, .2, .2, .3],
+            **params)
         self.model_name = 'ResNet50_vd_ssld'
 
 
 class ResNet101_vd(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet101_vd, self).__init__(
-            model_name='ResNet101_vd', num_classes=num_classes)
+            model_name='ResNet101_vd', num_classes=num_classes, **params)
 
 
 class ResNet101_vd_ssld(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet101_vd_ssld, self).__init__(
             model_name='ResNet101_vd',
             num_classes=num_classes,
-            lr_mult_list=[.1, .1, .2, .2, .3])
+            lr_mult_list=[.1, .1, .2, .2, .3],
+            **params)
         self.model_name = 'ResNet101_vd_ssld'
 
 
 class ResNet152_vd(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet152_vd, self).__init__(
-            model_name='ResNet152_vd', num_classes=num_classes)
+            model_name='ResNet152_vd', num_classes=num_classes, **params)
 
 
 class ResNet200_vd(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ResNet200_vd, self).__init__(
-            model_name='ResNet200_vd', num_classes=num_classes)
+            model_name='ResNet200_vd', num_classes=num_classes, **params)
 
 
 class AlexNet(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(AlexNet, self).__init__(
-            model_name='AlexNet', num_classes=num_classes)
+            model_name='AlexNet', num_classes=num_classes, **params)
 
     def _get_test_inputs(self, image_shape):
         if image_shape is not None:
@@ -580,13 +592,13 @@ class AlexNet(BaseClassifier):
 
 
 class DarkNet53(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(DarkNet53, self).__init__(
-            model_name='DarkNet53', num_classes=num_classes)
+            model_name='DarkNet53', num_classes=num_classes, **params)
 
 
 class MobileNetV1(BaseClassifier):
-    def __init__(self, num_classes=1000, scale=1.0):
+    def __init__(self, num_classes=1000, scale=1.0, **params):
         supported_scale = [.25, .5, .75, 1.0]
         if scale not in supported_scale:
             logging.warning("scale={} is not supported by MobileNetV1, "
@@ -598,11 +610,11 @@ class MobileNetV1(BaseClassifier):
             model_name = 'MobileNetV1_x' + str(scale).replace('.', '_')
         self.scale = scale
         super(MobileNetV1, self).__init__(
-            model_name=model_name, num_classes=num_classes)
+            model_name=model_name, num_classes=num_classes, **params)
 
 
 class MobileNetV2(BaseClassifier):
-    def __init__(self, num_classes=1000, scale=1.0):
+    def __init__(self, num_classes=1000, scale=1.0, **params):
         supported_scale = [.25, .5, .75, 1.0, 1.5, 2.0]
         if scale not in supported_scale:
             logging.warning("scale={} is not supported by MobileNetV2, "
@@ -613,11 +625,11 @@ class MobileNetV2(BaseClassifier):
         else:
             model_name = 'MobileNetV2_x' + str(scale).replace('.', '_')
         super(MobileNetV2, self).__init__(
-            model_name=model_name, num_classes=num_classes)
+            model_name=model_name, num_classes=num_classes, **params)
 
 
 class MobileNetV3_small(BaseClassifier):
-    def __init__(self, num_classes=1000, scale=1.0):
+    def __init__(self, num_classes=1000, scale=1.0, **params):
         supported_scale = [.35, .5, .75, 1.0, 1.25]
         if scale not in supported_scale:
             logging.warning("scale={} is not supported by MobileNetV3_small, "
@@ -626,11 +638,11 @@ class MobileNetV3_small(BaseClassifier):
         model_name = 'MobileNetV3_small_x' + str(float(scale)).replace('.',
                                                                        '_')
         super(MobileNetV3_small, self).__init__(
-            model_name=model_name, num_classes=num_classes)
+            model_name=model_name, num_classes=num_classes, **params)
 
 
 class MobileNetV3_small_ssld(BaseClassifier):
-    def __init__(self, num_classes=1000, scale=1.0):
+    def __init__(self, num_classes=1000, scale=1.0, **params):
         supported_scale = [.35, 1.0]
         if scale not in supported_scale:
             logging.warning(
@@ -640,12 +652,12 @@ class MobileNetV3_small_ssld(BaseClassifier):
         model_name = 'MobileNetV3_small_x' + str(float(scale)).replace('.',
                                                                        '_')
         super(MobileNetV3_small_ssld, self).__init__(
-            model_name=model_name, num_classes=num_classes)
+            model_name=model_name, num_classes=num_classes, **params)
         self.model_name = model_name + '_ssld'
 
 
 class MobileNetV3_large(BaseClassifier):
-    def __init__(self, num_classes=1000, scale=1.0):
+    def __init__(self, num_classes=1000, scale=1.0, **params):
         supported_scale = [.35, .5, .75, 1.0, 1.25]
         if scale not in supported_scale:
             logging.warning("scale={} is not supported by MobileNetV3_large, "
@@ -654,108 +666,110 @@ class MobileNetV3_large(BaseClassifier):
         model_name = 'MobileNetV3_large_x' + str(float(scale)).replace('.',
                                                                        '_')
         super(MobileNetV3_large, self).__init__(
-            model_name=model_name, num_classes=num_classes)
+            model_name=model_name, num_classes=num_classes, **params)
 
 
 class MobileNetV3_large_ssld(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(MobileNetV3_large_ssld, self).__init__(
-            model_name='MobileNetV3_large_x1_0', num_classes=num_classes)
+            model_name='MobileNetV3_large_x1_0',
+            num_classes=num_classes,
+            **params)
         self.model_name = 'MobileNetV3_large_x1_0_ssld'
 
 
 class DenseNet121(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(DenseNet121, self).__init__(
-            model_name='DenseNet121', num_classes=num_classes)
+            model_name='DenseNet121', num_classes=num_classes, **params)
 
 
 class DenseNet161(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(DenseNet161, self).__init__(
-            model_name='DenseNet161', num_classes=num_classes)
+            model_name='DenseNet161', num_classes=num_classes, **params)
 
 
 class DenseNet169(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(DenseNet169, self).__init__(
-            model_name='DenseNet169', num_classes=num_classes)
+            model_name='DenseNet169', num_classes=num_classes, **params)
 
 
 class DenseNet201(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(DenseNet201, self).__init__(
-            model_name='DenseNet201', num_classes=num_classes)
+            model_name='DenseNet201', num_classes=num_classes, **params)
 
 
 class DenseNet264(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(DenseNet264, self).__init__(
-            model_name='DenseNet264', num_classes=num_classes)
+            model_name='DenseNet264', num_classes=num_classes, **params)
 
 
 class HRNet_W18_C(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(HRNet_W18_C, self).__init__(
-            model_name='HRNet_W18_C', num_classes=num_classes)
+            model_name='HRNet_W18_C', num_classes=num_classes, **params)
 
 
 class HRNet_W30_C(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(HRNet_W30_C, self).__init__(
-            model_name='HRNet_W30_C', num_classes=num_classes)
+            model_name='HRNet_W30_C', num_classes=num_classes, **params)
 
 
 class HRNet_W32_C(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(HRNet_W32_C, self).__init__(
-            model_name='HRNet_W32_C', num_classes=num_classes)
+            model_name='HRNet_W32_C', num_classes=num_classes, **params)
 
 
 class HRNet_W40_C(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(HRNet_W40_C, self).__init__(
-            model_name='HRNet_W40_C', num_classes=num_classes)
+            model_name='HRNet_W40_C', num_classes=num_classes, **params)
 
 
 class HRNet_W44_C(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(HRNet_W44_C, self).__init__(
-            model_name='HRNet_W44_C', num_classes=num_classes)
+            model_name='HRNet_W44_C', num_classes=num_classes, **params)
 
 
 class HRNet_W48_C(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(HRNet_W48_C, self).__init__(
-            model_name='HRNet_W48_C', num_classes=num_classes)
+            model_name='HRNet_W48_C', num_classes=num_classes, **params)
 
 
 class HRNet_W64_C(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(HRNet_W64_C, self).__init__(
-            model_name='HRNet_W64_C', num_classes=num_classes)
+            model_name='HRNet_W64_C', num_classes=num_classes, **params)
 
 
 class Xception41(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(Xception41, self).__init__(
-            model_name='Xception41', num_classes=num_classes)
+            model_name='Xception41', num_classes=num_classes, **params)
 
 
 class Xception65(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(Xception65, self).__init__(
-            model_name='Xception65', num_classes=num_classes)
+            model_name='Xception65', num_classes=num_classes, **params)
 
 
 class Xception71(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(Xception71, self).__init__(
-            model_name='Xception71', num_classes=num_classes)
+            model_name='Xception71', num_classes=num_classes, **params)
 
 
 class ShuffleNetV2(BaseClassifier):
-    def __init__(self, num_classes=1000, scale=1.0):
+    def __init__(self, num_classes=1000, scale=1.0, **params):
         supported_scale = [.25, .33, .5, 1.0, 1.5, 2.0]
         if scale not in supported_scale:
             logging.warning("scale={} is not supported by ShuffleNetV2, "
@@ -763,7 +777,7 @@ class ShuffleNetV2(BaseClassifier):
             scale = 1.0
         model_name = 'ShuffleNetV2_x' + str(float(scale)).replace('.', '_')
         super(ShuffleNetV2, self).__init__(
-            model_name=model_name, num_classes=num_classes)
+            model_name=model_name, num_classes=num_classes, **params)
 
     def _get_test_inputs(self, image_shape):
         if image_shape is not None:
@@ -788,9 +802,9 @@ class ShuffleNetV2(BaseClassifier):
 
 
 class ShuffleNetV2_swish(BaseClassifier):
-    def __init__(self, num_classes=1000):
+    def __init__(self, num_classes=1000, **params):
         super(ShuffleNetV2_swish, self).__init__(
-            model_name='ShuffleNetV2_x1_5', num_classes=num_classes)
+            model_name='ShuffleNetV2_x1_5', num_classes=num_classes, **params)
 
     def _get_test_inputs(self, image_shape):
         if image_shape is not None:

paddlex/cv/models/load_model.py

@@ -45,7 +45,7 @@ def load_rcnn_inference_model(model_dir):
     return net_state_dict
 
 
-def load_model(model_dir):
+def load_model(model_dir, **params):
     """
     Load saved model from a given directory.
     Args:
@@ -69,6 +69,10 @@ def load_model(model_dir):
             format(paddlex.__version__, version))
 
     status = model_info['status']
+    with_net = params.get('with_net', True)
+    if not with_net:
+        assert status == 'Infer', \
+            "Only exported inference models can be deployed, current model status is {}".format(status)
 
     if not hasattr(paddlex.cv.models, model_info['Model']):
         raise Exception("There's no attribute {} in paddlex.cv.models".format(
@@ -76,51 +80,60 @@ def load_model(model_dir):
     if 'model_name' in model_info['_init_params']:
         del model_info['_init_params']['model_name']
 
+    model_info['_init_params'].update({'with_net': with_net})
+
     with paddle.utils.unique_name.guard():
         model = getattr(paddlex.cv.models, model_info['Model'])(
             **model_info['_init_params'])
-
-        if 'Transforms' in model_info:
-            model.test_transforms = build_transforms(model_info['Transforms'])
-
-        if '_Attributes' in model_info:
-            for k, v in model_info['_Attributes'].items():
-                if k in model.__dict__:
-                    model.__dict__[k] = v
-
-        if status == 'Pruned' or osp.exists(osp.join(model_dir, "prune.yml")):
-            with open(osp.join(model_dir, "prune.yml")) as f:
-                pruning_info = yaml.load(f.read(), Loader=yaml.Loader)
-                inputs = pruning_info['pruner_inputs']
-                if model.model_type == 'detector':
-                    inputs = [{
-                        k: paddle.to_tensor(v)
-                        for k, v in inputs.items()
-                    }]
-                    model.net.eval()
-                model.pruner = getattr(paddleslim, pruning_info['pruner'])(
-                    model.net, inputs=inputs)
-                model.pruning_ratios = pruning_info['pruning_ratios']
-                model.pruner.prune_vars(
-                    ratios=model.pruning_ratios,
-                    axis=paddleslim.dygraph.prune.filter_pruner.FILTER_DIM)
-
-        if status == 'Quantized':
-            with open(osp.join(model_dir, "quant.yml")) as f:
-                quant_info = yaml.load(f.read(), Loader=yaml.Loader)
-                model.quant_config = quant_info['quant_config']
-                model.quantizer = paddleslim.QAT(model.quant_config)
-                model.quantizer.quantize(model.net)
-
-        if status == 'Infer':
-            if model_info['Model'] in ['FasterRCNN', 'MaskRCNN']:
-                net_state_dict = load_rcnn_inference_model(model_dir)
+        if with_net:
+            if status == 'Pruned' or osp.exists(
+                    osp.join(model_dir, "prune.yml")):
+                with open(osp.join(model_dir, "prune.yml")) as f:
+                    pruning_info = yaml.load(f.read(), Loader=yaml.Loader)
+                    inputs = pruning_info['pruner_inputs']
+                    if model.model_type == 'detector':
+                        inputs = [{
+                            k: paddle.to_tensor(v)
+                            for k, v in inputs.items()
+                        }]
+                        model.net.eval()
+                    model.pruner = getattr(paddleslim, pruning_info['pruner'])(
+                        model.net, inputs=inputs)
+                    model.pruning_ratios = pruning_info['pruning_ratios']
+                    model.pruner.prune_vars(
+                        ratios=model.pruning_ratios,
+                        axis=paddleslim.dygraph.prune.filter_pruner.FILTER_DIM)
+
+            if status == 'Quantized' or osp.exists(
+                    osp.join(model_dir, "quant.yml")):
+                with open(osp.join(model_dir, "quant.yml")) as f:
+                    quant_info = yaml.load(f.read(), Loader=yaml.Loader)
+                    model.quant_config = quant_info['quant_config']
+                    model.quantizer = paddleslim.QAT(model.quant_config)
+                    model.quantizer.quantize(model.net)
+
+            if status == 'Infer':
+                if osp.exists(osp.join(model_dir, "quant.yml")):
+                    logging.error(
+                        "Exported quantized model can not be loaded, only deployment is supported.",
+                        exit=True)
+                model.net = model._build_inference_net()
+                if model_info['Model'] in ['FasterRCNN', 'MaskRCNN']:
+                    net_state_dict = load_rcnn_inference_model(model_dir)
+                else:
+                    net_state_dict = paddle.load(osp.join(model_dir, 'model'))
             else:
-                net_state_dict = paddle.load(osp.join(model_dir, 'model'))
-        else:
-            net_state_dict = paddle.load(osp.join(model_dir, 'model.pdparams'))
-        model.net.set_state_dict(net_state_dict)
-
-        logging.info("Model[{}] loaded.".format(model_info['Model']))
-        model.status = status
+                net_state_dict = paddle.load(
+                    osp.join(model_dir, 'model.pdparams'))
+            model.net.set_state_dict(net_state_dict)
+
+    if 'Transforms' in model_info:
+        model.test_transforms = build_transforms(model_info['Transforms'])
+
+    if '_Attributes' in model_info:
+        for k, v in model_info['_Attributes'].items():
+            if k in model.__dict__:
+                model.__dict__[k] = v
+    logging.info("Model[{}] loaded.".format(model_info['Model']))
+    model.status = status
     return model

paddlex/cv/models/segmenter.py

@@ -15,6 +15,7 @@
 import math
 import os.path as osp
 import numpy as np
+import cv2
 from collections import OrderedDict
 import paddle
 import paddle.nn.functional as F
@@ -39,6 +40,8 @@ class BaseSegmenter(BaseModel):
                  use_mixed_loss=False,
                  **params):
         self.init_params = locals()
+        if 'with_net' in self.init_params:
+            del self.init_params['with_net']
         super(BaseSegmenter, self).__init__('segmenter')
         if not hasattr(paddleseg.models, model_name):
             raise Exception("ERROR: There's no model named {}.".format(
@@ -48,7 +51,9 @@ class BaseSegmenter(BaseModel):
         self.use_mixed_loss = use_mixed_loss
         self.losses = None
         self.labels = None
-        self.net = self.build_net(**params)
+        if params.get('with_net', True):
+            params.pop('with_net', None)
+            self.net = self.build_net(**params)
         self.find_unused_parameters = True
 
     def build_net(self, **params):
@@ -99,19 +104,28 @@ class BaseSegmenter(BaseModel):
         outputs = OrderedDict()
         if mode == 'test':
             origin_shape = inputs[1]
-            score_map = self._postprocess(
-                logit, origin_shape, transforms=inputs[2])
-            label_map = paddle.argmax(
-                score_map, axis=1, keepdim=True, dtype='int32')
-            score_map = paddle.max(score_map, axis=1, keepdim=True)
-            score_map = paddle.squeeze(score_map)
-            label_map = paddle.squeeze(label_map)
-            outputs = {'label_map': label_map, 'score_map': score_map}
+            if self.status == 'Infer':
+                score_map, label_map = self._postprocess(
+                    net_out, origin_shape, transforms=inputs[2])
+            else:
+                logit = self._postprocess(
+                    logit, origin_shape, transforms=inputs[2])
+                score_map = paddle.transpose(
+                    F.softmax(
+                        logit, axis=1), perm=[0, 2, 3, 1])
+                label_map = paddle.argmax(
+                    score_map, axis=-1, keepdim=True, dtype='int32')
+            outputs['label_map'] = paddle.squeeze(label_map)
+            outputs['score_map'] = paddle.squeeze(score_map)
+
         if mode == 'eval':
-            pred = paddle.argmax(logit, axis=1, keepdim=True, dtype='int32')
+            if self.status == 'Infer':
+                pred = paddle.transpose(net_out[1], perm=[0, 3, 1, 2])
+            else:
+                pred = paddle.argmax(
+                    logit, axis=1, keepdim=True, dtype='int32')
             label = inputs[1]
             origin_shape = [label.shape[-2:]]
-            # TODO: 替换cv2后postprocess移出run
             pred = self._postprocess(pred, origin_shape, transforms=inputs[2])
             intersect_area, pred_area, label_area = paddleseg.utils.metrics.calculate_area(
                 pred, label, self.num_classes)
@@ -221,6 +235,10 @@ class BaseSegmenter(BaseModel):
                 `pretrain_weights` can be set simultaneously. Defaults to None.
 
         """
+        if self.status == 'Infer':
+            logging.error(
+                "Exported inference model does not support training.",
+                exit=True)
         if pretrain_weights is not None and resume_checkpoint is not None:
             logging.error(
                 "pretrain_weights and resume_checkpoint cannot be set simultaneously.",
@@ -449,7 +467,7 @@ class BaseSegmenter(BaseModel):
         Do inference.
         Args:
             Args:
-            img_file(List[np.ndarray or str], str or np.ndarray): img_file(list or str or np.array)：
+            img_file(List[np.ndarray or str], str or np.ndarray):
                 Image path or decoded image data in a BGR format, which also could constitute a list,
                 meaning all images to be predicted as a mini-batch.
             transforms(paddlex.transforms.Compose or None, optional):
@@ -460,7 +478,7 @@ class BaseSegmenter(BaseModel):
             {"label map": `label map`, "score_map": `score map`}.
             If img_file is a list, the result is a list composed of dicts with the corresponding fields:
             label_map(np.ndarray): the predicted label map
-            score_map(np.ndarray): the prediction score map
+            score_map(np.ndarray): the prediction score map (NHWC)
 
         """
         if transforms is None and not hasattr(self, 'test_transforms'):
@@ -491,9 +509,9 @@ class BaseSegmenter(BaseModel):
             prediction = {'label_map': label_map, 'score_map': score_map}
         return prediction
 
-    def _preprocess(self, images, transforms, model_type):
+    def _preprocess(self, images, transforms, to_tensor=True):
         arrange_transforms(
-            model_type=model_type, transforms=transforms, mode='test')
+            model_type=self.model_type, transforms=transforms, mode='test')
         batch_im = list()
         batch_ori_shape = list()
         for im in images:
@@ -504,7 +522,10 @@ class BaseSegmenter(BaseModel):
             im = transforms(sample)[0]
             batch_im.append(im)
             batch_ori_shape.append(ori_shape)
-        batch_im = paddle.to_tensor(batch_im)
+        if to_tensor:
+            batch_im = paddle.to_tensor(batch_im)
+        else:
+            batch_im = np.asarray(batch_im)
 
         return batch_im, batch_ori_shape
 
@@ -559,14 +580,19 @@ class BaseSegmenter(BaseModel):
     def _postprocess(self, batch_pred, batch_origin_shape, transforms):
         batch_restore_list = BaseSegmenter.get_transforms_shape_info(
             batch_origin_shape, transforms)
-        results = list()
+        if isinstance(batch_pred, (tuple, list)) and self.status == 'Infer':
+            return self._infer_postprocess(
+                batch_score_map=batch_pred[0],
+                batch_label_map=batch_pred[1],
+                batch_restore_list=batch_restore_list)
+        results = []
         for pred, restore_list in zip(batch_pred, batch_restore_list):
             pred = paddle.unsqueeze(pred, axis=0)
             for item in restore_list[::-1]:
-                # TODO: 替换成cv2的interpolate（部署阶段无法使用paddle op）
                 h, w = item[1][0], item[1][1]
                 if item[0] == 'resize':
-                    pred = F.interpolate(pred, (h, w), mode='nearest')
+                    pred = F.interpolate(
+                        pred, (h, w), mode='nearest', data_format='NCHW')
                 elif item[0] == 'padding':
                     x, y = item[2]
                     pred = pred[:, :, y:y + h, x:x + w]
@@ -576,14 +602,63 @@ class BaseSegmenter(BaseModel):
         batch_pred = paddle.concat(results, axis=0)
         return batch_pred
 
+    def _infer_postprocess(self, batch_score_map, batch_label_map,
+                           batch_restore_list):
+        score_maps = []
+        label_maps = []
+        for score_map, label_map, restore_list in zip(
+                batch_score_map, batch_label_map, batch_restore_list):
+            if not isinstance(score_map, np.ndarray):
+                score_map = paddle.unsqueeze(score_map, axis=0)
+                label_map = paddle.unsqueeze(label_map, axis=0)
+            for item in restore_list[::-1]:
+                h, w = item[1][0], item[1][1]
+                if item[0] == 'resize':
+                    if isinstance(score_map, np.ndarray):
+                        score_map = cv2.resize(
+                            score_map, (h, w), interpolation=cv2.INTER_LINEAR)
+                        label_map = cv2.resize(
+                            label_map, (h, w), interpolation=cv2.INTER_NEAREST)
+                    else:
+                        score_map = F.interpolate(
+                            score_map, (h, w),
+                            mode='bilinear',
+                            data_format='NHWC')
+                        label_map = F.interpolate(
+                            label_map, (h, w),
+                            mode='nearest',
+                            data_format='NHWC')
+                elif item[0] == 'padding':
+                    x, y = item[2]
+                    if isinstance(score_map, np.ndarray):
+                        score_map = score_map[..., y:y + h, x:x + w]
+                        label_map = label_map[..., y:y + h, x:x + w]
+                    else:
+                        score_map = score_map[:, :, y:y + h, x:x + w]
+                        label_map = label_map[:, :, y:y + h, x:x + w]
+                else:
+                    pass
+            score_maps.append(score_map)
+            label_maps.append(label_map)
+        if isinstance(score_maps[0], np.ndarray):
+            return np.stack(score_maps, axis=0), np.stack(label_maps, axis=0)
+        else:
+            return paddle.concat(
+                score_maps, axis=0), paddle.concat(
+                    label_maps, axis=0)
+
 
 class UNet(BaseSegmenter):
     def __init__(self,
                  num_classes=2,
                  use_mixed_loss=False,
                  use_deconv=False,
-                 align_corners=False):
-        params = {'use_deconv': use_deconv, 'align_corners': align_corners}
+                 align_corners=False,
+                 **params):
+        params.update({
+            'use_deconv': use_deconv,
+            'align_corners': align_corners
+        })
         super(UNet, self).__init__(
             model_name='UNet',
             num_classes=num_classes,
@@ -600,22 +675,26 @@ class DeepLabV3P(BaseSegmenter):
                  backbone_indices=(0, 3),
                  aspp_ratios=(1, 12, 24, 36),
                  aspp_out_channels=256,
-                 align_corners=False):
+                 align_corners=False,
+                 **params):
         self.backbone_name = backbone
         if backbone not in ['ResNet50_vd', 'ResNet101_vd']:
             raise ValueError(
                 "backbone: {} is not supported. Please choose one of "
                 "('ResNet50_vd', 'ResNet101_vd')".format(backbone))
-        with DisablePrint():
-            backbone = getattr(paddleseg.models, backbone)(
-                output_stride=output_stride)
-        params = {
+        if params.get('with_net', True):
+            with DisablePrint():
+                backbone = getattr(paddleseg.models, backbone)(
+                    output_stride=output_stride)
+        else:
+            backbone = None
+        params.update({
             'backbone': backbone,
             'backbone_indices': backbone_indices,
             'aspp_ratios': aspp_ratios,
             'aspp_out_channels': aspp_out_channels,
             'align_corners': align_corners
-        }
+        })
         super(DeepLabV3P, self).__init__(
             model_name='DeepLabV3P',
             num_classes=num_classes,
@@ -627,8 +706,9 @@ class FastSCNN(BaseSegmenter):
     def __init__(self,
                  num_classes=2,
                  use_mixed_loss=False,
-                 align_corners=False):
-        params = {'align_corners': align_corners}
+                 align_corners=False,
+                 **params):
+        params.update({'align_corners': align_corners})
         super(FastSCNN, self).__init__(
             model_name='FastSCNN',
             num_classes=num_classes,
@@ -641,17 +721,21 @@ class HRNet(BaseSegmenter):
                  num_classes=2,
                  width=48,
                  use_mixed_loss=False,
-                 align_corners=False):
+                 align_corners=False,
+                 **params):
         if width not in (18, 48):
             raise ValueError(
                 "width={} is not supported, please choose from [18, 48]".
                 format(width))
         self.backbone_name = 'HRNet_W{}'.format(width)
-        with DisablePrint():
-            backbone = getattr(paddleseg.models, self.backbone_name)(
-                align_corners=align_corners)
+        if params.get('with_net', True):
+            with DisablePrint():
+                backbone = getattr(paddleseg.models, self.backbone_name)(
+                    align_corners=align_corners)
+        else:
+            backbone = None
 
-        params = {'backbone': backbone, 'align_corners': align_corners}
+        params.update({'backbone': backbone, 'align_corners': align_corners})
         super(HRNet, self).__init__(
             model_name='FCN',
             num_classes=num_classes,
@@ -664,8 +748,9 @@ class BiSeNetV2(BaseSegmenter):
     def __init__(self,
                  num_classes=2,
                  use_mixed_loss=False,
-                 align_corners=False):
-        params = {'align_corners': align_corners}
+                 align_corners=False,
+                 **params):
+        params.update({'align_corners': align_corners})
         super(BiSeNetV2, self).__init__(
             model_name='BiSeNetV2',
             num_classes=num_classes,

paddlex/cv/models/utils/infer_nets.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+
+
+class PostProcessor(paddle.nn.Layer):
+    def __init__(self, model_type):
+        super(PostProcessor, self).__init__()
+        self.model_type = model_type
+
+    def forward(self, net_outputs):
+        if self.model_type == 'classifier':
+            outputs = paddle.nn.functional.softmax(net_outputs, axis=1)
+        else:
+            # score_map, label_map
+            logit = net_outputs[0]
+            outputs = paddle.transpose(paddle.nn.functional.softmax(logit, axis=1), perm=[0, 2, 3, 1]), \
+                      paddle.transpose(paddle.argmax(logit, axis=1, keepdim=True, dtype='int32'),
+                                       perm=[0, 2, 3, 1])
+
+        return outputs
+
+
+class InferNet(paddle.nn.Layer):
+    def __init__(self, net, model_type):
+        super(InferNet, self).__init__()
+        self.net = net
+        self.postprocessor = PostProcessor(model_type)
+
+    def forward(self, x):
+        net_outputs = self.net(x)
+        outputs = self.postprocessor(net_outputs)
+
+        return outputs

paddlex/cv/transforms/batch_operators.py

@@ -26,10 +26,14 @@ from paddlex.utils import logging
 
 
 class BatchCompose(Transform):
-    def __init__(self, batch_transforms=None, collate_batch=True):
+    def __init__(self,
+                 batch_transforms=None,
+                 collate_batch=True,
+                 return_list=False):
         super(BatchCompose, self).__init__()
         self.batch_transforms = batch_transforms
         self.collate_batch = collate_batch
+        self.return_list = return_list
 
     def __call__(self, samples):
         if self.batch_transforms is not None:
@@ -51,7 +55,13 @@ class BatchCompose(Transform):
                 if k in sample:
                     sample.pop(k)
 
-        if self.collate_batch:
+        if self.return_list:
+            batch_data = [{
+                k: np.expand_dims(
+                    sample[k], axis=0)
+                for k in sample
+            } for sample in samples]
+        elif self.collate_batch:
             batch_data = default_collate_fn(samples)
         else:
             batch_data = {}

paddlex/deploy.py

@@ -0,0 +1,276 @@
+# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path as osp
+import numpy as np
+from paddle.inference import Config
+from paddle.inference import create_predictor
+from paddle.inference import PrecisionType
+from paddlex.cv.models import load_model
+from paddlex.utils import logging, Timer
+
+
+class Predictor(object):
+    def __init__(self,
+                 model_dir,
+                 use_gpu=False,
+                 gpu_id=0,
+                 cpu_thread_num=1,
+                 use_mkl=True,
+                 mkl_thread_num=4,
+                 use_trt=False,
+                 use_glog=False,
+                 memory_optimize=True,
+                 max_trt_batch_size=1,
+                 trt_precision_mode='float32'):
+        """ 创建Paddle Predictor
+
+            Args:
+                model_dir: 模型路径（必须是导出的部署或量化模型）
+                use_gpu: 是否使用gpu，默认False
+                gpu_id: 使用gpu的id，默认0
+                cpu_thread_num=1：使用cpu进行预测时的线程数，默认为1
+                use_mkl: 是否使用mkldnn计算库，CPU情况下使用，默认False
+                mkl_thread_num: mkldnn计算线程数，默认为4
+                use_trt: 是否使用TensorRT，默认False
+                use_glog: 是否启用glog日志, 默认False
+                memory_optimize: 是否启动内存优化，默认True
+                max_trt_batch_size: 在使用TensorRT时配置的最大batch size，默认1
+                trt_precision_mode：在使用TensorRT时采用的精度，默认float32
+        """
+        self.model_dir = model_dir
+        self._model = load_model(model_dir, with_net=False)
+
+        if trt_precision_mode.lower() == 'float32':
+            trt_precision_mode = PrecisionType.Float32
+        elif trt_precision_mode.lower() == 'float16':
+            trt_precision_mode = PrecisionType.Float16
+        else:
+            logging.error(
+                "TensorRT precision mode {} is invalid. Supported modes are float32 and float16."
+                .format(trt_precision_mode),
+                exit=True)
+
+        self.predictor = self.create_predictor(
+            use_gpu=use_gpu,
+            gpu_id=gpu_id,
+            cpu_thread_num=cpu_thread_num,
+            use_mkl=use_mkl,
+            mkl_thread_num=mkl_thread_num,
+            use_trt=use_trt,
+            use_glog=use_glog,
+            memory_optimize=memory_optimize,
+            max_trt_batch_size=max_trt_batch_size,
+            trt_precision_mode=trt_precision_mode)
+        self.timer = Timer()
+
+    def create_predictor(self,
+                         use_gpu=True,
+                         gpu_id=0,
+                         cpu_thread_num=1,
+                         use_mkl=True,
+                         mkl_thread_num=4,
+                         use_trt=False,
+                         use_glog=False,
+                         memory_optimize=True,
+                         max_trt_batch_size=1,
+                         trt_precision_mode=PrecisionType.Float32):
+        config = Config(
+            osp.join(self.model_dir, 'model.pdmodel'),
+            osp.join(self.model_dir, 'model.pdiparams'))
+
+        if use_gpu:
+            # 设置GPU初始显存(单位M)和Device ID
+            config.enable_use_gpu(100, gpu_id)
+            config.switch_ir_optim(True)
+            if use_trt:
+                config.enable_tensorrt_engine(
+                    workspace_size=1 << 10,
+                    max_batch_size=max_trt_batch_size,
+                    min_subgraph_size=3,
+                    precision_mode=trt_precision_mode,
+                    use_static=False,
+                    use_calib_mode=False)
+        else:
+            config.disable_gpu()
+            config.set_cpu_math_library_num_threads(cpu_thread_num)
+            if use_mkl:
+                try:
+                    # cache 10 different shapes for mkldnn to avoid memory leak
+                    config.set_mkldnn_cache_capacity(10)
+                    config.enable_mkldnn()
+                    config.set_cpu_math_library_num_threads(mkl_thread_num)
+                except Exception as e:
+                    logging.warning(
+                        "The current environment does not support `mkldnn`, so disable mkldnn."
+                    )
+                    pass
+
+        if use_glog:
+            config.enable_glog_info()
+        else:
+            config.disable_glog_info()
+        if memory_optimize:
+            config.enable_memory_optim()
+        config.switch_use_feed_fetch_ops(False)
+        predictor = create_predictor(config)
+        return predictor
+
+    def preprocess(self, images, transforms):
+        preprocessed_samples = self._model._preprocess(
+            images, transforms, to_tensor=False)
+        if self._model.model_type == 'classifier':
+            preprocessed_samples = {'image': preprocessed_samples[0]}
+        elif self._model.model_type == 'segmenter':
+            preprocessed_samples = {
+                'image': preprocessed_samples[0],
+                'ori_shape': preprocessed_samples[1]
+            }
+        elif self._model.model_type == 'detector':
+            pass
+        else:
+            logging.error(
+                "Invalid model type {}".format(self._model.model_type),
+                exit=True)
+        return preprocessed_samples
+
+    def postprocess(self, net_outputs, topk=1, ori_shape=None,
+                    transforms=None):
+        if self._model.model_type == 'classifier':
+            true_topk = min(self._model.num_classes, topk)
+            preds = self._model._postprocess(net_outputs[0], true_topk)
+            if len(preds) == 1:
+                preds = preds[0]
+        elif self._model.model_type == 'segmenter':
+            score_map, label_map = self._model._postprocess(
+                net_outputs,
+                batch_origin_shape=ori_shape,
+                transforms=transforms.transforms)
+            score_map = np.squeeze(score_map)
+            label_map = np.squeeze(label_map)
+            if score_map.ndim == 3:
+                preds = {'label_map': label_map, 'score_map': score_map}
+            else:
+                preds = [{
+                    'label_map': l,
+                    'score_map': s
+                } for l, s in zip(label_map, score_map)]
+        elif self._model.model_type == 'detector':
+            if 'RCNN' in self._model.__class__.__name__:
+                net_outputs = [{
+                    k: v
+                    for k, v in zip(['bbox', 'bbox_num', 'mask'], res)
+                } for res in net_outputs]
+            else:
+                net_outputs = {
+                    k: v
+                    for k, v in zip(['bbox', 'bbox_num', 'mask'], net_outputs)
+                }
+            preds = self._model._postprocess(net_outputs)
+            if len(preds) == 1:
+                preds = preds[0]
+        else:
+            logging.error(
+                "Invalid model type {}.".format(self._model.model_type),
+                exit=True)
+
+        return preds
+
+    def raw_predict(self, inputs):
+        """ 接受预处理过后的数据进行预测
+
+            Args:
+                inputs(dict): 预处理过后的数据
+        """
+        input_names = self.predictor.get_input_names()
+        for name in input_names:
+            input_tensor = self.predictor.get_input_handle(name)
+            input_tensor.copy_from_cpu(inputs[name])
+
+        self.predictor.run()
+        output_names = self.predictor.get_output_names()
+        net_outputs = list()
+        for name in output_names:
+            output_tensor = self.predictor.get_output_handle(name)
+            net_outputs.append(output_tensor.copy_to_cpu())
+
+        return net_outputs
+
+    def _run(self, images, topk=1, transforms=None):
+        self.timer.preprocess_time_s.start()
+        preprocessed_input = self.preprocess(images, transforms)
+        self.timer.preprocess_time_s.end(iter_num=len(images))
+
+        ori_shape = None
+        self.timer.inference_time_s.start()
+        if 'RCNN' in self._model.__class__.__name__:
+            if len(preprocessed_input) > 1:
+                logging.warning(
+                    "{} only supports inference with batch size equal to 1."
+                    .format(self._model.__class__.__name__))
+            net_outputs = [
+                self.raw_predict(sample) for sample in preprocessed_input
+            ]
+            self.timer.inference_time_s.end(iter_num=len(images))
+        else:
+            net_outputs = self.raw_predict(preprocessed_input)
+            self.timer.inference_time_s.end(iter_num=1)
+            ori_shape = preprocessed_input.get('ori_shape', None)
+
+        self.timer.postprocess_time_s.start()
+        results = self.postprocess(
+            net_outputs, topk, ori_shape=ori_shape, transforms=transforms)
+        self.timer.postprocess_time_s.end(iter_num=len(images))
+
+        return results
+
+    def predict(self,
+                img_file,
+                topk=1,
+                transforms=None,
+                warmup_iters=0,
+                repeats=1):
+        """ 图片预测
+
+            Args:
+                img_file(List[np.ndarray or str], str or np.ndarray):
+                    图像路径；或者是解码后的排列格式为（H, W, C）且类型为float32且为BGR格式的数组。
+                topk(int): 分类预测时使用，表示预测前topk的结果。
+                transforms (paddlex.transforms): 数据预处理操作。
+                warmup_iters (int): 预热轮数，默认为0。
+                repeats (int): 重复次数，用于评估模型推理以及前后处理速度。若大于1，会预测repeats次取时间平均值。
+        """
+        if repeats < 1:
+            logging.error("`repeats` must be greater than 1.", exit=True)
+        if transforms is None and not hasattr(self._model, 'test_transforms'):
+            raise Exception("Transforms need to be defined, now is None.")
+        if transforms is None:
+            transforms = self._model.test_transforms
+        if isinstance(img_file, (str, np.ndarray)):
+            images = [img_file]
+        else:
+            images = img_file
+
+        for step in range(warmup_iters):
+            self._run(images=images, topk=topk, transforms=transforms)
+        self.timer.reset()
+
+        for step in range(repeats):
+            results = self._run(
+                images=images, topk=topk, transforms=transforms)
+
+        self.timer.repeats = repeats
+        self.timer.info(average=True)
+
+        return results

paddlex/utils/__init__.py

@@ -16,7 +16,7 @@ from . import logging
 from . import utils
 from .utils import (seconds_to_hms, get_encoding, get_single_card_bs, dict2str,
                     EarlyStop, path_normalization, is_pic, MyEncoder,
-                    DisablePrint)
+                    DisablePrint, Timer)
 from .checkpoint import get_pretrain_weights, load_pretrain_weights, load_checkpoint
 from .env import get_environ_info, get_num_workers, init_parallel_env
 from .download import download_and_decompress, decompress

paddlex/utils/utils.py

@@ -14,6 +14,7 @@
 
 import sys
 import os
+import time
 import math
 import chardet
 import json
@@ -138,3 +139,90 @@ class DisablePrint(object):
     def __exit__(self, exc_type, exc_val, exc_tb):
         sys.stdout.close()
         sys.stdout = self._original_stdout
+
+
+class Times(object):
+    def __init__(self):
+        self.time = 0.
+        # start time
+        self.st = 0.
+        # end time
+        self.et = 0.
+
+    def start(self):
+        self.st = time.time()
+
+    def end(self, iter_num=1, accumulative=True):
+        self.et = time.time()
+        if accumulative:
+            self.time += (self.et - self.st) / iter_num
+        else:
+            self.time = (self.et - self.st) / iter_num
+
+    def reset(self):
+        self.time = 0.
+        self.st = 0.
+        self.et = 0.
+
+    def value(self):
+        return round(self.time, 4)
+
+
+class Timer(Times):
+    def __init__(self):
+        super(Timer, self).__init__()
+        self.preprocess_time_s = Times()
+        self.inference_time_s = Times()
+        self.postprocess_time_s = Times()
+        self.img_num = 0
+        self.repeats = 0
+
+    def info(self, average=False):
+        total_time = self.preprocess_time_s.value(
+        ) + self.inference_time_s.value() + self.postprocess_time_s.value()
+        total_time = round(total_time, 4)
+        print("------------------ Inference Time Info ----------------------")
+        print("total_time(ms): {}, img_num: {}, batch_size: {}".format(
+            total_time * 1000, self.img_num, self.img_num / self.repeats))
+        preprocess_time = round(
+            self.preprocess_time_s.value() / self.repeats,
+            4) if average else self.preprocess_time_s.value()
+        postprocess_time = round(
+            self.postprocess_time_s.value() / self.repeats,
+            4) if average else self.postprocess_time_s.value()
+        inference_time = round(self.inference_time_s.value() / self.repeats,
+                               4) if average else self.inference_time_s.value()
+
+        average_latency = total_time / self.repeats
+        print("average latency time(ms): {:.2f}, QPS: {:2f}".format(
+            average_latency * 1000, 1 / average_latency))
+        print("preprocess_time_per_im(ms): {:.2f}, "
+              "inference_time_per_batch(ms): {:.2f}, "
+              "postprocess_time_per_im(ms): {:.2f}".format(
+                  preprocess_time * 1000, inference_time * 1000,
+                  postprocess_time * 1000))
+
+    def report(self, average=False):
+        dic = {}
+        dic['preprocess_time_s'] = round(
+            self.preprocess_time_s.value() / self.repeats,
+            4) if average else self.preprocess_time_s.value()
+        dic['postprocess_time_s'] = round(
+            self.postprocess_time_s.value() / self.repeats,
+            4) if average else self.postprocess_time_s.value()
+        dic['inference_time_s'] = round(
+            self.inference_time_s.value() / self.repeats,
+            4) if average else self.inference_time_s.value()
+        dic['img_num'] = self.img_num
+        total_time = self.preprocess_time_s.value(
+        ) + self.inference_time_s.value() + self.postprocess_time_s.value()
+        dic['total_time_s'] = round(total_time, 4)
+        dic['batch_size'] = self.img_num / self.repeats
+        return dic
+
+    def reset(self):
+        self.preprocess_time_s.reset()
+        self.inference_time_s.reset()
+        self.postprocess_time_s.reset()
+        self.img_num = 0
+        self.repeats = 0