merge paddle/develop

docs/apis/models.md

@@ -182,7 +182,7 @@ paddlex.det.FasterRCNN(num_classes=81, backbone='ResNet50', with_fpn=True, aspec
 **参数:**
 
 > - **num_classes** (int): 包含了背景类的类别数。默认为81。
-> - **backbone** (str): FasterRCNN的backbone网络，取值范围为['ResNet18', 'ResNet50', 'ResNet50vd', 'ResNet101', 'ResNet101vd']。默认为'ResNet50'。
+> - **backbone** (str): FasterRCNN的backbone网络，取值范围为['ResNet18', 'ResNet50', 'ResNet50_vd', 'ResNet101', 'ResNet101_vd']。默认为'ResNet50'。
 > - **with_fpn** (bool): 是否使用FPN结构。默认为True。
 > - **aspect_ratios** (list): 生成anchor高宽比的可选值。默认为[0.5, 1.0, 2.0]。
 > - **anchor_sizes** (list): 生成anchor大小的可选值。默认为[32, 64, 128, 256, 512]。
@@ -262,7 +262,7 @@ paddlex.det.MaskRCNN(num_classes=81, backbone='ResNet50', with_fpn=True, aspect_
 **参数:**
 
 > - **num_classes** (int): 包含了背景类的类别数。默认为81。
-> - **backbone** (str): MaskRCNN的backbone网络，取值范围为['ResNet18', 'ResNet50', 'ResNet50vd', 'ResNet101', 'ResNet101vd']。默认为'ResNet50'。
+> - **backbone** (str): MaskRCNN的backbone网络，取值范围为['ResNet18', 'ResNet50', 'ResNet50_vd', 'ResNet101', 'ResNet101_vd']。默认为'ResNet50'。
 > - **with_fpn** (bool): 是否使用FPN结构。默认为True。
 > - **aspect_ratios** (list): 生成anchor高宽比的可选值。默认为[0.5, 1.0, 2.0]。
 > - **anchor_sizes** (list): 生成anchor大小的可选值。默认为[32, 64, 128, 256, 512]。

docs/index.rst

@@ -22,7 +22,7 @@ PaddleX是基于飞桨技术生态的深度学习全流程开发工具。具备
    client_use.md
    FAQ.md
 
-* PaddleX版本: v0.1.6
+* PaddleX版本: v0.1.7
 * 项目官网: http://www.paddlepaddle.org.cn/paddle/paddlex  
 * 项目GitHub: https://github.com/PaddlePaddle/PaddleX/tree/develop  
 * 官方QQ用户群: 1045148026  

paddlex/__init__.py

@@ -13,6 +13,14 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+import os
+if 'FLAGS_eager_delete_tensor_gb' not in os.environ:
+    os.environ['FLAGS_eager_delete_tensor_gb'] = '0.0'
+if 'FLAGS_allocator_strategy' not in os.environ:
+    os.environ['FLAGS_allocator_strategy'] = 'auto_growth'
+if "CUDA_VISIBLE_DEVICES" in os.environ:
+    if os.environ["CUDA_VISIBLE_DEVICES"].count("-1") > 0:
+        os.environ["CUDA_VISIBLE_DEVICES"] = ""
 from .utils.utils import get_environ_info
 from . import cv
 from . import det
@@ -31,13 +39,13 @@ except:
         "[WARNING] pycocotools install: https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/install.md"
     )
 
-import paddlehub as hub
-if hub.version.hub_version < '1.6.2':
-    raise Exception("[ERROR] paddlehub >= 1.6.2 is required")
+#import paddlehub as hub
+#if hub.version.hub_version < '1.6.2':
+#    raise Exception("[ERROR] paddlehub >= 1.6.2 is required")
 
 env_info = get_environ_info()
 load_model = cv.models.load_model
 datasets = cv.datasets
 
 log_level = 2
-__version__ = '0.1.6.github'
+__version__ = '0.1.7.github'

paddlex/cv/datasets/dataset.py

@@ -254,3 +254,11 @@ class Dataset:
             buffer_size=self.buffer_size,
             batch_size=batch_size,
             drop_last=drop_last)
+
+    def set_num_samples(self, num_samples):
+        if num_samples > len(self.file_list):
+            logging.warning(
+                "You want set num_samples to {}, but your dataset only has {} samples, so we will keep your dataset num_samples as {}"
+                .format(num_samples, len(self.file_list), len(self.file_list)))
+            num_samples = len(self.file_list)
+        self.num_samples = num_samples

paddlex/cv/models/base.py

@@ -200,18 +200,31 @@ class BaseAPI:
         self.exe.run(startup_prog)
         if pretrain_weights is not None:
             logging.info(
-                "Load pretrain weights from {}.".format(pretrain_weights))
+                "Load pretrain weights from {}.".format(pretrain_weights),
+                use_color=True)
             paddlex.utils.utils.load_pretrain_weights(
                 self.exe, self.train_prog, pretrain_weights, fuse_bn)
         # 进行裁剪
         if sensitivities_file is not None:
+            import paddleslim
             from .slim.prune_config import get_sensitivities
             sensitivities_file = get_sensitivities(sensitivities_file, self,
                                                    save_dir)
             from .slim.prune import get_params_ratios, prune_program
+            logging.info(
+                "Start to prune program with eval_metric_loss = {}".format(
+                    eval_metric_loss),
+                use_color=True)
+            origin_flops = paddleslim.analysis.flops(self.test_prog)
             prune_params_ratios = get_params_ratios(
                 sensitivities_file, eval_metric_loss=eval_metric_loss)
             prune_program(self, prune_params_ratios)
+            current_flops = paddleslim.analysis.flops(self.test_prog)
+            remaining_ratio = current_flops / origin_flops
+            logging.info(
+                "Finish prune program, before FLOPs:{}, after prune FLOPs:{}, remaining ratio:{}"
+                .format(origin_flops, current_flops, remaining_ratio),
+                use_color=True)
             self.status = 'Prune'
 
     def get_model_info(self):
@@ -259,7 +272,10 @@ class BaseAPI:
             if osp.exists(save_dir):
                 os.remove(save_dir)
             os.makedirs(save_dir)
-        fluid.save(self.train_prog, osp.join(save_dir, 'model'))
+        if self.train_prog is not None:
+            fluid.save(self.train_prog, osp.join(save_dir, 'model'))
+        else:
+            fluid.save(self.test_prog, osp.join(save_dir, 'model'))
         model_info = self.get_model_info()
         model_info['status'] = self.status
         with open(
@@ -408,7 +424,7 @@ class BaseAPI:
             earlystop = EarlyStop(early_stop_patience, thresh)
         best_accuracy_key = ""
         best_accuracy = -1.0
-        best_model_epoch = 1
+        best_model_epoch = -1
         for i in range(num_epochs):
             records = list()
             step_start_time = time.time()
@@ -481,7 +497,7 @@ class BaseAPI:
                 current_save_dir = osp.join(save_dir, "epoch_{}".format(i + 1))
                 if not osp.isdir(current_save_dir):
                     os.makedirs(current_save_dir)
-                if eval_dataset is not None:
+                if eval_dataset is not None and eval_dataset.num_samples > 0:
                     self.eval_metrics, self.eval_details = self.evaluate(
                         eval_dataset=eval_dataset,
                         batch_size=eval_batch_size,
@@ -513,10 +529,11 @@ class BaseAPI:
                 self.save_model(save_dir=current_save_dir)
                 time_eval_one_epoch = time.time() - eval_epoch_start_time
                 eval_epoch_start_time = time.time()
-                logging.info(
-                    'Current evaluated best model in eval_dataset is epoch_{}, {}={}'
-                    .format(best_model_epoch, best_accuracy_key,
-                            best_accuracy))
+                if best_model_epoch > 0:
+                    logging.info(
+                        'Current evaluated best model in eval_dataset is epoch_{}, {}={}'
+                        .format(best_model_epoch, best_accuracy_key,
+                                best_accuracy))
                 if eval_dataset is not None and early_stop:
                     if earlystop(current_accuracy):
                         break

paddlex/cv/models/slim/prune_config.py

@@ -15,7 +15,7 @@
 import numpy as np
 import os.path as osp
 import paddle.fluid as fluid
-import paddlehub as hub
+#import paddlehub as hub
 import paddlex
 
 sensitivities_data = {
@@ -105,22 +105,26 @@ def get_sensitivities(flag, model, save_dir):
             model_type)
         url = sensitivities_data[model_type]
         fname = osp.split(url)[-1]
-        try:
-            hub.download(fname, save_path=save_dir)
-        except Exception as e:
-            if isinstance(e, hub.ResourceNotFoundError):
-                raise Exception(
-                    "Resource for model {}(key='{}') not found".format(
-                        model_type, fname))
-            elif isinstance(e, hub.ServerConnectionError):
-                raise Exception(
-                    "Cannot get reource for model {}(key='{}'), please check your internet connecgtion"
-                    .format(model_type, fname))
-            else:
-                raise Exception(
-                    "Unexpected error, please make sure paddlehub >= 1.6.2 {}".
-                    format(str(e)))
+        paddlex.utils.download(url, path=save_dir)
         return osp.join(save_dir, fname)
+
+
+#        try:
+#            hub.download(fname, save_path=save_dir)
+#        except Exception as e:
+#            if isinstance(e, hub.ResourceNotFoundError):
+#                raise Exception(
+#                    "Resource for model {}(key='{}') not found".format(
+#                        model_type, fname))
+#            elif isinstance(e, hub.ServerConnectionError):
+#                raise Exception(
+#                    "Cannot get reource for model {}(key='{}'), please check your internet connecgtion"
+#                    .format(model_type, fname))
+#            else:
+#                raise Exception(
+#                    "Unexpected error, please make sure paddlehub >= 1.6.2 {}".
+#                    format(str(e)))
+#        return osp.join(save_dir, fname)
     else:
         raise Exception(
             "sensitivities need to be defined as directory path or `DEFAULT`(download sensitivities automatically)."

paddlex/cv/models/utils/pretrain_weights.py

@@ -1,5 +1,5 @@
 import paddlex
-import paddlehub as hub
+#import paddlehub as hub
 import os
 import os.path as osp
 
@@ -85,40 +85,53 @@ def get_pretrain_weights(flag, model_type, backbone, save_dir):
                 backbone = 'DetResNet50'
         assert backbone in image_pretrain, "There is not ImageNet pretrain weights for {}, you may try COCO.".format(
             backbone)
-        try:
-            hub.download(backbone, save_path=new_save_dir)
-        except Exception as e:
-            if isinstance(e, hub.ResourceNotFoundError):
-                raise Exception(
-                    "Resource for backbone {} not found".format(backbone))
-            elif isinstance(e, hub.ServerConnectionError):
-                raise Exception(
-                    "Cannot get reource for backbone {}, please check your internet connecgtion"
-                    .format(backbone))
-            else:
-                raise Exception(
-                    "Unexpected error, please make sure paddlehub >= 1.6.2")
-        return osp.join(new_save_dir, backbone)
+        url = image_pretrain[backbone]
+        fname = osp.split(url)[-1].split('.')[0]
+        paddlex.utils.download_and_decompress(url, path=new_save_dir)
+        return osp.join(new_save_dir, fname)
+#        try:
+#            hub.download(backbone, save_path=new_save_dir)
+#        except Exception as e:
+#            if isinstance(e, hub.ResourceNotFoundError):
+#                raise Exception(
+#                    "Resource for backbone {} not found".format(backbone))
+#            elif isinstance(e, hub.ServerConnectionError):
+#                raise Exception(
+#                    "Cannot get reource for backbone {}, please check your internet connecgtion"
+#                    .format(backbone))
+#            else:
+#                raise Exception(
+#                    "Unexpected error, please make sure paddlehub >= 1.6.2")
+#        return osp.join(new_save_dir, backbone)
     elif flag == 'COCO':
         new_save_dir = save_dir
         if hasattr(paddlex, 'pretrain_dir'):
             new_save_dir = paddlex.pretrain_dir
-        assert backbone in coco_pretrain, "There is not COCO pretrain weights for {}, you may try ImageNet.".format(
-            backbone)
-        try:
-            hub.download(backbone, save_path=new_save_dir)
-        except Exception as e:
-            if isinstance(hub.ResourceNotFoundError):
-                raise Exception(
-                    "Resource for backbone {} not found".format(backbone))
-            elif isinstance(hub.ServerConnectionError):
-                raise Exception(
-                    "Cannot get reource for backbone {}, please check your internet connecgtion"
-                    .format(backbone))
-            else:
-                raise Exception(
-                    "Unexpected error, please make sure paddlehub >= 1.6.2")
-        return osp.join(new_save_dir, backbone)
+        url = coco_pretrain[backbone]
+        fname = osp.split(url)[-1].split('.')[0]
+        paddlex.utils.download_and_decompress(url, path=new_save_dir)
+        return osp.join(new_save_dir, fname)
+
+
+#        new_save_dir = save_dir
+#        if hasattr(paddlex, 'pretrain_dir'):
+#            new_save_dir = paddlex.pretrain_dir
+#        assert backbone in coco_pretrain, "There is not COCO pretrain weights for {}, you may try ImageNet.".format(
+#            backbone)
+#        try:
+#            hub.download(backbone, save_path=new_save_dir)
+#        except Exception as e:
+#            if isinstance(hub.ResourceNotFoundError):
+#                raise Exception(
+#                    "Resource for backbone {} not found".format(backbone))
+#            elif isinstance(hub.ServerConnectionError):
+#                raise Exception(
+#                    "Cannot get reource for backbone {}, please check your internet connecgtion"
+#                    .format(backbone))
+#            else:
+#                raise Exception(
+#                    "Unexpected error, please make sure paddlehub >= 1.6.2")
+#        return osp.join(new_save_dir, backbone)
     else:
         raise Exception(
             "pretrain_weights need to be defined as directory path or `IMAGENET` or 'COCO' (download pretrain weights automatically)."

paddlex/utils/utils.py

@@ -31,18 +31,7 @@ def seconds_to_hms(seconds):
     return hms_str
 
 
-def setting_environ_flags():
-    if 'FLAGS_eager_delete_tensor_gb' not in os.environ:
-        os.environ['FLAGS_eager_delete_tensor_gb'] = '0.0'
-    if 'FLAGS_allocator_strategy' not in os.environ:
-        os.environ['FLAGS_allocator_strategy'] = 'auto_growth'
-    if "CUDA_VISIBLE_DEVICES" in os.environ:
-        if os.environ["CUDA_VISIBLE_DEVICES"].count("-1") > 0:
-            os.environ["CUDA_VISIBLE_DEVICES"] = ""
-
-
 def get_environ_info():
-    setting_environ_flags()
     import paddle.fluid as fluid
     info = dict()
     info['place'] = 'cpu'

setup.py

@@ -19,7 +19,7 @@ long_description = "PaddleX. A end-to-end deeplearning model development toolkit
 
 setuptools.setup(
     name="paddlex",
-    version='0.1.6',
+    version='0.1.7',
     author="paddlex",
     author_email="paddlex@baidu.com",
     description=long_description,
@@ -29,9 +29,8 @@ setuptools.setup(
     packages=setuptools.find_packages(),
     setup_requires=['cython', 'numpy', 'sklearn'],
     install_requires=[
-        "pycocotools;platform_system!='Windows'", 
-        'pyyaml', 'colorama', 'tqdm', 'visualdl==1.3.0',
-        'paddleslim==1.0.1', 'paddlehub>=1.6.2'
+        "pycocotools;platform_system!='Windows'", 'pyyaml', 'colorama', 'tqdm',
+        'visualdl==1.3.0', 'paddleslim==1.0.1'
     ],
     classifiers=[
         "Programming Language :: Python :: 3",