combine gui branch with api branch

paddlex/__init__.py

@@ -43,10 +43,6 @@ except:
         "[WARNING] pycocotools install: https://paddlex.readthedocs.io/zh_CN/develop/install.html#pycocotools"
     )
 
-import paddlehub as hub
-if hub.version.hub_version < '1.8.2':
-    raise Exception("[ERROR] paddlehub >= 1.8.2 is required")
-
 env_info = get_environ_info()
 load_model = cv.models.load_model
 datasets = cv.datasets

paddlex/cv/datasets/voc.py

@@ -138,20 +138,25 @@ class VOCDetection(Dataset):
                 difficult = np.zeros((len(objs), 1), dtype=np.int32)
                 for i, obj in enumerate(objs):
                     pattern = re.compile('<name>', re.IGNORECASE)
-                    name_tag = pattern.findall(str(ET.tostringlist(obj)))[0][
-                        1:-1]
+                    name_tag = pattern.findall(str(ET.tostringlist(obj)))[0][1:
+                                                                             -1]
                     cname = obj.find(name_tag).text.strip()
                     gt_class[i][0] = cname2cid[cname]
                     pattern = re.compile('<difficult>', re.IGNORECASE)
-                    diff_tag = pattern.findall(str(ET.tostringlist(obj)))[0][
-                        1:-1]
+                    diff_tag = pattern.findall(str(ET.tostringlist(obj)))[0][1:
+                                                                             -1]
                     try:
                         _difficult = int(obj.find(diff_tag).text)
                     except Exception:
                         _difficult = 0
                     pattern = re.compile('<bndbox>', re.IGNORECASE)
-                    box_tag = pattern.findall(str(ET.tostringlist(obj)))[0][1:
-                                                                            -1]
+                    box_tag = pattern.findall(str(ET.tostringlist(obj)))
+                    if len(box_tag) == 0:
+                        logging.warning(
+                            "There's no field '<bndbox>' in one of object, so this object will be ignored. xml file: {}".
+                            format(xml_file))
+                        continue
+                    box_tag = box_tag[0][1:-1]
                     box_element = obj.find(box_tag)
                     pattern = re.compile('<xmin>', re.IGNORECASE)
                     xmin_tag = pattern.findall(

paddlex/cv/models/base.py

@@ -257,8 +257,8 @@ class BaseAPI:
             logging.info(
                 "Load pretrain weights from {}.".format(pretrain_weights),
                 use_color=True)
-            paddlex.utils.utils.load_pretrain_weights(
-                self.exe, self.train_prog, pretrain_weights, fuse_bn)
+            paddlex.utils.utils.load_pretrain_weights(self.exe, self.train_prog,
+                                                      pretrain_weights, fuse_bn)
         # 进行裁剪
         if sensitivities_file is not None:
             import paddleslim
@@ -364,9 +364,7 @@ class BaseAPI:
         logging.info("Model saved in {}.".format(save_dir))
 
     def export_inference_model(self, save_dir):
-        test_input_names = [
-            var.name for var in list(self.test_inputs.values())
-        ]
+        test_input_names = [var.name for var in list(self.test_inputs.values())]
         test_outputs = list(self.test_outputs.values())
         with fluid.scope_guard(self.scope):
             fluid.io.save_inference_model(
@@ -394,8 +392,7 @@ class BaseAPI:
 
         # 模型保存成功的标志
         open(osp.join(save_dir, '.success'), 'w').close()
-        logging.info("Model for inference deploy saved in {}.".format(
-            save_dir))
+        logging.info("Model for inference deploy saved in {}.".format(save_dir))
 
     def train_loop(self,
                    num_epochs,
@@ -480,6 +477,9 @@ class BaseAPI:
         best_accuracy = -1.0
         best_model_epoch = -1
         start_epoch = self.completed_epochs
+        # task_id: 目前由PaddleX GUI赋值
+        # 用于在VisualDL日志中注明所属任务id
+        task_id = getattr(paddlex, "task_id", "")
         for i in range(start_epoch, num_epochs):
             records = list()
             step_start_time = time.time()
@@ -510,8 +510,8 @@ class BaseAPI:
                     if use_vdl:
                         for k, v in step_metrics.items():
                             log_writer.add_scalar(
-                                'Metrics/Training(Step): {}'.format(k), v,
-                                num_steps)
+                                '{}-Metrics/Training(Step): {}'.format(
+                                    task_id, k), v, num_steps)
 
                     # 估算剩余时间
                     avg_step_time = np.mean(time_stat)
@@ -522,13 +522,11 @@ class BaseAPI:
                         eta = ((num_epochs - i) * total_num_steps - step - 1
                                ) * avg_step_time
                     if time_eval_one_epoch is not None:
-                        eval_eta = (
-                            total_eval_times - i // save_interval_epochs
-                        ) * time_eval_one_epoch
+                        eval_eta = (total_eval_times - i // save_interval_epochs
+                                    ) * time_eval_one_epoch
                     else:
-                        eval_eta = (
-                            total_eval_times - i // save_interval_epochs
-                        ) * total_num_steps_eval * avg_step_time
+                        eval_eta = (total_eval_times - i // save_interval_epochs
+                                    ) * total_num_steps_eval * avg_step_time
                     eta_str = seconds_to_hms(eta + eval_eta)
 
                     logging.info(
@@ -577,7 +575,8 @@ class BaseAPI:
                                 if v.size > 1:
                                     continue
                             log_writer.add_scalar(
-                                "Metrics/Eval(Epoch): {}".format(k), v, i + 1)
+                                "{}-Metrics/Eval(Epoch): {}".format(task_id, k),
+                                v, i + 1)
                 self.save_model(save_dir=current_save_dir)
                 if getattr(self, 'use_ema', False):
                     self.exe.run(self.ema.restore_program)

paddlex/cv/models/utils/pretrain_weights.py

@@ -1,6 +1,5 @@
 import paddlex
 import paddlex.utils.logging as logging
-import paddlehub as hub
 import os
 import os.path as osp
 
@@ -156,8 +155,7 @@ def get_pretrain_weights(flag, class_name, backbone, save_dir):
             logging.warning(warning_info.format(class_name, flag, 'IMAGENET'))
             flag = 'IMAGENET'
         elif class_name == 'FastSCNN':
-            logging.warning(
-                warning_info.format(class_name, flag, 'CITYSCAPES'))
+            logging.warning(warning_info.format(class_name, flag, 'CITYSCAPES'))
             flag = 'CITYSCAPES'
     elif flag == 'CITYSCAPES':
         model_name = '{}_{}'.format(class_name, backbone)
@@ -180,8 +178,7 @@ def get_pretrain_weights(flag, class_name, backbone, save_dir):
             logging.warning(warning_info.format(class_name, flag, 'COCO'))
             flag = 'COCO'
         elif class_name == 'FastSCNN':
-            logging.warning(
-                warning_info.format(class_name, flag, 'CITYSCAPES'))
+            logging.warning(warning_info.format(class_name, flag, 'CITYSCAPES'))
             flag = 'CITYSCAPES'
 
     if flag == 'IMAGENET':
@@ -207,6 +204,8 @@ def get_pretrain_weights(flag, class_name, backbone, save_dir):
             fname = osp.split(url)[-1].split('.')[0]
             paddlex.utils.download_and_decompress(url, path=new_save_dir)
             return osp.join(new_save_dir, fname)
+
+        import paddlehub as hub
         try:
             logging.info(
                 "Connecting PaddleHub server to get pretrain weights...")
@@ -246,6 +245,7 @@ def get_pretrain_weights(flag, class_name, backbone, save_dir):
             paddlex.utils.download_and_decompress(url, path=new_save_dir)
             return osp.join(new_save_dir, fname)
 
+        import paddlehub as hub
         try:
             logging.info(
                 "Connecting PaddleHub server to get pretrain weights...")

paddlex/cv/transforms/cls_transforms.py

@@ -68,13 +68,15 @@ class Compose(ClsTransform):
         if isinstance(im, np.ndarray):
             if len(im.shape) != 3:
                 raise Exception(
-                    "im should be 3-dimension, but now is {}-dimensions".
-                    format(len(im.shape)))
+                    "im should be 3-dimension, but now is {}-dimensions".format(
+                        len(im.shape)))
         else:
             try:
+                im_path = im
                 im = cv2.imread(im).astype('float32')
             except:
-                raise TypeError('Can\'t read The image file {}!'.format(im))
+                raise TypeError('Can\'t read The image file {}!'.format(
+                    im_path))
         im = im.astype('float32')
         im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
         for op in self.transforms:
@@ -140,8 +142,8 @@ class RandomCrop(ClsTransform):
             tuple: 当label为空时，返回的tuple为(im, )，对应图像np.ndarray数据；
                    当label不为空时，返回的tuple为(im, label)，分别对应图像np.ndarray数据、图像类别id。
         """
-        im = random_crop(im, self.crop_size, self.lower_scale,
-                         self.lower_ratio, self.upper_ratio)
+        im = random_crop(im, self.crop_size, self.lower_scale, self.lower_ratio,
+                         self.upper_ratio)
         if label is None:
             return (im, )
         else:
@@ -271,14 +273,12 @@ class ResizeByShort(ClsTransform):
         im_short_size = min(im.shape[0], im.shape[1])
         im_long_size = max(im.shape[0], im.shape[1])
         scale = float(self.short_size) / im_short_size
-        if self.max_size > 0 and np.round(scale *
-                                          im_long_size) > self.max_size:
+        if self.max_size > 0 and np.round(scale * im_long_size) > self.max_size:
             scale = float(self.max_size) / float(im_long_size)
         resized_width = int(round(im.shape[1] * scale))
         resized_height = int(round(im.shape[0] * scale))
         im = cv2.resize(
-            im, (resized_width, resized_height),
-            interpolation=cv2.INTER_LINEAR)
+            im, (resized_width, resized_height), interpolation=cv2.INTER_LINEAR)
 
         if label is None:
             return (im, )

paddlex/cv/transforms/seg_transforms.py

@@ -98,9 +98,11 @@ class Compose(SegTransform):
                     format(len(im.shape)))
         else:
             try:
+                im_path = im
                 im = Compose.read_img(im).astype('float32')
             except:
-                raise ValueError('Can\'t read The image file {}!'.format(im))
+                raise ValueError('Can\'t read The image file {}!'.format(
+                    im_path))
         im = im.astype('float32')
         if label is not None:
             if isinstance(label, np.ndarray):
@@ -455,8 +457,7 @@ class ResizeByShort(SegTransform):
         im_short_size = min(im.shape[0], im.shape[1])
         im_long_size = max(im.shape[0], im.shape[1])
         scale = float(self.short_size) / im_short_size
-        if self.max_size > 0 and np.round(scale *
-                                          im_long_size) > self.max_size:
+        if self.max_size > 0 and np.round(scale * im_long_size) > self.max_size:
             scale = float(self.max_size) / float(im_long_size)
         resized_width = int(round(im.shape[1] * scale))
         resized_height = int(round(im.shape[0] * scale))
@@ -732,12 +733,12 @@ class Padding(SegTransform):
             im = np.zeros((im_height + pad_height, im_width + pad_width,
                            im_channel)).astype(orig_im.dtype)
             for i in range(im_channel):
-                im[:, :, i] = np.pad(
-                    orig_im[:, :, i],
-                    pad_width=((0, pad_height), (0, pad_width)),
-                    mode='constant',
-                    constant_values=(self.im_padding_value[i],
-                                     self.im_padding_value[i]))
+                im[:, :, i] = np.pad(orig_im[:, :, i],
+                                     pad_width=((0, pad_height),
+                                                (0, pad_width)),
+                                     mode='constant',
+                                     constant_values=(self.im_padding_value[i],
+                                                      self.im_padding_value[i]))
 
             if label is not None:
                 label = np.pad(label,