support MaskFormer

paddlex/configs/modules/semantic_segmentation/MaskFormer_small.yaml

@@ -0,0 +1,42 @@
+Global:
+  model: MaskFormer_small
+  mode: check_dataset # check_dataset/train/evaluate/predict
+  dataset_dir: "/paddle/dataset/paddlex/seg/seg_optic_examples"
+  device: gpu:0,1,2,3
+  output: "output"
+
+CheckDataset:
+  convert: 
+    enable: False
+    src_dataset_type: null
+  split: 
+    enable: False
+    train_percent: null
+    val_percent: null
+
+Train:
+  epochs_iters: 500
+  num_classes: 2
+  batch_size: 2
+  learning_rate: 0.01
+  pretrain_weight_path: https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/MaskFormer_small_pretrained.pdparams
+  warmup_steps: 0
+  resume_path: null
+  log_interval: 10
+  eval_interval: 100
+  input_shape: [1, 3, 512, 512] # export the model with fixed input shape
+
+Evaluate:
+  weight_path: "output/best_model/model.pdparams"
+  log_interval: 10
+
+Export:
+  weight_path: https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/MaskFormer_small_pretrained.pdparams
+  input_shape: [1, 3, 512, 512]
+
+Predict:
+  batch_size: 1
+  model_dir: "output/best_model/inference"
+  input: "https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/general_semantic_segmentation_001.jpg"
+  kernel_option:
+    run_mode: paddle

paddlex/configs/modules/semantic_segmentation/MaskFormer_tiny.yaml

@@ -0,0 +1,42 @@
+Global:
+  model: MaskFormer_tiny
+  mode: check_dataset # check_dataset/train/evaluate/predict
+  dataset_dir: "/paddle/dataset/paddlex/seg/seg_optic_examples"
+  device: gpu:0,1,2,3
+  output: "output"
+
+CheckDataset:
+  convert: 
+    enable: False
+    src_dataset_type: null
+  split: 
+    enable: False
+    train_percent: null
+    val_percent: null
+
+Train:
+  epochs_iters: 500
+  num_classes: 2
+  batch_size: 2
+  learning_rate: 0.01
+  pretrain_weight_path: https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/MaskFormer_tiny_pretrained.pdparams
+  warmup_steps: 0
+  resume_path: null
+  log_interval: 10
+  eval_interval: 100
+  input_shape: [1, 3, 512, 512] # export the model with fixed input shape
+
+Evaluate:
+  weight_path: "output/best_model/model.pdparams"
+  log_interval: 10
+
+Export:
+  weight_path: https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/MaskFormer_tiny_pretrained.pdparams
+  input_shape: [1, 3, 512, 512]
+
+Predict:
+  batch_size: 1
+  model_dir: "output/best_model/inference"
+  input: "https://paddle-model-ecology.bj.bcebos.com/paddlex/imgs/demo_image/general_semantic_segmentation_001.jpg"
+  kernel_option:
+    run_mode: paddle

paddlex/inference/models_new/semantic_segmentation/predictor.py

@@ -124,8 +124,8 @@ class SegPredictor(BasicPredictor):
         batch_imgs = self.preprocessors["Resize"](
             imgs=batch_raw_imgs, target_size=target_size
         )
-        batch_imgs = self.preprocessors["ToCHW"](imgs=batch_imgs)
         batch_imgs = self.preprocessors["Normalize"](imgs=batch_imgs)
+        batch_imgs = self.preprocessors["ToCHW"](imgs=batch_imgs)
         x = self.preprocessors["ToBatch"](imgs=batch_imgs)
         batch_preds = self.infer(x=x)
         if len(batch_data) > 1:

paddlex/inference/utils/official_models.py

@@ -316,6 +316,8 @@ PP-LCNet_x1_0_vehicle_attribute_infer.tar",
     "PP-TSM-R50_8frames_uniform": "https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0b2/PP-TSM-R50_8frames_uniform_infer.tar",
     "PP-TSMv2-LCNetV2_8frames_uniform": "https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0b2/PP-TSMv2-LCNetV2_8frames_uniform_infer.tar",
     "PP-TSMv2-LCNetV2_16frames_uniform": "https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0b2/PP-TSMv2-LCNetV2_16frames_uniform_infer.tar",
+    "MaskFormer_tiny": "https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0b2/MaskFormer_tiny_infer.tar",
+    "MaskFormer_small": "https://paddle-model-ecology.bj.bcebos.com/paddlex/official_inference_model/paddle3.0b2/MaskFormer_small_infer.tar",
 }
 
 

paddlex/modules/semantic_segmentation/exportor.py

@@ -20,3 +20,12 @@ class SegExportor(BaseExportor):
     """Semantic Segmentation Model Exportor"""
 
     entities = MODELS
+
+    def get_export_kwargs(self):
+        """get key-value arguments of model export function"""
+        kwargs = super().get_export_kwargs()
+
+        input_shape = self.export_config.get("input_shape")
+        if input_shape is not None:
+            kwargs["input_shape"] = input_shape
+        return kwargs

paddlex/modules/semantic_segmentation/model_list.py

@@ -32,4 +32,6 @@ MODELS = [
     "SeaFormer_tiny",
     "SeaFormer_small",
     "SeaFormer_large",
+    "MaskFormer_tiny",
+    "MaskFormer_small",
 ]

paddlex/modules/semantic_segmentation/trainer.py

@@ -68,4 +68,6 @@ class SegTrainer(BaseTrainer):
             train_args["do_eval"] = True
             train_args["save_interval"] = self.train_config.eval_interval
         train_args["dy2st"] = self.train_config.get("dy2st", False)
+        if self.train_config.get("input_shape") is not None:
+            train_args["input_shape"] = self.train_config.input_shape
         return train_args

paddlex/repo_apis/PaddleSeg_api/configs/MaskFormer_small.yaml

@@ -0,0 +1,81 @@
+batch_size: 4
+iters: 160000
+
+train_dataset:
+  type: Dataset
+  dataset_root: data/Cityscapes/
+  train_path: data/Cityscapes/train.txt
+  num_classes: 150
+  transforms: 
+    - type: ResizeByShort
+      short_size: [256, 307, 358, 409, 460, 512, 563, 614, 665, 716, 768, 819, 870, 921, 972, 1024]
+      max_size: 2048
+    - type: RandomPaddingCrop
+      crop_size: [512, 512] 
+    - type: RandomDistort
+      brightness_range: 0.125
+      brightness_prob: 1.0
+      contrast_range: 0.5
+      contrast_prob: 1.0
+      saturation_range: 0.5
+      saturation_prob: 1.0
+      hue_range: 18
+      hue_prob: 1.0
+    - type: RandomHorizontalFlip
+    - type: GenerateInstanceTargets
+      num_classes: 150
+      ignore_index: 255
+    - type: Normalize
+      mean: [0.485, 0.456, 0.406]
+      std: [0.229, 0.224, 0.225]
+  mode: train
+
+val_dataset:
+  type: Dataset
+  dataset_root: datasets/Cityscapes
+  val_path: datasets/Cityscapes/val.txt
+  num_classes: 150
+  transforms:
+    - type: Resize
+      target_size: [512, 512]
+      keep_ratio: False
+      interp: LINEAR
+    - type: Normalize
+      mean: [0.485, 0.456, 0.406]
+      std: [0.229, 0.224, 0.225]
+  mode: val
+
+model:
+  type: MaskFormer
+  num_classes: 150
+  backbone:
+    type: SwinTransformer_small_patch4_window7_224_maskformer
+
+optimizer:
+  type: AdamW
+  weight_decay: 0.01
+  custom_cfg:
+  - name: backbone
+    lr_mult: 1.0
+  - name: norm
+    weight_decay_mult: 0.0
+  - name: relative_position_bias_table
+    weight_decay_mult: 0.0
+  grad_clip_cfg: 
+    name: ClipGradByNorm
+    clip_norm: 0.01
+
+lr_scheduler:
+  type: PolynomialDecay
+  warmup_iters: 1500
+  warmup_start_lr: 6.0e-11
+  learning_rate: 6.0e-05
+  end_lr: 0
+  power: 0.9
+
+loss:
+  types:
+    - type: MaskFormerLoss
+      num_classes: 150
+      eos_coef: 0.1
+  coef: [1]

paddlex/repo_apis/PaddleSeg_api/configs/MaskFormer_tiny.yaml

@@ -0,0 +1,81 @@
+batch_size: 4
+iters: 160000
+
+train_dataset:
+  type: Dataset
+  dataset_root: data/Cityscapes/
+  train_path: data/Cityscapes/train.txt
+  num_classes: 150
+  transforms: 
+    - type: ResizeByShort
+      short_size: [256, 307, 358, 409, 460, 512, 563, 614, 665, 716, 768, 819, 870, 921, 972, 1024]
+      max_size: 2048
+    - type: RandomPaddingCrop
+      crop_size: [512, 512] 
+    - type: RandomDistort
+      brightness_range: 0.125
+      brightness_prob: 1.0
+      contrast_range: 0.5
+      contrast_prob: 1.0
+      saturation_range: 0.5
+      saturation_prob: 1.0
+      hue_range: 18
+      hue_prob: 1.0
+    - type: RandomHorizontalFlip
+    - type: GenerateInstanceTargets
+      num_classes: 150
+      ignore_index: 255
+    - type: Normalize
+      mean: [0.485, 0.456, 0.406]
+      std: [0.229, 0.224, 0.225]
+  mode: train
+
+val_dataset:
+  type: Dataset
+  dataset_root: datasets/Cityscapes
+  val_path: datasets/Cityscapes/val.txt
+  num_classes: 150
+  transforms:
+    - type: Resize
+      target_size: [512, 512]
+      keep_ratio: False
+      interp: LINEAR
+    - type: Normalize
+      mean: [0.485, 0.456, 0.406]
+      std: [0.229, 0.224, 0.225]
+  mode: val
+
+model:
+  type: MaskFormer
+  num_classes: 150
+  backbone:
+    type: SwinTransformer_tiny_patch4_window7_224_maskformer
+
+optimizer:
+  type: AdamW
+  weight_decay: 0.01
+  custom_cfg:
+  - name: backbone
+    lr_mult: 1.0
+  - name: norm
+    weight_decay_mult: 0.0
+  - name: relative_position_bias_table
+    weight_decay_mult: 0.0
+  grad_clip_cfg: 
+    name: ClipGradByNorm
+    clip_norm: 0.01
+
+lr_scheduler:
+  type: PolynomialDecay
+  warmup_iters: 1500
+  warmup_start_lr: 6.0e-11
+  learning_rate: 6.0e-05
+  end_lr: 0
+  power: 0.9
+
+loss:
+  types:
+    - type: MaskFormerLoss
+      num_classes: 150
+      eos_coef: 0.1
+  coef: [1]

paddlex/repo_apis/PaddleSeg_api/seg/config.py

@@ -72,6 +72,15 @@ class SegConfig(BaseSegConfig):
         if "model" in self:
             self.model["num_classes"] = num_classes
 
+        if self.model_name in ["MaskFormer_tiny", "MaskFormer_small"]:
+            for tf_cfg in self.train_dataset["transforms"]:
+                if tf_cfg["type"] == "GenerateInstanceTargets":
+                    tf_cfg["num_classes"] = num_classes
+
+            losses = self.loss["types"]
+            for loss_cfg in losses:
+                loss_cfg["num_classes"] = num_classes
+
     def update_train_crop_size(self, crop_size: Union[int, list]):
         """update the image cropping size of training preprocessing
 

paddlex/repo_apis/PaddleSeg_api/seg/model.py

@@ -126,6 +126,10 @@ class SegModel(BaseModel):
         if log_iters is not None:
             cli_args.append(CLIArgument("--log_iters", log_iters))
 
+        input_shape = kwargs.pop("input_shape", None)
+        if input_shape is not None:
+            cli_args.append(CLIArgument("--input_shape", *input_shape))
+
         # Benchmarking mode settings
         benchmark = kwargs.pop("benchmark", None)
         if benchmark is not None:

paddlex/repo_apis/PaddleSeg_api/seg/register.py

@@ -238,6 +238,26 @@ register_model_info(
 )
 
 
+register_model_info(
+    {
+        "model_name": "MaskFormer_tiny",
+        "suite": "Seg",
+        "config_path": osp.join(PDX_CONFIG_DIR, "MaskFormer_tiny.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+
+register_model_info(
+    {
+        "model_name": "MaskFormer_small",
+        "suite": "Seg",
+        "config_path": osp.join(PDX_CONFIG_DIR, "MaskFormer_small.yaml"),
+        "supported_apis": ["train", "evaluate", "predict", "export"],
+    }
+)
+
+
 # For compatibility
 def _set_alias(model_name, alias):
     from ...base.register import get_registered_model_info