add 3d bevfusion pipeline (#2986)

* add 3d pipeline

* support predict from .tar

api_examples/pipelines/test_3d_bev_detection.py

@@ -0,0 +1,25 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddlex import create_pipeline
+
+pipeline = create_pipeline(pipeline="3d_bev_detection")
+output = pipeline.predict(
+    "https://paddle-model-ecology.bj.bcebos.com/paddlex/det_3d/demo_det_3d/nuscenes_demo_infer.tar"
+)
+
+for res in output:
+    print(res)
+    res.print()  ## 打印预测的结构化输出
+    res.save_to_json("./output/res.json")  ## 保存结果到json文件

docs/module_usage/tutorials/cv_modules/3d_bev_detection.md

@@ -349,12 +349,12 @@ python main.py -c paddlex/configs/modules/3d_bev_detection/BEVFusion.yaml \
 
 #### 4.4.1 模型推理
 
-* 通过命令行的方式进行推理预测，只需如下一条命令。运行以下代码前，请您下载[示例输入](https://paddle-model-ecology.bj.bcebos.com/paddlex/det_3d/demo_det_3d/nuscenes_infos_val.pkl)到本地。
+* 通过命令行的方式进行推理预测，只需如下一条命令。运行以下代码前，请您下载[示例数据] (https://paddle-model-ecology.bj.bcebos.com/paddlex/det_3d/demo_det_3d/nuscenes_demo_infer.tar)到本地。
 ```bash
 python main.py -c paddlex/configs/modules/3d_bev_detection/BEVFusion.yaml  \
     -o Global.mode=predict \
     -o Predict.model_dir="./output/best_model/inference" \
-    -o Predict.input="nuscenes_infos_val.pkl"
+    -o Predict.input="nuscenes_demo_infer.tar"
 ```
 与模型训练和评估类似，需要如下几步：
 

paddlex/configs/modules/3d_bev_detection/BEVFusion.yaml

@@ -33,6 +33,6 @@ Export:
 Predict:
   batch_size: 1
   model_dir: "output/best_model/inference"
-  input: "https://paddle-model-ecology.bj.bcebos.com/paddlex/det_3d/demo_det_3d/nuscenes_infos_val.pkl"
+  input: "https://paddle-model-ecology.bj.bcebos.com/paddlex/det_3d/demo_det_3d/nuscenes_demo_infer.tar"
   kernel_option:
     run_mode: paddle

paddlex/configs/pipelines/3d_bev_detection.yaml

@@ -0,0 +1,9 @@
+
+pipeline_name: 3d_bev_detection
+
+SubModules:
+  3DBEVDetection:
+    module_name: 3d_bev_detection
+    model_name: BEVFusion
+    model_dir: null
+    batch_size: 1

paddlex/inference/common/batch_sampler/det_3d_batch_sampler.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Any, Dict, List, Optional, Union
 import os
 import ast
 from pathlib import Path
 import numpy as np
 import pickle
-from typing import Any, Dict, List, Optional, Union
+import tarfile
+
 
 from ....utils import logging
 from ....utils.download import download
@@ -52,17 +54,37 @@ class Det3DBatchSampler(BaseBatchSampler):
             )
         self._batch_size = batch_size
 
-    def load_annotations(self, ann_file: str) -> List[Dict]:
+    def load_annotations(self, ann_file: str, data_root_dir: str) -> List[Dict]:
         """Load annotations from ann_file.
 
         Args:
             ann_file (str): Path of the annotation file.
+            data_root_dir: (str): Path of the data root directory.
 
         Returns:
             list[dict]: List of annotations sorted by timestamps.
         """
         data = pickle.load(open(ann_file, "rb"))
         data_infos = list(sorted(data["infos"], key=lambda e: e["timestamp"]))
+        # append root_dir to image and lidar filepaths
+        for item in data_infos:
+            # lidar data
+            lidar_path = item["lidar_path"]
+            new_lidar_path = os.path.join(data_root_dir, lidar_path)
+            item["lidar_path"] = new_lidar_path
+            # camera data
+            cam_data = item["cams"]
+            for cam_data_item_key in cam_data:
+                cam_data_item = cam_data[cam_data_item_key]
+                cam_data_item_path = cam_data_item["data_path"]
+                new_cam_data_item_path = os.path.join(data_root_dir, cam_data_item_path)
+                cam_data_item["data_path"] = new_cam_data_item_path
+            # sweep data
+            sweeps = item["sweeps"]
+            for sweep_item in sweeps:
+                sweep_item_path = sweep_item["data_path"]
+                new_sweep_item_path = os.path.join(data_root_dir, sweep_item_path)
+                sweep_item["data_path"] = new_sweep_item_path
         return data_infos
 
     def sample(self, inputs: Union[List[str], str]):
@@ -81,7 +103,12 @@ class Det3DBatchSampler(BaseBatchSampler):
                 logging.warning(
                     f"Not supported input data type! Only `str` is supported! So has been ignored: {input}."
                 )
-            self.data_infos = self.load_annotations(ann_path)
+            # extract tar file
+            tar_root_dir = os.path.dirname(ann_path)
+            self.extract_tar(ann_path, tar_root_dir)
+            data_root_dir, _ = os.path.splitext(ann_path)
+            ann_pkl_path = os.path.join(data_root_dir, "nuscenes_infos_val.pkl")
+            self.data_infos = self.load_annotations(ann_pkl_path, data_root_dir)
             sample_set.extend(self.data_infos)
 
         batch = []
@@ -98,3 +125,11 @@ class Det3DBatchSampler(BaseBatchSampler):
         raise NotImplementedError(
             "rand batch is not supported for 3D detection annotation data"
         )
+
+    def extract_tar(self, tar_path, extract_path="."):
+        try:
+            with tarfile.open(tar_path, "r") as tar:
+                tar.extractall(path=extract_path)
+                print(f"file extract to {extract_path}")
+        except Exception as e:
+            print(f"error occurred while extracting tar file: {e}")

paddlex/inference/pipelines/3d_bev_detection/__init__.py

@@ -0,0 +1,15 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .pipeline import BEVDet3DPipeline

paddlex/inference/pipelines/3d_bev_detection/pipeline.py

@@ -0,0 +1,67 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, Optional, Union, List
+import numpy as np
+from importlib import import_module
+from ...utils.pp_option import PaddlePredictorOption
+from ..base import BasePipeline
+
+module_3d_bev_detection_result = import_module(
+    ".result", "paddlex.inference.models.3d_bev_detection"
+)
+BEV3DDetResult = getattr(module_3d_bev_detection_result, "BEV3DDetResult")
+
+
+class BEVDet3DPipeline(BasePipeline):
+    """3D Detection Pipeline"""
+
+    entities = "3d_bev_detection"
+
+    def __init__(
+        self,
+        config: Dict,
+        device: str = None,
+        pp_option: PaddlePredictorOption = None,
+        use_hpip: bool = False,
+    ) -> None:
+        """
+        Initializes the class with given configurations and options.
+
+        Args:
+            config (Dict): Configuration dictionary containing model and other parameters.
+            device (str): The device to run the prediction on. Default is None.
+            pp_option (PaddlePredictorOption): Options for PaddlePaddle predictor. Default is None.
+            use_hpip (bool): Whether to use high-performance inference (hpip) for prediction. Defaults to False.
+        """
+        super().__init__(device=device, pp_option=pp_option, use_hpip=use_hpip)
+
+        bev_detection_3d_model_config = config["SubModules"]["3DBEVDetection"]
+        self.bev_detection_3d_model = self.create_model(bev_detection_3d_model_config)
+
+    def predict(
+        self,
+        input: Union[str, List[str], np.ndarray, List[np.ndarray]],
+        **kwargs,
+    ) -> BEV3DDetResult:
+        """Predicts 3D detection results for the given input.
+
+        Args:
+            input (str | list[str] | np.ndarray | list[np.ndarray]): The input path(s) to the 3d annotation pickle file.
+            **kwargs: Additional keyword arguments that can be passed to the function.
+
+        Returns:
+            BEV3DDetResult: The predicted 3d detection results.
+        """
+        yield from self.bev_detection_3d_model(input)

paddlex/inference/pipelines/__init__.py

@@ -14,6 +14,7 @@
 
 from pathlib import Path
 from typing import Any, Dict, Optional
+from importlib import import_module
 from .base import BasePipeline
 from ..utils.pp_option import PaddlePredictorOption
 from .components import BaseChat, BaseRetriever, BaseGeneratePrompt
@@ -52,6 +53,11 @@ from .keypoint_detection import KeypointDetectionPipeline
 from .open_vocabulary_detection import OpenVocabularyDetectionPipeline
 from .open_vocabulary_segmentation import OpenVocabularySegmentationPipeline
 
+module_3d_bev_detection = import_module(
+    ".3d_bev_detection", "paddlex.inference.pipelines"
+)
+BEVDet3DPipeline = getattr(module_3d_bev_detection, "BEVDet3DPipeline")
+
 
 def get_pipeline_path(pipeline_name: str) -> str:
     """

paddlex/utils/pipeline_arguments.py

@@ -335,4 +335,5 @@ PIPELINE_ARGUMENTS = {
             "help": "Sets the prompt for open vocabulary segmentation.",
         },
     ],
+    "3d_bev_detection": None,
 }