add 3 pipelines serve (#2975)

* add 3 pipelines serve

* fix sth

paddlex/inference/serving/basic_serving/_pipeline_apps/open_vocabulary_detection.py

@@ -0,0 +1,78 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+from fastapi import FastAPI
+
+from ...infra import utils as serving_utils
+from ...infra.config import AppConfig
+from ...infra.models import ResultResponse
+from ...schemas.open_vocabulary_detection import (
+    INFER_ENDPOINT,
+    InferRequest,
+    InferResult,
+)
+from .._app import create_app, primary_operation
+
+
+def create_pipeline_app(pipeline: Any, app_config: AppConfig) -> FastAPI:
+    app, ctx = create_app(
+        pipeline=pipeline, app_config=app_config, app_aiohttp_session=True
+    )
+
+    @primary_operation(
+        app,
+        INFER_ENDPOINT,
+        "infer",
+    )
+    async def _infer(request: InferRequest) -> ResultResponse[InferResult]:
+        pipeline = ctx.pipeline
+        aiohttp_session = ctx.aiohttp_session
+
+        file_bytes = await serving_utils.get_raw_bytes_async(
+            request.image, aiohttp_session
+        )
+        image = serving_utils.image_bytes_to_array(file_bytes)
+
+        result = (
+            await pipeline.infer(
+                image,
+                prompt=request.prompt,
+                thresholds=request.thresholds,
+            )
+        )[0]
+
+        objects: List[Dict[str, Any]] = []
+        for obj in result["boxes"]:
+            objects.append(
+                dict(
+                    bbox=obj["coordinate"],
+                    categoryName=obj["label"],
+                    score=obj["score"],
+                )
+            )
+        if ctx.config.visualize:
+            output_image_base64 = serving_utils.base64_encode(
+                serving_utils.image_to_bytes(result.img["res"])
+            )
+        else:
+            output_image_base64 = None
+
+        return ResultResponse[InferResult](
+            logId=serving_utils.generate_log_id(),
+            result=InferResult(detectedObjects=objects, image=output_image_base64),
+        )
+
+    return app

paddlex/inference/serving/basic_serving/_pipeline_apps/open_vocabulary_segmentation.py

@@ -0,0 +1,85 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+import numpy as np
+import pycocotools.mask as mask_util
+from fastapi import FastAPI
+
+from ...infra import utils as serving_utils
+from ...infra.config import AppConfig
+from ...infra.models import ResultResponse
+from ...schemas.open_vocabulary_segmentation import (
+    INFER_ENDPOINT,
+    InferRequest,
+    InferResult,
+)
+from .._app import create_app, primary_operation
+
+
+def _rle(mask: np.ndarray) -> str:
+    rle_res = mask_util.encode(np.asarray(mask[..., None], order="F", dtype="uint8"))[0]
+    return rle_res["counts"].decode("utf-8")
+
+
+def create_pipeline_app(pipeline: Any, app_config: AppConfig) -> FastAPI:
+    app, ctx = create_app(
+        pipeline=pipeline,
+        app_config=app_config,
+        app_aiohttp_session=True,
+    )
+
+    @primary_operation(
+        app,
+        INFER_ENDPOINT,
+        "infer",
+    )
+    async def _infer(request: InferRequest) -> ResultResponse[InferResult]:
+        pipeline = ctx.pipeline
+        aiohttp_session = ctx.aiohttp_session
+
+        file_bytes = await serving_utils.get_raw_bytes_async(
+            request.image, aiohttp_session
+        )
+        image = serving_utils.image_bytes_to_array(file_bytes)
+
+        result = (
+            await pipeline.infer(
+                image,
+                prompt=request.prompt,
+                prompt_type=request.promptType,
+            )
+        )[0]
+
+        rle_masks = [
+            dict(rleResult=_rle(mask), size=mask.shape) for mask in result["masks"]
+        ]
+        mask_infos = result["mask_infos"]
+
+        if ctx.config.visualize:
+            output_image_base64 = serving_utils.base64_encode(
+                serving_utils.image_to_bytes(result.img["res"])
+            )
+        else:
+            output_image_base64 = None
+
+        return ResultResponse[InferResult](
+            logId=serving_utils.generate_log_id(),
+            result=InferResult(
+                masks=rle_masks, maskInfos=mask_infos, image=output_image_base64
+            ),
+        )
+
+    return app

paddlex/inference/serving/basic_serving/_pipeline_apps/rotated_object_detection.py

@@ -0,0 +1,78 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+from fastapi import FastAPI
+
+from ...infra import utils as serving_utils
+from ...infra.config import AppConfig
+from ...infra.models import ResultResponse
+from ...schemas.rotated_object_detection import (
+    INFER_ENDPOINT,
+    InferRequest,
+    InferResult,
+)
+from .._app import create_app, primary_operation
+
+
+def create_pipeline_app(pipeline: Any, app_config: AppConfig) -> FastAPI:
+    app, ctx = create_app(
+        pipeline=pipeline, app_config=app_config, app_aiohttp_session=True
+    )
+
+    @primary_operation(
+        app,
+        INFER_ENDPOINT,
+        "infer",
+    )
+    async def _infer(request: InferRequest) -> ResultResponse[InferResult]:
+        pipeline = ctx.pipeline
+        aiohttp_session = ctx.aiohttp_session
+
+        file_bytes = await serving_utils.get_raw_bytes_async(
+            request.image, aiohttp_session
+        )
+        image = serving_utils.image_bytes_to_array(file_bytes)
+
+        result = (
+            await pipeline.infer(
+                image,
+                threshold=request.threshold,
+            )
+        )[0]
+
+        objects: List[Dict[str, Any]] = []
+        for obj in result["boxes"]:
+            objects.append(
+                dict(
+                    bbox=obj["coordinate"],
+                    categoryId=obj["cls_id"],
+                    categoryName=obj["label"],
+                    score=obj["score"],
+                )
+            )
+        if ctx.config.visualize:
+            output_image_base64 = serving_utils.base64_encode(
+                serving_utils.image_to_bytes(result.img["res"])
+            )
+        else:
+            output_image_base64 = None
+
+        return ResultResponse[InferResult](
+            logId=serving_utils.generate_log_id(),
+            result=InferResult(detectedObjects=objects, image=output_image_base64),
+        )
+
+    return app

paddlex/inference/serving/schemas/instance_segmentation.py

@@ -22,7 +22,6 @@ from .shared import image_segmentation, object_detection
 __all__ = [
     "INFER_ENDPOINT",
     "InferRequest",
-    "Mask",
     "Instance",
     "InferResult",
     "PRIMARY_OPERATIONS",
@@ -36,17 +35,12 @@ class InferRequest(BaseModel):
     threshold: Optional[float] = None
 
 
-class Mask(BaseModel):
-    rleResult: str
-    size: image_segmentation.Size
-
-
 class Instance(BaseModel):
     bbox: object_detection.BoundingBox
     categoryId: int
     categoryName: str
     score: float
-    mask: Mask
+    mask: image_segmentation.Mask
 
 
 class InferResult(BaseModel):

paddlex/inference/serving/schemas/open_vocabulary_detection.py

@@ -0,0 +1,52 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Final, List, Optional, Union
+
+from pydantic import BaseModel
+
+from ..infra.models import PrimaryOperations
+from .shared import object_detection
+
+__all__ = [
+    "INFER_ENDPOINT",
+    "InferRequest",
+    "DetectedObject",
+    "InferResult",
+    "PRIMARY_OPERATIONS",
+]
+
+INFER_ENDPOINT: Final[str] = "/open-vocabulary-detection"
+
+
+class InferRequest(BaseModel):
+    image: str
+    prompt: str
+    thresholds: Optional[Dict[str, float]] = None
+
+
+class DetectedObject(BaseModel):
+    bbox: object_detection.BoundingBox
+    categoryName: str
+    score: float
+
+
+class InferResult(BaseModel):
+    detectedObjects: List[DetectedObject]
+    image: Optional[str] = None
+
+
+PRIMARY_OPERATIONS: Final[PrimaryOperations] = {
+    "infer": (INFER_ENDPOINT, InferRequest, InferResult),
+}

paddlex/inference/serving/schemas/open_vocabulary_segmentation.py

@@ -0,0 +1,52 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Final, List, Optional
+
+from pydantic import BaseModel
+
+from ..infra.models import PrimaryOperations
+from .shared import image_segmentation
+
+__all__ = [
+    "INFER_ENDPOINT",
+    "InferRequest",
+    "MaskInfo",
+    "InferResult",
+    "PRIMARY_OPERATIONS",
+]
+
+INFER_ENDPOINT: Final[str] = "/open-vocabulary-segmentation"
+
+
+class InferRequest(BaseModel):
+    image: str
+    prompt: List[List[float]]
+    promptType: str
+
+
+class MaskInfo(BaseModel):
+    label: str
+    prompt: List[float]
+
+
+class InferResult(BaseModel):
+    masks: List[image_segmentation.Mask]
+    maskInfos: List[MaskInfo]
+    image: Optional[str] = None
+
+
+PRIMARY_OPERATIONS: Final[PrimaryOperations] = {
+    "infer": (INFER_ENDPOINT, InferRequest, InferResult),
+}

paddlex/inference/serving/schemas/rotated_object_detection.py

@@ -0,0 +1,52 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Final, List, Optional, Union
+
+from pydantic import BaseModel
+
+from ..infra.models import PrimaryOperations
+from .shared import object_detection
+
+__all__ = [
+    "INFER_ENDPOINT",
+    "InferRequest",
+    "DetectedObject",
+    "InferResult",
+    "PRIMARY_OPERATIONS",
+]
+
+INFER_ENDPOINT: Final[str] = "/rotated-object-detection"
+
+
+class InferRequest(BaseModel):
+    image: str
+    threshold: Optional[Union[float, Dict[int, float]]] = None
+
+
+class DetectedObject(BaseModel):
+    bbox: object_detection.RotatedBoundingBox
+    categoryId: int
+    categoryName: str
+    score: float
+
+
+class InferResult(BaseModel):
+    detectedObjects: List[DetectedObject]
+    image: Optional[str] = None
+
+
+PRIMARY_OPERATIONS: Final[PrimaryOperations] = {
+    "infer": (INFER_ENDPOINT, InferRequest, InferResult),
+}

paddlex/inference/serving/schemas/shared/image_segmentation.py

@@ -14,10 +14,15 @@
 
 from typing import List
 
-from pydantic import Field
+from pydantic import Field, BaseModel
 from typing_extensions import Annotated, TypeAlias
 
-__all__ = ["Size"]
+__all__ = ["Size", "Mask"]
 
 
 Size: TypeAlias = Annotated[List[int], Field(min_length=2, max_length=2)]
+
+
+class Mask(BaseModel):
+    rleResult: str
+    size: Size

paddlex/inference/serving/schemas/shared/object_detection.py

@@ -17,6 +17,9 @@ from typing import List
 from pydantic import Field
 from typing_extensions import Annotated, TypeAlias
 
-__all__ = ["BoundingBox"]
+__all__ = ["BoundingBox", "RotatedBoundingBox"]
 
 BoundingBox: TypeAlias = Annotated[List[float], Field(min_length=4, max_length=4)]
+RotatedBoundingBox: TypeAlias = Annotated[
+    List[float], Field(min_length=8, max_length=8)
+]