// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "model_deploy/engine/include/openvino_engine.h" namespace PaddleDeploy { bool Model::OpenVinoEngineInit(const OpenVinoEngineConfig& engine_config) { infer_engine_ = std::make_shared(); InferenceConfig config("openvino"); *(config.openvino_config) = engine_config; return infer_engine_->Init(config); } bool OpenVinoEngine::Init(const InferenceConfig& infer_config) { const OpenVinoEngineConfig& engine_config = *(infer_config.openvino_config); InferenceEngine::Core ie; InferenceEngine::CNNNetwork network = ie.ReadNetwork( engine_config.xml_file_, engine_config.bin_file_); inputInfo_ = network.getInputsInfo(); out_maps_ = network.getOutputsInfo(); network.setBatchSize(engine_config.batch_size_); if (engine_config.device_ == "MYRIAD") { std::map networkConfig; networkConfig["VPU_HW_STAGES_OPTIMIZATION"] = "NO"; executable_network_ = ie.LoadNetwork( network, engine_config.device_, networkConfig); } else { executable_network_ = ie.LoadNetwork(network, engine_config.device_); } return true; } bool OpenVinoEngine::Infer(const std::vector &inputs, std::vector *outputs) { InferenceEngine::InferRequest infer_request = executable_network_.CreateInferRequest(); int i = 0; for (const auto & item : inputInfo_) { InferenceEngine::TensorDesc input_tensor; InferenceEngine::Blob::Ptr input_blob = infer_request.GetBlob(item.first); InferenceEngine::MemoryBlob::Ptr input_mem_blob = InferenceEngine::as(input_blob); auto mem_blob_holder = input_mem_blob->wmap(); int size = std::accumulate(inputs[i].shape.begin(), inputs[i].shape.end(), 1, std::multiplies()); if (inputs[i].dtype == 0) { input_tensor.setPrecision(InferenceEngine::Precision::FP32); float *blob_data = mem_blob_holder.as(); memcpy(blob_data, inputs[i].data.data(), size * sizeof(float)); } else if (inputs[i].dtype == 1) { input_tensor.setPrecision(InferenceEngine::Precision::U64); int64_t *blob_data = mem_blob_holder.as(); memcpy(blob_data, inputs[i].data.data(), size * sizeof(int64_t)); } else if (inputs[i].dtype == 2) { input_tensor.setPrecision(InferenceEngine::Precision::I32); int *blob_data = mem_blob_holder.as(); memcpy(blob_data, inputs[i].data.data(), size * sizeof(int)); } else if (inputs[i].dtype == 3) { input_tensor.setPrecision(InferenceEngine::Precision::U8); uint8_t *blob_data = mem_blob_holder.as(); memcpy(blob_data, inputs[i].data.data(), size * sizeof(uint8_t)); infer_request.SetBlob(inputs[i].name, input_blob); } i += 1; } // do inference infer_request.Infer(); for (const auto & output_map : out_maps_) { DataBlob output; std::string name = output_map.first; output.name = name; InferenceEngine::Blob::Ptr output_ptr = infer_request.GetBlob(name); InferenceEngine::MemoryBlob::CPtr moutput = InferenceEngine::as(output_ptr); InferenceEngine::TensorDesc blob_output = moutput->getTensorDesc(); InferenceEngine::SizeVector output_shape = blob_output.getDims(); int size = 1; output.shape.clear(); for (auto& i : output_shape) { size *= i; output.shape.push_back(static_cast(i)); } GetDtype(blob_output, &output); auto moutputHolder = moutput->rmap(); if (output.dtype == 0) { output.data.resize(size * sizeof(float)); float* data = moutputHolder.as(); memcpy(output.data.data(), data, size * sizeof(float)); } else if (output.dtype == 1) { output.data.resize(size * sizeof(int64_t)); int64_t* data = moutputHolder.as(); memcpy(output.data.data(), data, size * sizeof(int64_t)); } else if (output.dtype == 2) { output.data.resize(size * sizeof(int)); int* data = moutputHolder.as(); memcpy(output.data.data(), data, size * sizeof(int)); } else if (output.dtype == 3) { output.data.resize(size * sizeof(uint8_t)); uint8_t* data = moutputHolder.as(); memcpy(output.data.data(), data, size * sizeof(uint8_t)); } outputs->push_back(std::move(output)); } return true; } bool OpenVinoEngine::GetDtype(const InferenceEngine::TensorDesc &output_blob, DataBlob *output) { InferenceEngine::Precision output_precision = output_blob.getPrecision(); if (output_precision == 10) { output->dtype = FLOAT32; } else if (output_precision == 72) { output->dtype = INT64; } else if (output_precision == 70) { output->dtype = INT32; } else if (output_precision == 40) { output->dtype = INT8; } else { std::cout << "can't paser the precision type" << std::endl; return false; } return true; } } // namespace PaddleDeploy