// // Copyright © 2017 Arm Ltd. All rights reserved. // SPDX-License-Identifier: MIT // #define LOG_TAG "ArmnnDriver" #include "ArmnnPreparedModel_1_2.hpp" #include "Utils.hpp" #include #include #include #include #include #include using namespace android; using namespace android::hardware; namespace { static const V1_2::Timing g_NoTiming = {.timeOnDevice = UINT64_MAX, .timeInDriver = UINT64_MAX}; using namespace armnn_driver; using TimePoint = std::chrono::steady_clock::time_point; TimePoint Now() { return std::chrono::steady_clock::now(); } unsigned long MicrosecondsDuration(TimePoint endPoint, TimePoint startPoint) { return static_cast(std::chrono::duration_cast( endPoint - startPoint).count()); } void NotifyCallbackAndCheck(const ::android::sp& callback, V1_0::ErrorStatus errorStatus, std::vector, const V1_2::Timing, std::string callingFunction) { Return returned = callback->notify(errorStatus); // This check is required, if the callback fails and it isn't checked it will bring down the service if (!returned.isOk()) { ALOGE("ArmnnDriver::%s: hidl callback failed to return properly: %s", callingFunction.c_str(), returned.description().c_str()); } } void NotifyCallbackAndCheck(const ::android::sp& callback, V1_0::ErrorStatus errorStatus, std::vector outputShapes, const V1_2::Timing timing, std::string callingFunction) { Return returned = callback->notify_1_2(errorStatus, outputShapes, timing); // This check is required, if the callback fails and it isn't checked it will bring down the service if (!returned.isOk()) { ALOGE("ArmnnDriver::%s: hidl callback failed to return properly: %s", callingFunction.c_str(), returned.description().c_str()); } } bool ValidateRequestArgument(const V1_0::RequestArgument& requestArg, const armnn::TensorInfo& tensorInfo) { if (requestArg.dimensions.size() != 0) { if (requestArg.dimensions.size() != tensorInfo.GetNumDimensions()) { ALOGE("Mismatched dimensions (request argument: %zu, expected: %u)", requestArg.dimensions.size(), tensorInfo.GetNumDimensions()); return false; } for (unsigned int d = 0; d < tensorInfo.GetNumDimensions(); ++d) { if (requestArg.dimensions[d] != 0 && requestArg.dimensions[d] != tensorInfo.GetShape()[d]) { ALOGE("Mismatched size for dimension %d (request argument: %u, expected %u)", d, requestArg.dimensions[d], tensorInfo.GetShape()[d]); return false; } } } return true; } armnn::Tensor GetTensorForRequestArgument(const V1_0::RequestArgument& requestArg, const armnn::TensorInfo& tensorInfo, const std::vector<::android::nn::RunTimePoolInfo>& requestPools) { if (!ValidateRequestArgument(requestArg, tensorInfo)) { return armnn::Tensor(); } return armnn::Tensor(tensorInfo, GetMemoryFromPool(requestArg.location, requestPools)); } inline std::string BuildTensorName(const char* tensorNamePrefix, std::size_t index) { return tensorNamePrefix + std::to_string(index); } } // anonymous namespace using namespace android::hardware; namespace armnn_driver { template RequestThread ArmnnPreparedModel_1_2::m_RequestThread; template template void ArmnnPreparedModel_1_2::DumpTensorsIfRequired(char const* tensorNamePrefix, const TensorBindingCollection& tensorBindings) { if (!m_RequestInputsAndOutputsDumpDir.empty()) { const std::string requestName = std::to_string(m_NetworkId) + "_" + std::to_string(m_RequestCount) + ".dump"; for (std::size_t i = 0u; i < tensorBindings.size(); ++i) { DumpTensor(m_RequestInputsAndOutputsDumpDir, requestName, BuildTensorName(tensorNamePrefix, i), tensorBindings[i].second); } } } template ArmnnPreparedModel_1_2::ArmnnPreparedModel_1_2(armnn::NetworkId networkId, armnn::IRuntime* runtime, const V1_2::Model& model, const std::string& requestInputsAndOutputsDumpDir, const bool gpuProfilingEnabled) : m_NetworkId(networkId) , m_Runtime(runtime) , m_Model(model) , m_RequestCount(0) , m_RequestInputsAndOutputsDumpDir(requestInputsAndOutputsDumpDir) , m_GpuProfilingEnabled(gpuProfilingEnabled) { // Enable profiling if required. m_Runtime->GetProfiler(m_NetworkId)->EnableProfiling(m_GpuProfilingEnabled); } template ArmnnPreparedModel_1_2::~ArmnnPreparedModel_1_2() { // Get a hold of the profiler used by this model. std::shared_ptr profiler = m_Runtime->GetProfiler(m_NetworkId); // Unload the network associated with this model. m_Runtime->UnloadNetwork(m_NetworkId); // Dump the profiling info to a file if required. DumpJsonProfilingIfRequired(m_GpuProfilingEnabled, m_RequestInputsAndOutputsDumpDir, m_NetworkId, profiler.get()); } template Return ArmnnPreparedModel_1_2::execute(const V1_0::Request& request, const ::android::sp& callback) { if (callback.get() == nullptr) { ALOGE("ArmnnPreparedModel_1_2::execute invalid callback passed"); return V1_0::ErrorStatus::INVALID_ARGUMENT; } auto cb = [callback](V1_0::ErrorStatus errorStatus, std::vector outputShapes, const V1_2::Timing& timing, std::string callingFunction) { NotifyCallbackAndCheck(callback, errorStatus, outputShapes, timing, callingFunction); }; return Execute(request, V1_2::MeasureTiming::NO, cb); } template Return ArmnnPreparedModel_1_2::execute_1_2( const V1_0::Request& request, V1_2::MeasureTiming measureTiming, const sp& callback) { if (callback.get() == nullptr) { ALOGE("ArmnnPreparedModel_1_2::execute_1_2 invalid callback passed"); return V1_0::ErrorStatus::INVALID_ARGUMENT; } auto cb = [callback](V1_0::ErrorStatus errorStatus, std::vector outputShapes, const V1_2::Timing& timing, std::string callingFunction) { NotifyCallbackAndCheck(callback, errorStatus, outputShapes, timing, callingFunction); }; return Execute(request, measureTiming, cb); } template Return ArmnnPreparedModel_1_2::PrepareMemoryForInputs( armnn::InputTensors& inputs, const V1_0::Request& request, const std::vector& memPools) { inputs.reserve(request.inputs.size()); for (unsigned int i = 0; i < request.inputs.size(); i++) { const auto& inputArg = request.inputs[i]; const armnn::TensorInfo inputTensorInfo = m_Runtime->GetInputTensorInfo(m_NetworkId, i); const armnn::Tensor inputTensor = GetTensorForRequestArgument(inputArg, inputTensorInfo, memPools); uint32_t poolIndex = inputArg.location.poolIndex; if (poolIndex >= memPools.size()) { ALOGE("Cannot execute request. Error converting request input %u to tensor: wrong poolIndex", i); return V1_0::ErrorStatus::GENERAL_FAILURE; } uint8_t* inputTensorBegin = static_cast(inputTensor.GetMemoryArea()); if (inputTensorBegin == nullptr) { ALOGE("Cannot execute request. Error converting request input %u to tensor", i); return V1_0::ErrorStatus::GENERAL_FAILURE; } const size_t inputTensorSize = inputTensorInfo.GetNumBytes(); uint8_t* memoryPoolBegin = memPools[poolIndex].getBuffer(); uint32_t memoryPoolSize = memPools[poolIndex].getSize(); bool inputTensorIsOutOfMemoryRage = (inputTensorBegin + inputTensorSize) > (memoryPoolBegin + memoryPoolSize); if (inputTensorIsOutOfMemoryRage) { ALOGE("Cannot execute request. Error converting request input %u to tensor: out of Memory Pool", i); return V1_0::ErrorStatus::GENERAL_FAILURE; } inputs.emplace_back(i, inputTensor); } return V1_0::ErrorStatus::NONE; } template Return ArmnnPreparedModel_1_2::PrepareMemoryForOutputs( armnn::OutputTensors& outputs, std::vector &outputShapes, const V1_0::Request& request, const std::vector& memPools) { outputs.reserve(request.outputs.size()); for (unsigned int i = 0; i < request.outputs.size(); i++) { const auto& outputArg = request.outputs[i]; const armnn::TensorInfo outputTensorInfo = m_Runtime->GetOutputTensorInfo(m_NetworkId, i); const armnn::Tensor outputTensor = GetTensorForRequestArgument(outputArg, outputTensorInfo, memPools); uint8_t* outputTensorBegin = static_cast(outputTensor.GetMemoryArea()); if (outputTensorBegin == nullptr) { ALOGE("Cannot execute request. Error converting request output %u to tensor", i); return V1_0::ErrorStatus::GENERAL_FAILURE; } const size_t outputSize = outputTensorInfo.GetNumBytes(); if (outputArg.location.length < outputSize) { ALOGW("ArmnnPreparedModel_1_2::Execute failed: outputArg.location.length < outputSize"); return V1_0::ErrorStatus::OUTPUT_INSUFFICIENT_SIZE; } const size_t bufferSize = memPools.at(outputArg.location.poolIndex).getSize(); if (bufferSize < outputSize) { ALOGW("ArmnnPreparedModel_1_2::Execute failed: bufferSize < outputSize"); return V1_0::ErrorStatus::OUTPUT_INSUFFICIENT_SIZE; } uint32_t poolIndex = outputArg.location.poolIndex; if (poolIndex >= memPools.size()) { ALOGE("Cannot execute request. Error converting request output %u to tensor: wrong poolIndex", i); return V1_0::ErrorStatus::GENERAL_FAILURE; } uint8_t* memoryPoolBegin = memPools[poolIndex].getBuffer(); uint32_t memoryPoolSize = memPools[poolIndex].getSize(); bool outputTensorIsOutOfMemoryRage = (outputTensorBegin + outputSize) > (memoryPoolBegin + memoryPoolSize); if (outputTensorIsOutOfMemoryRage) { ALOGE("Cannot execute request. Error converting request output %u to tensor: out of Memory Pool", i); return V1_0::ErrorStatus::GENERAL_FAILURE; } outputs.emplace_back(i, outputTensor); outputShapes[i] = ComputeShape(outputTensorInfo); } return V1_0::ErrorStatus::NONE; } template Return ArmnnPreparedModel_1_2::PrepareMemoryForIO( armnn::InputTensors& inputs, armnn::OutputTensors& outputs, std::vector& memPools, const V1_0::Request& request, CallbackAsync_1_2 callback) { if (!setRunTimePoolInfosFromHidlMemories(&memPools, request.pools)) { callback(V1_0::ErrorStatus::GENERAL_FAILURE, {}, g_NoTiming, "ArmnnPreparedModel_1_2::execute"); return V1_0::ErrorStatus::GENERAL_FAILURE; } // add the inputs and outputs with their data try { if (PrepareMemoryForInputs(inputs, request, memPools) != V1_0::ErrorStatus::NONE) { callback(V1_0::ErrorStatus::GENERAL_FAILURE, {}, g_NoTiming, "ArmnnPreparedModel_1_2::execute"); return V1_0::ErrorStatus::GENERAL_FAILURE; } std::vector outputShapes(request.outputs.size()); auto errorStatus = PrepareMemoryForOutputs(outputs, outputShapes, request, memPools); if (errorStatus != V1_0::ErrorStatus::NONE) { callback(errorStatus, outputShapes, g_NoTiming, "ArmnnPreparedModel_1_2::Execute"); return errorStatus; } } catch (armnn::Exception& e) { ALOGW("armnn::Exception caught while preparing for EnqueueWorkload: %s", e.what()); callback(V1_0::ErrorStatus::GENERAL_FAILURE, {}, g_NoTiming, "ArmnnPreparedModel_1_2::execute"); return V1_0::ErrorStatus::GENERAL_FAILURE; } catch (std::exception& e) { ALOGE("std::exception caught while preparing for EnqueueWorkload: %s", e.what()); callback(V1_0::ErrorStatus::GENERAL_FAILURE, {}, g_NoTiming, "ArmnnPreparedModel_1_2::execute"); return V1_0::ErrorStatus::GENERAL_FAILURE; } return V1_0::ErrorStatus::NONE; } template Return ArmnnPreparedModel_1_2::executeSynchronously(const V1_0::Request& request, V1_2::MeasureTiming measureTiming, V1_2::IPreparedModel::executeSynchronously_cb cb) { ALOGV("ArmnnPreparedModel_1_2::executeSynchronously(): %s", GetModelSummary(m_Model).c_str()); m_RequestCount++; if (cb == nullptr) { ALOGE("ArmnnPreparedModel_1_2::executeSynchronously invalid callback passed"); return Void(); } TimePoint driverStart; if (measureTiming == V1_2::MeasureTiming::YES) { driverStart = Now(); } if (!android::nn::validateRequest(request, m_Model)) { ALOGE("ArmnnPreparedModel_1_2::executeSynchronously invalid request model"); cb(V1_0::ErrorStatus::INVALID_ARGUMENT, {}, g_NoTiming); return Void(); } auto cbWrapper = [cb](V1_0::ErrorStatus errorStatus, std::vector outputShapes, const V1_2::Timing& timing, std::string) { cb(errorStatus, outputShapes, timing); }; // map the memory pool into shared pointers // use a shared memory pools vector on the heap, as it is passed to the request thread auto memPools = std::make_shared>(); // allocate the tensors on the heap, as they are passed to the request thread auto inputs = std::make_shared(); auto outputs = std::make_shared(); auto prepareStatus = PrepareMemoryForIO(*inputs, *outputs, *memPools, request, cbWrapper); if (prepareStatus != V1_0::ErrorStatus::NONE) { return Void(); } ALOGV("ArmnnPreparedModel_1_2::executeSynchronously() before Execution"); CallbackContext_1_2 cbCtx; cbCtx.callback = cbWrapper; cbCtx.ctx.measureTimings = measureTiming; cbCtx.ctx.driverStart = driverStart; ExecuteGraph(memPools, *inputs, *outputs, cbCtx); return Void(); } template template bool ArmnnPreparedModel_1_2::ExecuteGraph( std::shared_ptr>& pMemPools, armnn::InputTensors& inputTensors, armnn::OutputTensors& outputTensors, CallbackContext cb) { ALOGV("ArmnnPreparedModel_1_2::ExecuteGraph(...)"); TimePoint driverEnd, deviceStart, deviceEnd; DumpTensorsIfRequired("Input", inputTensors); std::vector outputShapes(outputTensors.size()); for (unsigned int i = 0; i < outputTensors.size(); i++) { std::pair outputTensorPair = outputTensors[i]; const armnn::Tensor outputTensor = outputTensorPair.second; const armnn::TensorInfo outputTensorInfo = outputTensor.GetInfo(); outputShapes[i] = ComputeShape(outputTensorInfo); } // run it try { if (cb.ctx.measureTimings == V1_2::MeasureTiming::YES) { deviceStart = Now(); } armnn::Status status = m_Runtime->EnqueueWorkload(m_NetworkId, inputTensors, outputTensors); if (cb.ctx.measureTimings == V1_2::MeasureTiming::YES) { deviceEnd = Now(); } if (status != armnn::Status::Success) { ALOGW("EnqueueWorkload failed"); cb.callback(V1_0::ErrorStatus::GENERAL_FAILURE, {}, g_NoTiming, "ArmnnPreparedModel_1_2::ExecuteGraph"); return false; } } catch (armnn::Exception& e) { ALOGW("armnn:Exception caught from EnqueueWorkload: %s", e.what()); cb.callback(V1_0::ErrorStatus::GENERAL_FAILURE, {}, g_NoTiming, "ArmnnPreparedModel_1_2::ExecuteGraph"); return false; } catch (std::exception& e) { ALOGE("std::exception caught from EnqueueWorkload: %s", e.what()); cb.callback(V1_0::ErrorStatus::GENERAL_FAILURE, {}, g_NoTiming, "ArmnnPreparedModel_1_2::ExecuteGraph"); return false; } CommitPools(*pMemPools); DumpTensorsIfRequired("Output", outputTensors); if (cb.ctx.measureTimings == V1_2::MeasureTiming::YES) { driverEnd = Now(); V1_2::Timing timing; timing.timeOnDevice = MicrosecondsDuration(deviceEnd, deviceStart); timing.timeInDriver = MicrosecondsDuration(driverEnd, cb.ctx.driverStart); ALOGV("ArmnnPreparedModel_1_2::execute timing - Device = %" PRIu64 " Driver = %" PRIu64, timing.timeOnDevice, timing.timeInDriver); cb.callback(V1_0::ErrorStatus::NONE, outputShapes, timing, "ArmnnPreparedModel_1_2::ExecuteGraph"); } else { cb.callback(V1_0::ErrorStatus::NONE, outputShapes, g_NoTiming, "ArmnnPreparedModel_1_2::ExecuteGraph"); } return true; } template bool ArmnnPreparedModel_1_2::ExecuteWithDummyInputs() { std::vector> storage; armnn::InputTensors inputTensors; for (unsigned int i = 0; i < getMainModel(m_Model).inputIndexes.size(); i++) { const armnn::TensorInfo inputTensorInfo = m_Runtime->GetInputTensorInfo(m_NetworkId, i); storage.emplace_back(inputTensorInfo.GetNumBytes()); const armnn::ConstTensor inputTensor(inputTensorInfo, storage.back().data()); inputTensors.emplace_back(i, inputTensor); } armnn::OutputTensors outputTensors; for (unsigned int i = 0; i < getMainModel(m_Model).outputIndexes.size(); i++) { const armnn::TensorInfo outputTensorInfo = m_Runtime->GetOutputTensorInfo(m_NetworkId, i); storage.emplace_back(outputTensorInfo.GetNumBytes()); const armnn::Tensor outputTensor(outputTensorInfo, storage.back().data()); outputTensors.emplace_back(i, outputTensor); } auto nullCallback = [](V1_0::ErrorStatus, std::vector, const V1_2::Timing&, std::string) {}; CallbackContext_1_2 callbackContext; callbackContext.callback = nullCallback; callbackContext.ctx.measureTimings = V1_2::MeasureTiming::NO; auto memPools = std::make_shared>(); return ExecuteGraph(memPools, inputTensors, outputTensors, callbackContext); } template Return ArmnnPreparedModel_1_2::Execute(const V1_0::Request& request, V1_2::MeasureTiming measureTiming, CallbackAsync_1_2 callback) { ExecutionContext_1_2 ctx; if (measureTiming == V1_2::MeasureTiming::YES) { ctx.measureTimings = measureTiming; ctx.driverStart = Now(); } ALOGV("ArmnnPreparedModel_1_2::execute(): %s", GetModelSummary(m_Model).c_str()); m_RequestCount++; if (!android::nn::validateRequest(request, m_Model)) { callback(V1_0::ErrorStatus::INVALID_ARGUMENT, {}, g_NoTiming, "ArmnnPreparedModel_1_2::execute"); return V1_0::ErrorStatus::INVALID_ARGUMENT; } if (!m_RequestInputsAndOutputsDumpDir.empty()) { ALOGD("Dumping inputs and outputs for request %" PRIuPTR, reinterpret_cast(&callback)); } // map the memory pool into shared pointers // use a shared memory pools vector on the heap, as it is passed to the request thread auto memPools = std::make_shared>(); // allocate the tensors on the heap, as they are passed to the request thread auto inputTensors = std::make_shared(); auto outputTensors = std::make_shared(); auto prepareStatus = PrepareMemoryForIO(*inputTensors, *outputTensors, *memPools, request, callback); switch(prepareStatus) { case V1_0::ErrorStatus::OUTPUT_INSUFFICIENT_SIZE: return V1_0::ErrorStatus::NONE; case V1_0::ErrorStatus::GENERAL_FAILURE: return V1_0::ErrorStatus::GENERAL_FAILURE; default: {} } ALOGV("ArmnnPreparedModel_1_2::execute(...) before PostMsg"); // post the request for asynchronous execution CallbackContext_1_2 cb; cb.callback = callback; cb.ctx = ctx; m_RequestThread.PostMsg(this, memPools, inputTensors, outputTensors, cb); ALOGV("ArmnnPreparedModel_1_2::execute(...) after PostMsg"); return V1_0::ErrorStatus::NONE; } template Return ArmnnPreparedModel_1_2::configureExecutionBurst( const sp& callback, const MQDescriptorSync& requestChannel, const MQDescriptorSync& resultChannel, V1_2::IPreparedModel::configureExecutionBurst_cb cb) { ALOGV("ArmnnPreparedModel_1_2::configureExecutionBurst"); const sp burst = ExecutionBurstServer::create(callback, requestChannel, resultChannel, this); if (burst == nullptr) { cb(V1_0::ErrorStatus::GENERAL_FAILURE, {}); } else { cb(V1_0::ErrorStatus::NONE, burst); } return Void(); } #if defined(ARMNN_ANDROID_NN_V1_2) || defined(ARMNN_ANDROID_NN_V1_3) template class ArmnnPreparedModel_1_2; template bool ArmnnPreparedModel_1_2::ExecuteGraph( std::shared_ptr>& pMemPools, armnn::InputTensors& pInputTensors, armnn::OutputTensors& pOutputTensors, CallbackContext_1_2 cb); #endif } // namespace armnn_driver