diff --git a/backends/arm/runtime/EthosUBackend.cpp b/backends/arm/runtime/EthosUBackend.cpp index 1305c5b4995..2e752e99995 100644 --- a/backends/arm/runtime/EthosUBackend.cpp +++ b/backends/arm/runtime/EthosUBackend.cpp @@ -56,16 +56,30 @@ namespace arm { extern "C" { void __attribute__((weak)) EthosUBackend_execute_begin() {} void __attribute__((weak)) EthosUBackend_execute_end() {} +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) +void __attribute__((weak)) EthosUBackend_delegate_begin(const void*) {} +void __attribute__((weak)) EthosUBackend_delegate_end() {} +#endif __attribute__((weak)) unsigned char* ethosu_fast_scratch = nullptr; __attribute__((weak)) size_t ethosu_fast_scratch_size = 0; } class EthosUBackendExecuteCallbacks { public: +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) + explicit EthosUBackendExecuteCallbacks(const void* handle) { + EthosUBackend_execute_begin(); + EthosUBackend_delegate_begin(handle); + } +#else EthosUBackendExecuteCallbacks() { EthosUBackend_execute_begin(); } +#endif ~EthosUBackendExecuteCallbacks() { +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) + EthosUBackend_delegate_end(); +#endif EthosUBackend_execute_end(); } }; @@ -152,7 +166,11 @@ class EthosUBackend final : public ::executorch::runtime::BackendInterface { // and EthosUBackend_execute_end() is called while CollectArm_CPU_Cycles is // in scope. e.g. We meassure from now until we exit this metod (in any way // we might do it). +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) + EthosUBackendExecuteCallbacks CollectArm_CPU_Cycles(input_handle); +#else EthosUBackendExecuteCallbacks CollectArm_CPU_Cycles; +#endif ExecutionHandle* execution_handle = static_cast(input_handle); diff --git a/backends/arm/runtime/EthosUBackend_Internal.h b/backends/arm/runtime/EthosUBackend_Internal.h index 48fc4aa3a79..01069a9af72 100644 --- a/backends/arm/runtime/EthosUBackend_Internal.h +++ b/backends/arm/runtime/EthosUBackend_Internal.h @@ -74,6 +74,10 @@ struct ExecutionHandle { extern "C" { void EthosUBackend_execute_begin(); void EthosUBackend_execute_end(); +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) +void EthosUBackend_delegate_begin(const void* handle); +void EthosUBackend_delegate_end(); +#endif extern unsigned char* ethosu_fast_scratch; extern size_t ethosu_fast_scratch_size; } diff --git a/examples/arm/executor_runner/CMakeLists.txt b/examples/arm/executor_runner/CMakeLists.txt index ba80df19ddf..4280dde15d8 100644 --- a/examples/arm/executor_runner/CMakeLists.txt +++ b/examples/arm/executor_runner/CMakeLists.txt @@ -54,6 +54,13 @@ set(ET_NUM_INFERENCES option(ET_LOG_DUMP_INPUT "Dump input in log" OFF) option(ET_LOG_DUMP_OUTPUT "Dump output in log" ON) +option(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING + "Report Ethos-U PMU statistics per delegate" OFF +) +set(ET_ARM_ETHOSU_MAX_PROFILED_DELEGATES + "16" + CACHE STRING "Maximum delegates tracked by per-delegate profiling" +) option(ET_BUNDLE_IO "Set to compile in BundleIO support" OFF) set(BUNDLED_PROGRAM_LIBRARY_DIR @@ -338,6 +345,24 @@ if(ET_NUM_INFERENCES) ) endif() +if(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) + if(NOT ET_ARM_ETHOSU_MAX_PROFILED_DELEGATES MATCHES "^[1-9][0-9]*$") + message( + FATAL_ERROR + "ET_ARM_ETHOSU_MAX_PROFILED_DELEGATES must be a positive integer" + ) + endif() + target_compile_definitions( + arm_executor_runner + PRIVATE + ET_ARM_ETHOSU_PER_DELEGATE_PROFILING + ET_ARM_ETHOSU_MAX_PROFILED_DELEGATES=${ET_ARM_ETHOSU_MAX_PROFILED_DELEGATES} + ) + target_compile_definitions( + executorch_delegate_ethos_u PRIVATE ET_ARM_ETHOSU_PER_DELEGATE_PROFILING + ) +endif() + if(ET_LOG_DUMP_INPUT) target_compile_definitions(arm_executor_runner PUBLIC ET_LOG_DUMP_INPUT) endif() diff --git a/examples/arm/executor_runner/arm_perf_monitor.cpp b/examples/arm/executor_runner/arm_perf_monitor.cpp index 59daede6920..9d92a878a2b 100644 --- a/examples/arm/executor_runner/arm_perf_monitor.cpp +++ b/examples/arm/executor_runner/arm_perf_monitor.cpp @@ -40,6 +40,36 @@ uint64_t ethosu_ArmBackendExecuteCycleCount = 0; uint64_t ethosu_ArmWhenNPURunCycleCountStart = 0; uint64_t ethosu_ArmWhenNPURunCycleCount = 0; uint64_t ethosu_pmuCycleCount = 0; +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) +struct DelegateStats { + const void* handle = nullptr; + uint64_t backend_invocations = 0; + uint64_t npu_invocations = 0; + uint64_t pmu_cycles = 0; + std::array pmu_events{}; +}; + +std::array + ethosu_delegateStats; +size_t ethosu_delegateCount = 0; +DelegateStats* ethosu_activeDelegate = nullptr; +bool ethosu_delegateCapacityExceeded = false; + +DelegateStats* get_delegate_stats(const void* handle) { + for (size_t i = 0; i < ethosu_delegateCount; ++i) { + if (ethosu_delegateStats[i].handle == handle) { + return ðosu_delegateStats[i]; + } + } + if (ethosu_delegateCount == ethosu_delegateStats.size()) { + ethosu_delegateCapacityExceeded = true; + return nullptr; + } + DelegateStats& stats = ethosu_delegateStats[ethosu_delegateCount++]; + stats.handle = handle; + return &stats; +} +#endif std::array ethosu_pmuEventCounts = {0}; // ethosu_pmuCountersUsed should match numbers of counters setup in @@ -50,6 +80,19 @@ static_assert(ETHOSU_PMU_NCOUNTERS >= ethosu_pmuCountersUsed); extern "C" { +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) +void EthosUBackend_delegate_begin(const void* handle) { + ethosu_activeDelegate = get_delegate_stats(handle); + if (ethosu_activeDelegate != nullptr) { + ethosu_activeDelegate->backend_invocations++; + } +} + +void EthosUBackend_delegate_end() { + ethosu_activeDelegate = nullptr; +} +#endif + // Callback invoked at start of NPU execution void ethosu_inference_begin(struct ethosu_driver* drv, void*) { // Enable PMU @@ -100,10 +143,27 @@ void ethosu_inference_begin(struct ethosu_driver* drv, void*) { // Callback invoked at end of NPU execution void ethosu_inference_end(struct ethosu_driver* drv, void*) { ethosu_delegation_count++; +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) + const uint64_t pmu_cycles = ETHOSU_PMU_Get_CCNTR(drv); + ethosu_pmuCycleCount += pmu_cycles; + if (ethosu_activeDelegate != nullptr) { + ethosu_activeDelegate->npu_invocations++; + ethosu_activeDelegate->pmu_cycles += pmu_cycles; + } +#else ethosu_pmuCycleCount += ETHOSU_PMU_Get_CCNTR(drv); +#endif for (size_t i = 0; i < ethosu_pmuCountersUsed; i++) { +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) + const uint64_t event_count = ETHOSU_PMU_Get_EVCNTR(drv, i); + ethosu_pmuEventCounts[i] += event_count; + if (ethosu_activeDelegate != nullptr) { + ethosu_activeDelegate->pmu_events[i] += event_count; + } +#else ethosu_pmuEventCounts[i] += ETHOSU_PMU_Get_EVCNTR(drv, i); +#endif } ETHOSU_PMU_Disable(drv); // Add Cortex-M cycle clock used during this NPU execution @@ -131,6 +191,12 @@ void StartMeasurements() { ethosu_ArmBackendExecuteCycleCount = 0; ethosu_ArmWhenNPURunCycleCount = 0; ethosu_pmuCycleCount = 0; +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) + ethosu_delegateStats = {}; + ethosu_delegateCount = 0; + ethosu_activeDelegate = nullptr; + ethosu_delegateCapacityExceeded = false; +#endif for (size_t i = 0; i < ethosu_pmuCountersUsed; i++) { ethosu_pmuEventCounts[i] = 0; @@ -221,6 +287,39 @@ void StopMeasurements(int num_inferences) { ethosu_pmuEventCounts[i], (double)ethosu_pmuEventCounts[i] / num_inferences); } +#if defined(ET_ARM_ETHOSU_PER_DELEGATE_PROFILING) + ET_LOG(Info, "Ethos-U per-delegate PMU report:"); + for (size_t delegate_id = 0; delegate_id < ethosu_delegateCount; + ++delegate_id) { + const DelegateStats& stats = ethosu_delegateStats[delegate_id]; + ET_LOG( + Info, + "Ethos-U delegate %zu: %" PRIu64 " backend invocations, %" PRIu64 + " NPU invocations", + delegate_id, + stats.backend_invocations, + stats.npu_invocations); + ET_LOG( + Info, + "Ethos-U delegate %zu PMU cycles: %" PRIu64, + delegate_id, + stats.pmu_cycles); + for (size_t event = 0; event < ethosu_pmuCountersUsed; ++event) { + ET_LOG( + Info, + "Ethos-U delegate %zu PMU counter %zu: %" PRIu64, + delegate_id, + event, + stats.pmu_events[event]); + } + } + if (ethosu_delegateCapacityExceeded) { + ET_LOG( + Error, + "Ethos-U per-delegate profiling exceeded its capacity of %zu delegates", + ethosu_delegateStats.size()); + } +#endif #if defined(ETHOSU55) || defined(ETHOSU65) ET_LOG( Info,