diff --git a/backends/qualcomm/aot/python/PyQnnManagerAdaptor.cpp b/backends/qualcomm/aot/python/PyQnnManagerAdaptor.cpp index 8d7e8aae6bc..49e062d4329 100644 --- a/backends/qualcomm/aot/python/PyQnnManagerAdaptor.cpp +++ b/backends/qualcomm/aot/python/PyQnnManagerAdaptor.cpp @@ -239,6 +239,10 @@ PYBIND11_MODULE(PyQnnManagerAdaptor, m) { const std::vector&, std::vector>>&>( &PyQnnManager::Compile)) + .def("CreateDlc", &PyQnnManager::CreateDlc) + .def("CompileToDlc", &PyQnnManager::CompileToDlc) + .def("GetDlcBinary", &PyQnnManager::GetDlcBinary) + .def("FreeDlc", &PyQnnManager::FreeDlc) .def("Destroy", &PyQnnManager::Destroy) .def("DestroyContext", &PyQnnManager::DestroyContext) .def("IsAvailable", &PyQnnManager::IsAvailable) diff --git a/backends/qualcomm/aot/python/PyQnnManagerAdaptor.h b/backends/qualcomm/aot/python/PyQnnManagerAdaptor.h index 3ce7c233559..2b220ad6d17 100644 --- a/backends/qualcomm/aot/python/PyQnnManagerAdaptor.h +++ b/backends/qualcomm/aot/python/PyQnnManagerAdaptor.h @@ -245,7 +245,7 @@ class PyQnnManager { return qnn_manager_->IsNodeSupportedByBackend(op_wrappers); } - py::array_t Compile( + py::bytes Compile( const std::vector& graph_names, std::vector>>& op_wrappers) { QnnExecuTorchContextBinary binary_info; @@ -254,7 +254,7 @@ class PyQnnManager { if (qnn_manager_->Compile(graph_names[i], op_wrappers[i]) != executorch::runtime::Error::Ok) { QNN_EXECUTORCH_LOG_ERROR("Fail to compile QNN graph"); - return py::array_t(0); + return py::bytes("", 0); } } auto qnn_executorch_options = GetQnnExecuTorchOptions( @@ -262,15 +262,47 @@ class PyQnnManager { if (qnn_executorch_options->saver() || qnn_manager_->GetContextBinary(binary_info) != executorch::runtime::Error::Ok) { - return py::array_t(0); + return py::bytes("", 0); } - // allocate py::array (to pass the result of the C++ function to Python) - auto result = py::array_t(binary_info.nbytes); - auto result_buffer = result.request(); - char* result_ptr = (char*)result_buffer.ptr; - std::memcpy(result_ptr, binary_info.buffer, binary_info.nbytes); - return result; + return py::bytes( + reinterpret_cast(binary_info.buffer), binary_info.nbytes); + } + py::int_ CreateDlc() { + void* handle = nullptr; + if (qnn_manager_->CreateDlc(handle) != Error::Ok) { + throw std::runtime_error("Failed to create QNN DLC"); + } + return reinterpret_cast(handle); + } + + void CompileToDlc( + const std::vector& graph_names, + std::vector>>& op_wrappers, + uintptr_t dlc_handle) { + for (uint32_t i = 0; i < graph_names.size(); ++i) { + if (qnn_manager_->Compile(graph_names[i], op_wrappers[i]) != Error::Ok) { + throw std::runtime_error("Failed to compile QNN graph"); + } + } + if (qnn_manager_->AddContextToDlc(reinterpret_cast(dlc_handle)) != + Error::Ok) { + throw std::runtime_error("Failed to add QNN context to DLC"); + } + } + + py::bytes GetDlcBinary(uintptr_t dlc_handle) { + std::vector binary; + if (qnn_manager_->GetDlcBinary( + reinterpret_cast(dlc_handle), binary) != Error::Ok) { + throw std::runtime_error("Failed to get QNN DLC binary"); + } + return py::bytes( + reinterpret_cast(binary.data()), binary.size()); + } + + void FreeDlc(uintptr_t dlc_handle) { + qnn_manager_->FreeDlc(reinterpret_cast(dlc_handle)); } void Destroy() { diff --git a/backends/qualcomm/aot/wrappers/OpWrapper.h b/backends/qualcomm/aot/wrappers/OpWrapper.h index 2acf430c0a9..4b6cf32fade 100644 --- a/backends/qualcomm/aot/wrappers/OpWrapper.h +++ b/backends/qualcomm/aot/wrappers/OpWrapper.h @@ -116,6 +116,22 @@ class OpWrapper final { } return raw_params; } + + void ResetGraphState() { + for (const auto& tensor_wrapper : input_tensors_) { + tensor_wrapper->ResetGraphState(); + } + for (const auto& tensor_wrapper : output_tensors_) { + tensor_wrapper->ResetGraphState(); + } + for (const auto& param : params_) { + auto* tensor_param = dynamic_cast(param.get()); + if (tensor_param != nullptr) { + tensor_param->GetTensorWrapper()->ResetGraphState(); + } + } + } + Qnn_OpConfig_t GetOpConfig(); private: diff --git a/backends/qualcomm/aot/wrappers/TensorWrapper.h b/backends/qualcomm/aot/wrappers/TensorWrapper.h index 98f59532afb..3b255afa51a 100644 --- a/backends/qualcomm/aot/wrappers/TensorWrapper.h +++ b/backends/qualcomm/aot/wrappers/TensorWrapper.h @@ -60,6 +60,11 @@ class TensorWrapper { created_ = true; } + void ResetGraphState() { + created_ = false; + QNN_TENSOR_VER_PTR(tensor_)->id = 0u; + } + // Return true if the tensor is static: bool IsTensorStatic() const { return QNN_TENSOR_VER_PTR(tensor_)->type == QNN_TENSOR_TYPE_STATIC; diff --git a/backends/qualcomm/export_utils.py b/backends/qualcomm/export_utils.py index fcfb5461969..6ff28aed2fe 100644 --- a/backends/qualcomm/export_utils.py +++ b/backends/qualcomm/export_utils.py @@ -16,7 +16,7 @@ import tempfile import types from dataclasses import dataclass, fields -from typing import Callable, List, Optional, Set, Tuple, Union +from typing import Callable, List, Optional, Sequence, Set, Tuple, Union import numpy as np import torch @@ -676,10 +676,18 @@ def make_quantizer( is_qat=False, submodule_qconfig_list: Optional[List[Tuple[Callable, ModuleQConfig]]] = None, backend=QnnExecuTorchBackendType.kHtpBackend, - soc_model="SM8750", + soc_model: Union[str, QcomChipset, Sequence[Union[str, QcomChipset]]] = "SM8750", eps=None, ): - quantizer = QnnQuantizer(backend=backend, soc_model=getattr(QcomChipset, soc_model)) + def to_chipset(model: Union[str, QcomChipset]) -> QcomChipset: + return model if isinstance(model, QcomChipset) else getattr(QcomChipset, model) + + soc_models = ( + [to_chipset(model) for model in soc_model] + if not isinstance(soc_model, (str, QcomChipset)) + else to_chipset(soc_model) + ) + quantizer = QnnQuantizer(backend=backend, soc_model=soc_models) quantizer.add_custom_quant_annotations(custom_annotations) quantizer.set_default_quant_config( quant_dtype, diff --git a/backends/qualcomm/partition/qnn_partitioner.py b/backends/qualcomm/partition/qnn_partitioner.py index 0d3fc3d3aa8..77a89db9bac 100644 --- a/backends/qualcomm/partition/qnn_partitioner.py +++ b/backends/qualcomm/partition/qnn_partitioner.py @@ -74,9 +74,15 @@ def __init__( # checker (e.g. the LPAI fallback pass) are distinguishable in the logs. self.phase = phase self.nodes_to_wrappers = defaultdict(dict) - self.qnn_manager = get_current_qnn_manager( - python_options.backend_options.backend_type, compiler_specs + target_socs = ( + [target.soc_info.soc_model for target in python_options.fcb_options.targets] + if python_options.fcb_options is not None + else [python_options.soc_info.soc_model] ) + self.qnn_managers = [ + get_current_qnn_manager(compiler_specs, soc_model) + for soc_model in target_socs + ] def is_node_supported(self, _, node: torch.fx.Node) -> bool: if node.op != "call_function" or node.target in not_supported_operator: @@ -128,8 +134,10 @@ def is_node_supported(self, _, node: torch.fx.Node) -> bool: op_wrapper_list.append(op_wrapper) if op_wrapper is not None: - supported = self.qnn_manager.IsNodeSupportedByBackend( - [op_wrapper.GetOpWrapper() for op_wrapper in op_wrapper_list] + wrappers = [op.GetOpWrapper() for op in op_wrapper_list] + supported = all( + manager.IsNodeSupportedByBackend(wrappers) + for manager in self.qnn_managers ) self.nodes_to_wrappers.clear() diff --git a/backends/qualcomm/qnn_preprocess.py b/backends/qualcomm/qnn_preprocess.py index 0d8fcd6fe3b..1f0204e02ff 100644 --- a/backends/qualcomm/qnn_preprocess.py +++ b/backends/qualcomm/qnn_preprocess.py @@ -6,7 +6,7 @@ import logging from collections import defaultdict -from typing import Dict, final, List +from typing import Dict, final, List, Literal, Tuple, Union import executorch.backends.qualcomm.python.PyQnnManagerAdaptor as PyQnnManager import torch # noqa: F401 @@ -19,6 +19,7 @@ from executorch.backends.qualcomm.serialization.qc_schema import ( QnnExecuTorchBackendType, QnnExecuTorchOpPackageInfo, + QnnExecuTorchOptions, ) from executorch.backends.qualcomm.serialization.qc_schema_serialize import ( flatbuffer_to_option, @@ -165,9 +166,7 @@ def preprocess( ) -> PreprocessResult: option = generate_qnn_executorch_option(compile_specs) obj_options = flatbuffer_to_option(option) - qnn_manager = get_current_qnn_manager( - obj_options.backend_options.backend_type, compile_specs - ) + qnn_manager = get_current_qnn_manager(compile_specs) qnn_manager.InitContext([DEFAULT_GRAPH_NAME]) py_op_wrapper_list = QnnBackend._build_op_wrappers( edge_program, @@ -190,103 +189,191 @@ def preprocess( qnn_manager.DestroyContext() # For now, debug_handle_map is not used by QNN ExecuTorch return PreprocessResult( - processed_bytes=bytes(qnn_context_binary), + processed_bytes=qnn_context_binary, debug_handle_map={}, ) + @staticmethod + def _populate_delegate_mapping( + debug_handle_builder: DelegateMappingBuilder, + num_partitions: int, + edge_programs: Dict[str, List[ExportedProgram]], + ): + for i in range(num_partitions): + for programs in edge_programs.values(): + for node in programs[i].graph.nodes: + # Skip multi-output nodes: devtools only supports + # single-output intermediate capture (len == 1). + if ( + (handle_id := node.meta.get(DEBUG_HANDLE_KEY)) + and QCOM_TENSOR_NAME in node.meta + and len(node.meta[QCOM_TENSOR_NAME]) == 1 + ): + debug_handle_builder.insert_delegate_mapping_entry( + handles=handle_id, + identifier=node.meta[QCOM_TENSOR_NAME][0], + ) + + @staticmethod + def _get_op_wrappers( + option: QnnExecuTorchOptions, + num_partitions: int, + edge_programs: Dict[str, List[ExportedProgram]], + ) -> Tuple[ + Literal["ctx_binary", "op_wrapper"], + Union[ + List[Dict[str, bytes]], + List[Dict[str, List[PyQnnManager.OpWrapper]]], + ], + ]: + py_op_wrapper_list, ctx_binary_list = [], [] + wrapper_type = None + for i in range(num_partitions): + subgraph_op_wrapper, subgraph_ctx_binary = dict(), dict() + for key, programs in edge_programs.items(): + logger.info( + f"Extracting OpWrapper for Method({key}): ({i+1}/{num_partitions})" + ) + py_op_wrappers = QnnBackend._build_op_wrappers( + programs[i], + option.dump_intermediate_outputs, + option.op_package_options.op_package_infos, + option.use_mha2sha, + option.backend_options.backend_type, + ) + if isinstance(py_op_wrappers, bytes): + # ensure not mixed + if wrapper_type and wrapper_type != "ctx_binary": + raise RuntimeError("Hybrid compilation is not supported") + wrapper_type = "ctx_binary" + + subgraph_ctx_binary[key] = py_op_wrappers + else: + # ensure not mixed + if wrapper_type and wrapper_type != "op_wrapper": + raise RuntimeError("Hybrid compilation is not supported") + wrapper_type = "op_wrapper" + + subgraph_op_wrapper[key] = [ + py_op_wrapper.GetOpWrapper() for py_op_wrapper in py_op_wrappers + ] + # append + match wrapper_type: + case "op_wrapper": + py_op_wrapper_list.append(subgraph_op_wrapper) + case "ctx_binary": + ctx_binary_list.append(subgraph_ctx_binary) + case _: + raise ValueError("Unexpected wrapper_type") + return ( + wrapper_type, + py_op_wrapper_list if wrapper_type == "op_wrapper" else ctx_binary_list, + ) + + @staticmethod + def _get_compile_func(qnn_manager: PyQnnManager.QnnManager): + def compile_func(graph_names, op_wrapper_list): + qnn_manager.InitContext(graph_names) + try: + qnn_context_binary = qnn_manager.Compile(graph_names, op_wrapper_list) + finally: + qnn_manager.DestroyContext() + return qnn_context_binary + + return compile_func + + @staticmethod + def _get_compile_func_fcb(qnn_managers: List[PyQnnManager.QnnManager]): + def compile_func(graph_names, op_wrapper_list): + dlc_handle = qnn_managers[0].CreateDlc() + try: + for qnn_manager in qnn_managers: + qnn_manager.InitContext(graph_names) + try: + qnn_manager.CompileToDlc( + graph_names, op_wrapper_list, dlc_handle + ) + finally: + qnn_manager.DestroyContext() + dlc_binary = bytes(qnn_managers[0].GetDlcBinary(dlc_handle)) + finally: + qnn_managers[0].FreeDlc(dlc_handle) + return dlc_binary + + return compile_func + @staticmethod def preprocess_multimethod( # noqa: C901 edge_programs: Dict[str, List[ExportedProgram]], compile_specs: Dict[str, List[List[CompileSpec]]], - ) -> PreprocessResult: + ) -> Dict[str, List[PreprocessResult]]: # TODO: refactor QnnManager to consume multiple compile_spec # take first compile_specs here for the same partitions graph_names = list(edge_programs.keys()) compile_spec = list(compile_specs.values())[0][0] option = flatbuffer_to_option(compile_spec[0].value) # check if each graph has equal number of partitions - num_sub_graphs = set() + num_partitions = set() for edge_program in edge_programs.values(): - num_sub_graphs.add(len(edge_program)) + num_partitions.add(len(edge_program)) # this constraint is dedicated to weight-sharing scenario assert ( - len(num_sub_graphs) == 1 + len(num_partitions) == 1 ), "Only graphs with the same number of partitions could be used" - all_processed_results = {key: [] for key in edge_programs.keys()} - num_sub_graphs = next(iter(num_sub_graphs)) - qnn_manager = get_current_qnn_manager( - option.backend_options.backend_type, compile_spec - ) + num_partitions = next(iter(num_partitions)) + + # populate debug handle mapping debug_handle_builder = DelegateMappingBuilder(generated_identifiers=False) - for i in range(num_sub_graphs): - # e.g. 2 methods (x, y) with 3 subgraphs(partitions) - # > context_binary_0: [x.subgraph_0, y.subgraph_0] - # > context_binary_1: [x.subgraph_1, y.subgraph_1] - # > context_binary_2: [x.subgraph_2, y.subgraph_2] - qnn_manager.InitContext(graph_names) - py_op_wrapper_list, ctx_binary_list = [], [] - for j, programs in enumerate(edge_programs.values()): - logger.info(f"Processing Method({j}): ({i+1}/{num_sub_graphs})") - py_op_wrappers = QnnBackend._build_op_wrappers( - programs[i], - qnn_manager.IsTensorDump(), - option.op_package_options.op_package_infos, - option.use_mha2sha, - option.backend_options.backend_type, - ) - if qnn_manager.IsTensorDump(): - for node in programs[i].graph.nodes: - # Skip multi-output nodes: devtools only supports - # single-output intermediate capture (len == 1). - if ( - (handle_id := node.meta.get(DEBUG_HANDLE_KEY)) - and QCOM_TENSOR_NAME in node.meta - and len(node.meta[QCOM_TENSOR_NAME]) == 1 - ): - debug_handle_builder.insert_delegate_mapping_entry( - handles=handle_id, - identifier=node.meta[QCOM_TENSOR_NAME][0], + if option.dump_intermediate_outputs: + QnnBackend._populate_delegate_mapping( + debug_handle_builder, num_partitions, edge_programs + ) + + # get op_wrapper_list or ctx_binary_list for embedded mode. + wrapper_type, op_wrappers = QnnBackend._get_op_wrappers( + option, num_partitions, edge_programs + ) + + all_processed_results = {key: [] for key in edge_programs} + match wrapper_type: + case "ctx_binary": + for i in range(num_partitions): + for key in edge_programs: + all_processed_results[key].append( + PreprocessResult( + processed_bytes=op_wrappers[i][key], + debug_handle_map=debug_handle_builder.get_delegate_mapping(), ) - if isinstance(py_op_wrappers, bytes): - ctx_binary_list.append(py_op_wrappers) + ) + case "op_wrapper": + if option.fcb_options is not None: + qnn_managers = [ + get_current_qnn_manager(compile_spec, target.soc_info.soc_model) + for target in option.fcb_options.targets + ] + compile_func = QnnBackend._get_compile_func_fcb(qnn_managers) else: - py_op_wrapper_list.append( - [ - py_op_wrapper.GetOpWrapper() - for py_op_wrapper in py_op_wrappers - ] - ) - if len(py_op_wrapper_list) == len(edge_programs.values()): - qnn_context_binary = qnn_manager.Compile( - graph_names, py_op_wrapper_list - ) - if option.saver: - # TODO: Currently, only the first method is saved. Update this logic if saving multiple methods becomes necessary in the future. - exit( - f"Record all QNN API calls from saver backend at: {option.saver_output_dir}" - ) - assert ( - len(qnn_context_binary) != 0 - ), "Failed to generate Qnn context binary." - qnn_manager.DestroyContext() - # methods should share the same context binary for current partition - for key in edge_programs.keys(): - all_processed_results[key].append( - PreprocessResult( - processed_bytes=bytes(qnn_context_binary), - debug_handle_map=debug_handle_builder.get_delegate_mapping(), + qnn_manager = get_current_qnn_manager(compile_spec) + compile_func = QnnBackend._get_compile_func(qnn_manager) + for i in range(num_partitions): + op_wrapper_list = list(op_wrappers[i].values()) + context_binary = compile_func(graph_names, op_wrapper_list) + if option.saver: + # TODO: Currently, only the first method is saved. Update this logic if saving multiple methods becomes necessary in the future. + exit( + f"Record all QNN API calls from saver backend at: {option.saver_output_dir}" ) - ) - elif len(ctx_binary_list) == len(edge_programs.values()): - for i, key in enumerate(edge_programs.keys()): - all_processed_results[key].append( - PreprocessResult( - processed_bytes=ctx_binary_list[i], - debug_handle_map=debug_handle_builder.get_delegate_mapping(), + assert ( + len(context_binary) != 0 + ), "Failed to generate Qnn context binary." + for key in edge_programs: + all_processed_results[key].append( + PreprocessResult( + processed_bytes=context_binary, + debug_handle_map=debug_handle_builder.get_delegate_mapping(), + ) ) - ) - else: - raise RuntimeError("Hybrid compilation is not supported") - + case _: + raise ValueError("Unexpected wrapper type") return all_processed_results diff --git a/backends/qualcomm/quantizer/quantizer.py b/backends/qualcomm/quantizer/quantizer.py index ffe3e1d43a5..f5b10fffa62 100644 --- a/backends/qualcomm/quantizer/quantizer.py +++ b/backends/qualcomm/quantizer/quantizer.py @@ -10,7 +10,7 @@ from enum import IntEnum, unique from functools import partial from operator import attrgetter -from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple +from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple, Union # To support quantize op lowering in AOT try: @@ -345,7 +345,9 @@ class QnnQuantizer(Quantizer): Args: backend: QnnQuantizer uses the backend_type to dynamically load the appropriate backend rules as needed. - soc_model: QnnQuantizer checks each operation according to the soc_model. For example, LPBQ requires V69 or a newer version. + soc_model: + QnnQuantizer checks each operation according to the soc_model. + For an FCB target list, validation uses the target with the lowest HTP architecture. strict: When enabled (default), the validation stage raises a ValueError if quantization constraints are not met. In this mode, all quantization constraints must be satisfied to fully delegate to the QNN Backend. @@ -376,13 +378,26 @@ class QnnQuantizer(Quantizer): def __init__( self, backend: QnnExecuTorchBackendType = QnnExecuTorchBackendType.kHtpBackend, - soc_model: QcomChipset = QcomChipset.SM8750, + soc_model: Union[QcomChipset, Sequence[QcomChipset]] = QcomChipset.SM8750, strict: bool = True, ): super().__init__() self.strict = strict self.backend = backend - self.soc_info = _soc_info_table[soc_model] + self.soc_models = ( + (soc_model,) if isinstance(soc_model, QcomChipset) else tuple(soc_model) + ) + if not self.soc_models: + raise ValueError("soc_model must not be empty") + if len(self.soc_models) > 1 and backend != QnnExecuTorchBackendType.kHtpBackend: + raise ValueError( + "multiple soc_model (FCB) is only supported for HTP backend" + ) + least_soc_model = min( + self.soc_models, + key=lambda model: _soc_info_table[model].htp_info.htp_arch, + ) + self.soc_info = _soc_info_table[least_soc_model] # Lazy load rules and constraints of current backend self._rules_map, self._constraint_cache = load_backend_rules_and_constraints( @@ -396,8 +411,8 @@ def __init__( # convolution, so a guard applied at lowering time comes after the risk has passed. disable_mkldnn_on_amd() - # Load backend_opinfo of current backend and soc_model - self.backend_opinfo = get_backend_opinfo(str(backend), soc_model) + # Validate against the least capable FCB target. + self.backend_opinfo = get_backend_opinfo(str(backend), least_soc_model) self.default_quant_config = ModuleQConfig() self.submodule_qconfig_list: List[ diff --git a/backends/qualcomm/runtime/QnnManager.cpp b/backends/qualcomm/runtime/QnnManager.cpp index 0f1169d26a8..d81e3f9c4bf 100644 --- a/backends/qualcomm/runtime/QnnManager.cpp +++ b/backends/qualcomm/runtime/QnnManager.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -46,6 +47,22 @@ int ExtractMutableBufferNumber(const std::string& name) { return -1; } +class OpWrapperGraphStateGuard final { + public: + explicit OpWrapperGraphStateGuard( + const std::vector>& op_wrappers) + : op_wrappers_(op_wrappers) {} + + ~OpWrapperGraphStateGuard() { + for (const auto& op_wrapper : op_wrappers_) { + op_wrapper->ResetGraphState(); + } + } + + private: + const std::vector>& op_wrappers_; +}; + QnnManager::~QnnManager() { Destroy(); } @@ -228,6 +245,99 @@ Error QnnManager::RegisterCustomMem( return Error::Ok; } +Error QnnManager::GetContextBinarySize(uint64_t& out_size) { + Qnn_ContextBinarySize_t binary_size = 0; + auto error = + backend_bundle_ptr_->implementation->GetQnnInterface() + .qnn_context_get_binary_size( + backend_params_ptr_->qnn_context_ptr_->GetHandle(), &binary_size); + if (error != QNN_SUCCESS) { + QNN_EXECUTORCH_LOG_ERROR( + "Failed to get context binary size. Error %d", + QNN_GET_ERROR_CODE(error)); + return Error::Internal; + } + out_size = binary_size; + return Error::Ok; +} + +Error QnnManager::CreateDlc(void*& out_dlc_handle) { +#if QNN_EXECUTORCH_SUPPORTS_FCB + QnnSystemDlc_Handle_t handle = nullptr; + auto error = + backend_bundle_ptr_->system_implementation->GetQnnSystemInterface() + .qnn_system_dlc_create_with_destination_dir( + backend_bundle_ptr_->qnn_logger_ptr->GetHandle(), + nullptr, + &handle); + if (error != QNN_SUCCESS) { + QNN_EXECUTORCH_LOG_ERROR( + "Failed to create DLC. Error %d", QNN_GET_ERROR_CODE(error)); + return Error::Internal; + } + out_dlc_handle = handle; + return Error::Ok; +#else + (void)out_dlc_handle; + QNN_EXECUTORCH_LOG_ERROR( + "FCB is not supported by this QNN SDK; Compilation with QAIRT SDK 2.48 or newer is required."); + return Error::NotSupported; +#endif +} + +Error QnnManager::AddContextToDlc(void* dlc_handle) { +#if QNN_EXECUTORCH_SUPPORTS_FCB + auto error = + backend_bundle_ptr_->implementation->GetQnnInterface() + .qnn_context_add_to_dlc( + backend_params_ptr_->qnn_context_ptr_->GetHandle(), dlc_handle); + if (error != QNN_SUCCESS) { + QNN_EXECUTORCH_LOG_ERROR( + "Failed to add context to DLC. Error %d", QNN_GET_ERROR_CODE(error)); + return Error::Internal; + } + return Error::Ok; +#else + (void)dlc_handle; + QNN_EXECUTORCH_LOG_ERROR( + "FCB is not supported by this QNN SDK; Compilation with QAIRT SDK 2.48 or newer is required."); + return Error::NotSupported; +#endif +} + +Error QnnManager::GetDlcBinary(void* dlc_handle, std::vector& binary) { +#if QNN_EXECUTORCH_SUPPORTS_FCB + Qnn_SystemDlcBinarySize_t size = 0; + auto& system = + backend_bundle_ptr_->system_implementation->GetQnnSystemInterface(); + if (system.qnn_system_dlc_get_binary_size(dlc_handle, &size) != QNN_SUCCESS) { + return Error::Internal; + } + binary.resize(size); + Qnn_SystemDlcBinarySize_t written = 0; + auto error = system.qnn_system_dlc_get_binary( + dlc_handle, binary.data(), size, &written); + if (error != QNN_SUCCESS || written > size) { + QNN_EXECUTORCH_LOG_ERROR( + "Failed to get DLC binary. Error %d", QNN_GET_ERROR_CODE(error)); + return Error::Internal; + } + binary.resize(written); + return Error::Ok; +#else + (void)dlc_handle; + (void)binary; + QNN_EXECUTORCH_LOG_ERROR( + "FCB is not supported by this QNN SDK; Compilation with QAIRT SDK 2.48 or newer is required."); + return Error::NotSupported; +#endif +} + +void QnnManager::FreeDlc(void* dlc_handle) { + backend_bundle_ptr_->system_implementation->GetQnnSystemInterface() + .qnn_system_dlc_free(dlc_handle); +} + Error QnnManager::InitBackend() { // Get or create the shared backend bundle Error err = QnnBackendUnifiedRegistry::GetInstance().GetOrCreateBackendBundle( @@ -621,6 +731,7 @@ Error QnnManager::CompileDlc() { Error QnnManager::Compile( const std::string& graph_name, std::vector>& op_wrappers) { + OpWrapperGraphStateGuard reset_graph_state(op_wrappers); Qnn_ErrorHandle_t error = QNN_SUCCESS; QnnGraph* qnn_graph_ptr = backend_params_ptr_->qnn_graph_ptr_.get(); diff --git a/backends/qualcomm/runtime/QnnManager.h b/backends/qualcomm/runtime/QnnManager.h index ef0385c517b..b8c9810cc95 100644 --- a/backends/qualcomm/runtime/QnnManager.h +++ b/backends/qualcomm/runtime/QnnManager.h @@ -19,6 +19,7 @@ #include #include +#include namespace executorch { namespace backends { @@ -80,11 +81,18 @@ class QnnManager { executorch::runtime::Error GetContextBinary( QnnExecuTorchContextBinary& qnn_executorch_context_binary); + executorch::runtime::Error GetContextBinarySize(uint64_t& size); executorch::runtime::Error CompileDlc(); executorch::runtime::Error Compile( const std::string& graph_name, std::vector>& op_wrappers); + executorch::runtime::Error CreateDlc(void*& dlc_handle); + executorch::runtime::Error AddContextToDlc(void* dlc_handle); + executorch::runtime::Error GetDlcBinary( + void* dlc_handle, + std::vector& binary); + void FreeDlc(void* dlc_handle); executorch::runtime::Error RegisterMem( void* data_ptr, diff --git a/backends/qualcomm/runtime/backends/CMakeLists.txt b/backends/qualcomm/runtime/backends/CMakeLists.txt index 88ab07c3c16..03a1a52d05e 100644 --- a/backends/qualcomm/runtime/backends/CMakeLists.txt +++ b/backends/qualcomm/runtime/backends/CMakeLists.txt @@ -15,6 +15,7 @@ target_sources( target_sources( qnn_function_interface INTERFACE ${CMAKE_CURRENT_LIST_DIR}/QnnFunctionInterface.h + ${CMAKE_CURRENT_LIST_DIR}/QnnSdkCompatibility.h ) # qnn_sys_implementation @@ -28,6 +29,7 @@ target_sources( target_sources( qnn_sys_function_interface INTERFACE ${CMAKE_CURRENT_LIST_DIR}/QnnSysFunctionInterface.h + ${CMAKE_CURRENT_LIST_DIR}/QnnSdkCompatibility.h ) # qnn_logger diff --git a/backends/qualcomm/runtime/backends/QnnBackendCache.cpp b/backends/qualcomm/runtime/backends/QnnBackendCache.cpp index 6e234e9c960..b0cd576eff3 100644 --- a/backends/qualcomm/runtime/backends/QnnBackendCache.cpp +++ b/backends/qualcomm/runtime/backends/QnnBackendCache.cpp @@ -8,6 +8,7 @@ #include #include +#include namespace executorch { namespace backends { @@ -50,7 +51,7 @@ Error QnnBackendCache::GetQnnGraphInfoFromBinary( } else if (binaryinfo->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_2) { num_graphs = binaryinfo->contextBinaryInfoV2.numGraphs; graphs = binaryinfo->contextBinaryInfoV2.graphs; -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 21) } else if (binaryinfo->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_3) { num_graphs = binaryinfo->contextBinaryInfoV3.numGraphs; graphs = binaryinfo->contextBinaryInfoV3.graphs; @@ -66,7 +67,7 @@ Error QnnBackendCache::GetQnnGraphInfoFromBinary( RetrieveGraphInfo(graphs[i].graphInfoV1); } else if (graphs->version == QNN_SYSTEM_CONTEXT_GRAPH_INFO_VERSION_2) { RetrieveGraphInfo(graphs[i].graphInfoV2); -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 21) } else if (graphs->version == QNN_SYSTEM_CONTEXT_GRAPH_INFO_VERSION_3) { RetrieveGraphInfo(graphs[i].graphInfoV3); #endif @@ -79,6 +80,53 @@ Error QnnBackendCache::GetQnnGraphInfoFromBinary( return Error::Ok; } +Error QnnBackendCache::GetQnnGraphInfoFromDlc() { +#if QNN_EXECUTORCH_SUPPORTS_FCB + const QnnSystemInterface& qnn_sys_interface = + qnn_sys_impl_->GetQnnSystemInterface(); + QnnSystemDlc_RecordHandle_t* records = nullptr; + uint32_t count = 0; + auto error = qnn_sys_interface.qnn_system_dlc_create_from_binary( + nullptr, + static_cast(qnn_context_blob_.buffer), + qnn_context_blob_.nbytes, + &fcb_dlc_handle_); + if (error != QNN_SUCCESS) { + QNN_EXECUTORCH_LOG_ERROR( + "[FCB] class=MalformedDlc error=%d", QNN_GET_ERROR_CODE(error)); + return Error::Internal; + } + error = qnn_sys_interface.qnn_system_dlc_get_records_by_type( + fcb_dlc_handle_, + QNN_SYSTEM_DLC_RECORD_PREFIX_HTP_CACHE_RECORD, + 1, + &records, + &count); + if (error != QNN_SUCCESS || count != 1) { + QNN_EXECUTORCH_LOG_ERROR( + "[FCB] class=OffListSoc records=%u error=%d", + count, + QNN_GET_ERROR_CODE(error)); + return Error::Internal; + } + const uint8_t* context_binary = nullptr; + uint64_t context_binary_size = 0; + error = qnn_sys_interface.qnn_system_dlc_read_record_data_memory_mapped( + records[0], &context_binary, &context_binary_size); + if (error != QNN_SUCCESS || context_binary_size > UINT32_MAX) { + QNN_EXECUTORCH_LOG_ERROR( + "[FCB] class=MetadataIncompat error=%d", QNN_GET_ERROR_CODE(error)); + return Error::Internal; + } + return GetQnnGraphInfoFromBinary( + const_cast(context_binary), + static_cast(context_binary_size)); +#else + QNN_EXECUTORCH_LOG_ERROR( + "FCB is not supported by this QNN SDK; Compilation with QAIRT SDK 2.48 or newer is required."); + return Error::NotSupported; +#endif +} Error QnnBackendCache::Configure(const std::vector& graph_names) { if (qnn_context_blob_.buffer == nullptr) { @@ -115,24 +163,34 @@ Error QnnBackendCache::Configure(const std::vector& graph_names) { qnn_context_blob_.nbytes = context_size; } - status = GetQnnGraphInfoFromBinary( - static_cast(qnn_context_blob_.buffer), - qnn_context_blob_.nbytes); + status = is_fcb_ ? GetQnnGraphInfoFromDlc() + : GetQnnGraphInfoFromBinary( + static_cast(qnn_context_blob_.buffer), + qnn_context_blob_.nbytes); + + if (status != Error::Ok && is_fcb_) { + QNN_EXECUTORCH_LOG_ERROR("Failed to get Graph Info from input FCB DLC"); + return status; + } if (status == Error::Internal) { - // online prepare state_ = ONLINE_PREPARE; } return Error::Ok; } QnnBackendCache::~QnnBackendCache() { - Qnn_ErrorHandle_t error = QNN_SUCCESS; + const QnnSystemInterface& qnn_sys_interface = + qnn_sys_impl_->GetQnnSystemInterface(); + if (fcb_dlc_handle_ != nullptr) { + if (qnn_sys_interface.qnn_system_dlc_free(fcb_dlc_handle_) != QNN_SUCCESS) { + QNN_EXECUTORCH_LOG_WARN("[FCB] Failed to free DLC handle."); + } + fcb_dlc_handle_ = nullptr; + } if (sys_context_handle_ != nullptr) { - const QnnSystemInterface& qnn_sys_interface = - qnn_sys_impl_->GetQnnSystemInterface(); - error = qnn_sys_interface.qnn_system_context_free(sys_context_handle_); - if (error != QNN_SUCCESS) { + if (qnn_sys_interface.qnn_system_context_free(sys_context_handle_) != + QNN_SUCCESS) { QNN_EXECUTORCH_LOG_WARN("Failed to free QNN system context."); } sys_context_handle_ = nullptr; diff --git a/backends/qualcomm/runtime/backends/QnnBackendCache.h b/backends/qualcomm/runtime/backends/QnnBackendCache.h index 0f09855e3d7..297c6049132 100644 --- a/backends/qualcomm/runtime/backends/QnnBackendCache.h +++ b/backends/qualcomm/runtime/backends/QnnBackendCache.h @@ -28,8 +28,11 @@ class QnnBackendCache { }; explicit QnnBackendCache( const QnnExecuTorchContextBinary& qnn_context_blob, - QnnSystemImplementation* qnn_sys_impl) - : qnn_sys_impl_(qnn_sys_impl), qnn_context_blob_(qnn_context_blob) {} + QnnSystemImplementation* qnn_sys_impl, + bool is_fcb = false) + : qnn_sys_impl_(qnn_sys_impl), + qnn_context_blob_(qnn_context_blob), + is_fcb_(is_fcb) {} virtual ~QnnBackendCache(); QnnBackendCache(const QnnBackendCache&) = delete; QnnBackendCache(QnnBackendCache&&) = delete; @@ -74,6 +77,7 @@ class QnnBackendCache { executorch::runtime::Error GetQnnGraphInfoFromBinary( void* buffer, uint32_t nbytes); + executorch::runtime::Error GetQnnGraphInfoFromDlc(); template void RetrieveGraphInfo(const INFO& info); @@ -82,6 +86,8 @@ class QnnBackendCache { QnnExecuTorchContextBinary qnn_context_blob_; QnnSystemContext_Handle_t sys_context_handle_{nullptr}; + bool is_fcb_; + QnnSystemDlc_Handle_t fcb_dlc_handle_{nullptr}; std::vector graph_names_; std::unordered_map> input_tensor_structs_; diff --git a/backends/qualcomm/runtime/backends/QnnBackendCommon.h b/backends/qualcomm/runtime/backends/QnnBackendCommon.h index e146a67d772..da9ce5232d7 100644 --- a/backends/qualcomm/runtime/backends/QnnBackendCommon.h +++ b/backends/qualcomm/runtime/backends/QnnBackendCommon.h @@ -17,7 +17,6 @@ #include "HTP/QnnHtpCommon.h" #include "QnnBackend.h" -#include "QnnCommon.h" #include "QnnTypes.h" #include "Saver/QnnSaverCommon.h" diff --git a/backends/qualcomm/runtime/backends/QnnBackendFactory.cpp b/backends/qualcomm/runtime/backends/QnnBackendFactory.cpp index 1b2a89eceb0..91ab88dcbf6 100644 --- a/backends/qualcomm/runtime/backends/QnnBackendFactory.cpp +++ b/backends/qualcomm/runtime/backends/QnnBackendFactory.cpp @@ -63,7 +63,9 @@ std::unique_ptr QnnBackendFactory::Create( } backend_params->qnn_backend_cache_ptr_ = std::make_unique( - qnn_context_blob, system_implementation_ptr); + qnn_context_blob, + system_implementation_ptr, + options->fcb_options() != nullptr); backend_params->qnn_context_ptr_ = std::make_unique( implementation_ptr, @@ -72,6 +74,7 @@ std::unique_ptr QnnBackendFactory::Create( qnn_device_ptr, backend_params->qnn_backend_cache_ptr_.get(), htp_options, + options->fcb_options(), qnn_dlc_manager, get_option(options->profile_level(), QNN_RUNTIME_PROFILE_LEVEL)); diff --git a/backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.cpp b/backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.cpp index 3c31de6bbe3..e29eea85d90 100644 --- a/backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.cpp +++ b/backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.cpp @@ -50,6 +50,8 @@ Error QnnBackendUnifiedRegistry::GetOrCreateBackendBundle( get_option(options->log_level(), QNN_RUNTIME_LOG_LEVEL); QnnExecuTorchBackendType backend_type = options->backend_options()->backend_type(); + const auto bundle_key = + std::make_pair(backend_type, options->soc_info()->soc_model()); if (current_lib_path.empty()) { switch (backend_type) { @@ -76,7 +78,7 @@ Error QnnBackendUnifiedRegistry::GetOrCreateBackendBundle( } // Check if resources already exist - auto it = qnn_backend_bundles_map_.find(backend_type); + auto it = qnn_backend_bundles_map_.find(bundle_key); if (it != qnn_backend_bundles_map_.end()) { // Create new shared_ptr that shares ownership of the managed object. if (auto existing_bundle = it->second.lock()) { @@ -171,8 +173,7 @@ Error QnnBackendUnifiedRegistry::GetOrCreateBackendBundle( // weak_ptr under this key, and emplace will not replace it, so every later // request would miss the cache and build another backend for the same type. // CleanupExpired() cannot be used from here -- it takes mutex_, already held. - qnn_backend_bundles_map_.insert_or_assign( - backend_type, bundle); // Store weak_ptr to the bundle + qnn_backend_bundles_map_.insert_or_assign(bundle_key, bundle); return Error::Ok; } diff --git a/backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.h b/backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.h index 9d6fce27631..de4f143db23 100644 --- a/backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.h +++ b/backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.h @@ -17,6 +17,7 @@ #include +#include #include #include #include @@ -109,9 +110,9 @@ class QnnBackendUnifiedRegistry { } } - // Stores the collection of shared resources, with backend_type being used as - // the key. - std::unordered_map> + std::map< + std::pair, + std::weak_ptr> qnn_backend_bundles_map_; std::mutex mutex_; // Protects access to resources and ensures atomic diff --git a/backends/qualcomm/runtime/backends/QnnFunctionInterface.h b/backends/qualcomm/runtime/backends/QnnFunctionInterface.h index 33b3bd808e5..6249b0f3fb1 100644 --- a/backends/qualcomm/runtime/backends/QnnFunctionInterface.h +++ b/backends/qualcomm/runtime/backends/QnnFunctionInterface.h @@ -7,6 +7,8 @@ */ #pragma once +#include + #include "QnnInterface.h" #include "Saver/QnnSaver.h" @@ -59,6 +61,9 @@ class QnnInterface { context_create_from_binary, contextCreateFromBinary); DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); +#if QNN_EXECUTORCH_SUPPORTS_FCB + DEFINE_SHIM_FUNCTION_INTERFACE(context_add_to_dlc, contextAddToDlc); +#endif // --------- QnnGraph --------- DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); diff --git a/backends/qualcomm/runtime/backends/QnnGraphCommon.h b/backends/qualcomm/runtime/backends/QnnGraphCommon.h index 70d4a59f465..ae273d97e16 100644 --- a/backends/qualcomm/runtime/backends/QnnGraphCommon.h +++ b/backends/qualcomm/runtime/backends/QnnGraphCommon.h @@ -15,7 +15,6 @@ #include -#include "QnnCommon.h" namespace executorch { namespace backends { namespace qnn { diff --git a/backends/qualcomm/runtime/backends/QnnSdkCompatibility.h b/backends/qualcomm/runtime/backends/QnnSdkCompatibility.h new file mode 100644 index 00000000000..0b0ea038889 --- /dev/null +++ b/backends/qualcomm/runtime/backends/QnnSdkCompatibility.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) Qualcomm Innovation Center, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#pragma once + +#include "QnnCommon.h" + +// QnnCommon.h exposes the QNN API ABI version, not the QAIRT SDK product +// release. Use this helper for compile-time header/API availability checks. +#define QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(major, minor) \ + ((QNN_API_VERSION_MAJOR > (major)) || \ + (QNN_API_VERSION_MAJOR == (major) && QNN_API_VERSION_MINOR >= (minor))) + +// Product-release requirement: FCB requires QAIRT SDK 2.48 or newer. That +// release provides QNN API 2.37, which adds the required core, System DLC, +// and HTP declarations. Keep this mapping here when a new SDK release changes +// the FCB interface requirement. +#define QNN_EXECUTORCH_FCB_MIN_QAIRT_SDK_VERSION "2.48" + +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 37) +#define QNN_EXECUTORCH_SUPPORTS_FCB 1 +#else +#define QNN_EXECUTORCH_SUPPORTS_FCB 0 +#endif diff --git a/backends/qualcomm/runtime/backends/QnnSysFunctionInterface.h b/backends/qualcomm/runtime/backends/QnnSysFunctionInterface.h index 4dc0e6a8b2b..006aad2b6cf 100644 --- a/backends/qualcomm/runtime/backends/QnnSysFunctionInterface.h +++ b/backends/qualcomm/runtime/backends/QnnSysFunctionInterface.h @@ -7,6 +7,8 @@ */ #pragma once +#include + #include "System/QnnSystemInterface.h" #include @@ -49,6 +51,24 @@ class QnnSystemInterface { system_dlc_create_from_binary, systemDlcCreateFromBinary); DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_dlc_free, systemDlcFree); +#if QNN_EXECUTORCH_SUPPORTS_FCB + DEFINE_SHIM_FUNCTION_SYS_INTERFACE( + system_dlc_create_with_destination_dir, + systemDlcCreateWithDestinationDir); + DEFINE_SHIM_FUNCTION_SYS_INTERFACE( + system_dlc_get_binary_size, + systemDlcGetBinarySize); + DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_dlc_get_binary, systemDlcGetBinary); + DEFINE_SHIM_FUNCTION_SYS_INTERFACE( + system_dlc_get_records_by_type, + systemDlcGetRecordsByType); + DEFINE_SHIM_FUNCTION_SYS_INTERFACE( + system_dlc_get_record_data_size, + systemDlcGetRecordDataSize); + DEFINE_SHIM_FUNCTION_SYS_INTERFACE( + system_dlc_read_record_data_memory_mapped, + systemDlcReadRecordDataMemoryMapped); +#endif private: const QnnSystemInterface_t* qnn_sys_interface_{nullptr}; diff --git a/backends/qualcomm/runtime/backends/htp/HtpBackendCache.cpp b/backends/qualcomm/runtime/backends/htp/HtpBackendCache.cpp index be95c9cfdbf..e989e0c7812 100644 --- a/backends/qualcomm/runtime/backends/htp/HtpBackendCache.cpp +++ b/backends/qualcomm/runtime/backends/htp/HtpBackendCache.cpp @@ -17,7 +17,7 @@ using executorch::runtime::Error; Error HtpBackendCache::RetrieveBackendBinaryInfo( const QnnSystemContext_BinaryInfo_t* binaryinfo) { QnnHtpSystemContext_HwBlobInfo_t* htp_hwblobinfo = nullptr; -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 21) std::vector htp_graphblobinfos; std::uint32_t num_graphs; @@ -29,7 +29,7 @@ Error HtpBackendCache::RetrieveBackendBinaryInfo( } else if (binaryinfo->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_2) { htp_hwblobinfo = static_cast( binaryinfo->contextBinaryInfoV2.hwInfoBlob); -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 21) } else if (binaryinfo->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_3) { num_graphs = binaryinfo->contextBinaryInfoV3.numGraphs; for (size_t i = 0; i < num_graphs; ++i) { @@ -57,7 +57,7 @@ Error HtpBackendCache::RetrieveBackendBinaryInfo( } } -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 21) if (htp_graphblobinfos.size() > 0) { // After version 2.21, we need to get spill fill buffer size from graph // blob info instead of hw blob info. If there are multiple graphs, we diff --git a/backends/qualcomm/runtime/backends/htp/HtpBackendCache.h b/backends/qualcomm/runtime/backends/htp/HtpBackendCache.h index 3a39cfcaa81..b178622c041 100644 --- a/backends/qualcomm/runtime/backends/htp/HtpBackendCache.h +++ b/backends/qualcomm/runtime/backends/htp/HtpBackendCache.h @@ -15,8 +15,10 @@ class HtpBackendCache : public QnnBackendCache { public: explicit HtpBackendCache( const QnnExecuTorchContextBinary& qnn_context_blob, - QnnSystemImplementation* qnn_sys_impl) - : QnnBackendCache(qnn_context_blob, qnn_sys_impl), spill_fill_buf_(0) {} + QnnSystemImplementation* qnn_sys_impl, + bool is_fcb) + : QnnBackendCache(qnn_context_blob, qnn_sys_impl, is_fcb), + spill_fill_buf_(0) {} ~HtpBackendCache() override = default; uint64_t GetSpillFillBufferSize() { diff --git a/backends/qualcomm/runtime/backends/htp/HtpContext.h b/backends/qualcomm/runtime/backends/htp/HtpContext.h index a18559f2e82..c98af1c3f14 100644 --- a/backends/qualcomm/runtime/backends/htp/HtpContext.h +++ b/backends/qualcomm/runtime/backends/htp/HtpContext.h @@ -26,6 +26,7 @@ class HtpContext : public QnnContext { QnnDevice* device, QnnBackendCache* cache, const QnnExecuTorchHtpBackendOptions* htp_options, + const QnnExecuTorchFcbOptions* fcb_options, QnnDlcManager* qnn_dlc_manager, const QnnExecuTorchProfileLevel& profile_level) : QnnContext( @@ -37,7 +38,7 @@ class HtpContext : public QnnContext { qnn_dlc_manager, profile_level) { htp_context_custom_config_ = std::make_unique( - this, htp_options, profile_level); + this, htp_options, fcb_options, profile_level); } ~HtpContext() {} diff --git a/backends/qualcomm/runtime/backends/htp/HtpContextCustomConfig.h b/backends/qualcomm/runtime/backends/htp/HtpContextCustomConfig.h index 61a395fcb5b..5cad6b8c4f7 100644 --- a/backends/qualcomm/runtime/backends/htp/HtpContextCustomConfig.h +++ b/backends/qualcomm/runtime/backends/htp/HtpContextCustomConfig.h @@ -27,10 +27,12 @@ class HtpContextCustomConfig { explicit HtpContextCustomConfig( const QnnContext* context, const QnnExecuTorchHtpBackendOptions* htp_options, + const QnnExecuTorchFcbOptions* fcb_options, const QnnExecuTorchProfileLevel& profile_level) : profile_level_(profile_level), context_(context), - htp_options_(htp_options) {} + htp_options_(htp_options), + fcb_options_(fcb_options) {} std::vector CreateContextCustomConfig(); @@ -50,6 +52,7 @@ class HtpContextCustomConfig { std::vector> htp_context_config_; [[maybe_unused]] const QnnExecuTorchHtpBackendOptions* htp_options_; + [[maybe_unused]] const QnnExecuTorchFcbOptions* fcb_options_; }; } // namespace qnn diff --git a/backends/qualcomm/runtime/backends/htp/host/HtpContextCustomConfig.cpp b/backends/qualcomm/runtime/backends/htp/host/HtpContextCustomConfig.cpp index 037998132a8..72a9f0ef6d8 100644 --- a/backends/qualcomm/runtime/backends/htp/host/HtpContextCustomConfig.cpp +++ b/backends/qualcomm/runtime/backends/htp/host/HtpContextCustomConfig.cpp @@ -8,6 +8,7 @@ #include #include +#include namespace executorch { namespace backends { @@ -25,6 +26,19 @@ HtpContextCustomConfig::CreateContextCustomConfig() { p_custom_config->weightSharingEnabled = true; ret.push_back(static_cast(p_custom_config)); } + if (fcb_options_ != nullptr && fcb_options_->fcb_reference_weight_sharing()) { +#if QNN_EXECUTORCH_SUPPORTS_FCB + p_custom_config = AllocContextCustomConfig(); + p_custom_config->option = + QNN_HTP_CONTEXT_CONFIG_OPTION_REFERENCE_WEIGHT_SHARING_ENABLED; + p_custom_config->referenceWeightSharingEnabled = true; + ret.push_back(static_cast(p_custom_config)); +#else + QNN_EXECUTORCH_LOG_ERROR( + "FCB reference weight sharing is not supported by this QNN SDK; " + "Compilation with QAIRT SDK 2.48 or newer is required."); +#endif + } return ret; } diff --git a/backends/qualcomm/runtime/backends/ir/IrBackend.h b/backends/qualcomm/runtime/backends/ir/IrBackend.h index 72bb59c84f9..55e4be5d6df 100644 --- a/backends/qualcomm/runtime/backends/ir/IrBackend.h +++ b/backends/qualcomm/runtime/backends/ir/IrBackend.h @@ -8,7 +8,7 @@ #pragma once #include -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 23) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 23) #include "IR/QnnIrCommon.h" #endif #include "QnnTypes.h" @@ -24,7 +24,7 @@ class IrBackend : public QnnBackend { Qnn_Version_t GetExpectedBackendVersion() const override { Qnn_Version_t backend_version; -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 23) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 23) backend_version.major = QNN_IR_API_VERSION_MAJOR; backend_version.minor = QNN_IR_API_VERSION_MINOR; backend_version.patch = QNN_IR_API_VERSION_PATCH; diff --git a/backends/qualcomm/runtime/backends/lpai/LpaiBackendCustomConfig.cpp b/backends/qualcomm/runtime/backends/lpai/LpaiBackendCustomConfig.cpp index 7a2f6a1c11a..c59d3c8f80a 100644 --- a/backends/qualcomm/runtime/backends/lpai/LpaiBackendCustomConfig.cpp +++ b/backends/qualcomm/runtime/backends/lpai/LpaiBackendCustomConfig.cpp @@ -42,7 +42,7 @@ LpaiBackendCustomConfig::CreateBackendCustomConfig() { std::unordered_map lpai_hw_ver = { {LpaiHardwareVersion::V6, QNN_LPAI_BACKEND_HW_VERSION_V6}, -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 29) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 29) {LpaiHardwareVersion::V7, QNN_LPAI_BACKEND_HW_VERSION_V7}, #endif }; diff --git a/backends/qualcomm/runtime/backends/lpai/LpaiGraph.cpp b/backends/qualcomm/runtime/backends/lpai/LpaiGraph.cpp index 7373ceff8d8..33c8fba898e 100644 --- a/backends/qualcomm/runtime/backends/lpai/LpaiGraph.cpp +++ b/backends/qualcomm/runtime/backends/lpai/LpaiGraph.cpp @@ -126,7 +126,7 @@ Error LpaiGraph::AfterRetrieveGraph(const std::string& graph_name) { Error LpaiGraph::AfterCreateGraph(const std::string& graph_name) { std::vector graph_custom_config; -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 29) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 29) QnnLpaiGraph_CustomConfig_t* p_custom_config = nullptr; p_custom_config = AllocGraphCustomConfig(); diff --git a/backends/qualcomm/runtime/backends/lpai/LpaiGraph.h b/backends/qualcomm/runtime/backends/lpai/LpaiGraph.h index f6380c20376..07a885d8058 100644 --- a/backends/qualcomm/runtime/backends/lpai/LpaiGraph.h +++ b/backends/qualcomm/runtime/backends/lpai/LpaiGraph.h @@ -90,7 +90,7 @@ class LpaiGraph : public QnnGraph { lpai_prepare_.emplace_back( std::make_unique()); lpai_prepare_.back()->enablePerLayer = 0; -#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 29) +#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 29) lpai_prepare_.back()->enableCoreSelection = nullptr; #endif return lpai_prepare_.back().get(); diff --git a/backends/qualcomm/serialization/qc_compiler_spec.fbs b/backends/qualcomm/serialization/qc_compiler_spec.fbs index 4d3d59c4fbe..0c133dd8b06 100644 --- a/backends/qualcomm/serialization/qc_compiler_spec.fbs +++ b/backends/qualcomm/serialization/qc_compiler_spec.fbs @@ -325,6 +325,22 @@ table QnnExecuTorchBackendOptions { lpai_options:QnnExecuTorchLpaiBackendOptions; } +/// One HTP target in an offline-prepared multi-SoC DLC. +table QnnExecuTorchFcbTarget { + soc_info:SocInfo; + htp_options:QnnExecuTorchHtpBackendOptions; +} + +/// Options for an offline-prepared multi-SoC HTP DLC. +table QnnExecuTorchFcbOptions { + /// One SoC and HTP-option pair per context binary. + targets:[QnnExecuTorchFcbTarget]; + + /// Enable host-AOT reference-weight sharing while appending FCB contexts. + /// Has no effect unless fcb_options is present. Default: true. + fcb_reference_weight_sharing:bool = true; +} + table QnnExecuTorchOptions { /// Specify SoC to compile or execute for. soc_info:SocInfo; @@ -365,6 +381,9 @@ table QnnExecuTorchOptions { /// This experimental parameter is used to decide whether to enable multi-head attention to single-head attention pass, aiming to reduce time consumption in AOT and improve performance on HTP. use_mha2sha:bool; + + /// Emit a multi-SoC DLC instead of one device context binary. + fcb_options:QnnExecuTorchFcbOptions; } root_type QnnExecuTorchOptions; diff --git a/backends/qualcomm/serialization/qc_schema.py b/backends/qualcomm/serialization/qc_schema.py index b3992bc97f1..f9b70d7c22d 100644 --- a/backends/qualcomm/serialization/qc_schema.py +++ b/backends/qualcomm/serialization/qc_schema.py @@ -288,6 +288,19 @@ class QnnExecuTorchOpPackageOptions: op_package_infos: List[QnnExecuTorchOpPackageInfo] = field(default_factory=list) +@dataclass +class QnnExecuTorchFcbTarget: + soc_info: SocInfo + htp_options: QnnExecuTorchHtpBackendOptions + + +@dataclass +class QnnExecuTorchFcbOptions: + targets: List[QnnExecuTorchFcbTarget] = field(default_factory=list) + # Applies only while appending host-AOT contexts to an FCB DLC. + fcb_reference_weight_sharing: bool = True + + @dataclass class QnnExecuTorchOptions: soc_info: SocInfo @@ -305,3 +318,4 @@ class QnnExecuTorchOptions: default_factory=QnnExecuTorchOpPackageOptions ) use_mha2sha: bool = False + fcb_options: Optional[QnnExecuTorchFcbOptions] = None diff --git a/backends/qualcomm/tests/test_qnn_compiler_spec.py b/backends/qualcomm/tests/test_qnn_compiler_spec.py new file mode 100644 index 00000000000..711f85be95b --- /dev/null +++ b/backends/qualcomm/tests/test_qnn_compiler_spec.py @@ -0,0 +1,113 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest + +from executorch.backends.qualcomm.serialization.qc_schema import ( + QcomChipset, + QnnExecuTorchBackendType, +) +from executorch.backends.qualcomm.serialization.qc_schema_serialize import ( + flatbuffer_to_option, +) +from executorch.backends.qualcomm.utils.utils import ( + generate_gpu_compiler_spec, + generate_htp_compiler_spec, + generate_qnn_executorch_compiler_spec, +) + + +class TestQnnCompilerSpec(unittest.TestCase): + def test_fcb_pairs_soc_and_htp_options(self): + first = generate_htp_compiler_spec(use_fp16=False) + second = generate_htp_compiler_spec(use_fp16=True) + option = flatbuffer_to_option( + generate_qnn_executorch_compiler_spec( + soc_model=[QcomChipset.SM8650, QcomChipset.SM8750], + backend_options=[first, second], + )[0].value + ) + + self.assertTrue(option.fcb_options.fcb_reference_weight_sharing) + self.assertEqual( + [target.soc_info.soc_model for target in option.fcb_options.targets], + [QcomChipset.SM8650, QcomChipset.SM8750], + ) + self.assertEqual( + [target.htp_options.precision for target in option.fcb_options.targets], + [first.htp_options.precision, second.htp_options.precision], + ) + + def test_fcb_rejects_invalid_target_configuration(self): + htp = generate_htp_compiler_spec(use_fp16=False) + cases = [ + ( + { + "soc_model": [QcomChipset.SM8650], + "backend_options": [htp], + }, + "at least two", + ), + ( + { + "soc_model": [QcomChipset.SM8650, QcomChipset.SM8750], + "backend_options": [htp], + }, + "equal-length", + ), + ( + { + "soc_model": [QcomChipset.SM8650, QcomChipset.SM8750], + "backend_options": [htp, generate_gpu_compiler_spec()], + }, + "HTP", + ), + ( + { + "soc_model": [QcomChipset.SM8650, QcomChipset.SM8750], + "backend_options": [htp, htp], + "online_prepare": True, + }, + "offline_prepare", + ), + ( + { + "soc_model": QcomChipset.SM8650, + "backend_options": [htp, htp], + }, + "lists", + ), + ] + for kwargs, message in cases: + with self.subTest(kwargs=kwargs), self.assertRaisesRegex( + ValueError, message + ): + generate_qnn_executorch_compiler_spec(**kwargs) + + def test_fcb_rejects_dlbc_when_reference_sharing_enabled(self): + htp = generate_htp_compiler_spec(use_fp16=False, use_dlbc=True) + with self.assertRaisesRegex(ValueError, "DLBC"): + generate_qnn_executorch_compiler_spec( + soc_model=[QcomChipset.SM8650, QcomChipset.SM8750], + backend_options=[htp, htp], + ) + + def test_single_target_remains_non_fcb(self): + option = flatbuffer_to_option( + generate_qnn_executorch_compiler_spec( + soc_model=QcomChipset.SM8650, + backend_options=generate_htp_compiler_spec(use_fp16=False), + )[0].value + ) + self.assertIsNone(option.fcb_options) + self.assertEqual( + option.backend_options.backend_type, + QnnExecuTorchBackendType.kHtpBackend, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/qualcomm/tests/test_qnn_manager_lifecycle.py b/backends/qualcomm/tests/test_qnn_manager_lifecycle.py new file mode 100644 index 00000000000..5f4a5b3aeac --- /dev/null +++ b/backends/qualcomm/tests/test_qnn_manager_lifecycle.py @@ -0,0 +1,63 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest +from unittest.mock import Mock, patch + +from executorch.backends.qualcomm.serialization.qc_schema import QcomChipset +from executorch.backends.qualcomm.utils import qnn_manager_lifecycle as lifecycle +from executorch.backends.qualcomm.utils.utils import ( + generate_htp_compiler_spec, + generate_qnn_executorch_compiler_spec, +) + + +class TestQnnManagerLifecycle(unittest.TestCase): + def setUp(self): + lifecycle._current_qnn_managers.active_registry = None + self.addCleanup( + setattr, lifecycle._current_qnn_managers, "active_registry", None + ) + + def test_lazy_fcb_lookup_creates_each_target_once(self): + specs = generate_qnn_executorch_compiler_spec( + soc_model=[QcomChipset.SM8650, QcomChipset.SM8750], + backend_options=[ + generate_htp_compiler_spec(use_fp16=False), + generate_htp_compiler_spec(use_fp16=True), + ], + ) + managers = [Mock(), Mock()] + for manager in managers: + manager.InitBackend.return_value = Mock(value=0) + + with patch.object( + lifecycle.PyQnnManager, "QnnManager", side_effect=managers + ) as create: + first = lifecycle.get_current_qnn_manager(specs, QcomChipset.SM8650) + second = lifecycle.get_current_qnn_manager(specs, QcomChipset.SM8750) + self.assertIs( + first, lifecycle.get_current_qnn_manager(specs, QcomChipset.SM8650) + ) + + self.assertEqual(create.call_count, 2) + self.assertIs(first, managers[0]) + self.assertIs(second, managers[1]) + + def test_fcb_lookup_rejects_unknown_target(self): + specs = generate_qnn_executorch_compiler_spec( + soc_model=[QcomChipset.SM8650, QcomChipset.SM8750], + backend_options=[ + generate_htp_compiler_spec(use_fp16=False), + generate_htp_compiler_spec(use_fp16=False), + ], + ) + with self.assertRaisesRegex(ValueError, "do not target SM8850"): + lifecycle.get_current_qnn_manager(specs, QcomChipset.SM8850) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/qualcomm/utils/qnn_manager_lifecycle.py b/backends/qualcomm/utils/qnn_manager_lifecycle.py index c5f42043d94..b36c7f78ffe 100644 --- a/backends/qualcomm/utils/qnn_manager_lifecycle.py +++ b/backends/qualcomm/utils/qnn_manager_lifecycle.py @@ -1,15 +1,17 @@ import contextlib -import logging +import copy import threading from typing import Dict, List import executorch.backends.qualcomm.python.PyQnnManagerAdaptor as PyQnnManager from executorch.backends.qualcomm.partition.utils import generate_qnn_executorch_option from executorch.backends.qualcomm.serialization.qc_schema import ( + QcomChipset, QnnExecuTorchBackendType, ) from executorch.backends.qualcomm.serialization.qc_schema_serialize import ( flatbuffer_to_option, + option_to_flatbuffer, ) from executorch.backends.qualcomm.utils.qnn_sdk_setup import ( disable_mkldnn_on_amd, @@ -23,85 +25,102 @@ class QnnManagerRegistry: def __init__(self): - # Registry stores {backend_type: QnnManager instance} self._registry = {} def get_or_create_qnn_manager( - self, backend_type: QnnExecuTorchBackendType, option: bytes + self, backend_type: QnnExecuTorchBackendType, option: bytes, soc_model=None ) -> PyQnnManager.QnnManager: # Outside the branch below, so reusing a cached manager still re-applies them. Both are # cheap on a repeat call, and the AMD guard has to hold for every lowering, not only the # one that happened to build the manager. setup_qnn_sdk() disable_mkldnn_on_amd() - - if backend_type not in self._registry: + key = (backend_type, soc_model) + if key not in self._registry: qnn_manager = PyQnnManager.QnnManager(option) err = qnn_manager.InitBackend() if err.value != 0: + breakpoint() raise RuntimeError( f"Failed to initialize QNN backend for {backend_type.name}. " "Ensure QNN SDK libraries are available " "(e.g. LD_LIBRARY_PATH includes $QNN_SDK_ROOT/lib/x86_64-linux-clang/)." ) - self._registry[backend_type] = qnn_manager - return self._registry[backend_type] - - def destroy_qnn_manager(self, backend_type: QnnExecuTorchBackendType): - if backend_type in self._registry: - self._registry[backend_type].Destroy() - del self._registry[backend_type] - else: - logging.warning( - f"Attempted to destroy non-existent QnnManager for backend type {backend_type.name}" + self._registry[key] = qnn_manager + return self._registry[key] + + def destroy_all(self): + for qnn_manager in self._registry.values(): + qnn_manager.Destroy() + self._registry.clear() + + +def _get_target_option( + compile_specs: List[CompileSpec], soc_model: QcomChipset | None +) -> tuple[QnnExecuTorchBackendType, QcomChipset, bytes]: + option = generate_qnn_executorch_option(compile_specs) + python_options = flatbuffer_to_option(option) + fcb_options = python_options.fcb_options + if fcb_options is None: + target_soc_model = python_options.soc_info.soc_model + if soc_model is not None and soc_model != target_soc_model: + raise ValueError(f"compile specs do not target {soc_model.name}") + return python_options.backend_options.backend_type, target_soc_model, option + for target in fcb_options.targets: + if target.soc_info.soc_model == soc_model: + target_options = copy.deepcopy(python_options) + target_options.soc_info = target.soc_info + target_options.backend_options.htp_options = target.htp_options + return ( + target_options.backend_options.backend_type, + soc_model, + option_to_flatbuffer(target_options), ) + if soc_model is None: + raise ValueError("FCB manager lookup requires soc_model") + raise ValueError(f"FCB compile specs do not target {soc_model.name}") + + +def _get_current_registry() -> QnnManagerRegistry: + active_registry = getattr(_current_qnn_managers, "active_registry", None) + if active_registry is None: + active_registry = QnnManagerRegistry() + _current_qnn_managers.active_registry = active_registry + return active_registry @contextlib.contextmanager def QnnManagerContext(compile_specs: Dict[str, List[CompileSpec]]): - # Create a new registry for the current context current_context_registry = QnnManagerRegistry() + previous_registry = getattr(_current_qnn_managers, "active_registry", None) _current_qnn_managers.active_registry = current_context_registry - - backend_types_in_this_context = set() - try: for compile_spec_list in compile_specs.values(): - option = generate_qnn_executorch_option(compile_spec_list) - python_options = flatbuffer_to_option(option) - backend_type = python_options.backend_options.backend_type - - # Use the current_context_registry to get/create the manager - current_context_registry.get_or_create_qnn_manager(backend_type, option) - backend_types_in_this_context.add(backend_type) + option = flatbuffer_to_option( + generate_qnn_executorch_option(compile_spec_list) + ) + targets = ( + [target.soc_info.soc_model for target in option.fcb_options.targets] + if option.fcb_options is not None + else [option.soc_info.soc_model] + ) + for soc_model in targets: + get_current_qnn_manager(compile_spec_list, soc_model) yield finally: - # Destroy only the managers created within this context - for backend_type in backend_types_in_this_context: - current_context_registry.destroy_qnn_manager(backend_type) - - # Clear the active registry reference - _current_qnn_managers.active_registry = None + current_context_registry.destroy_all() + _current_qnn_managers.active_registry = previous_registry def get_current_qnn_manager( - backend_type: QnnExecuTorchBackendType, compile_specs: List[CompileSpec] + compile_specs: List[CompileSpec], soc_model: QcomChipset | None = None ) -> PyQnnManager.QnnManager: - """ - Retrieves the QnnManager instance active for the current QnnManagerContext invocation. - Return a new QnnManger if no QnnManager is active for the given backend_type in the current context. - """ - active_registry = getattr(_current_qnn_managers, "active_registry", None) - if active_registry is None or backend_type not in active_registry._registry: - logging.warning( - f"No QnnManager active for backend type {backend_type.name} in the current QnnManagerContext. " - "It would be better to use to_edge_transform_and_lower_to_qnn to lowering to QNN Backend." - ) - return QnnManagerRegistry().get_or_create_qnn_manager( - backend_type, generate_qnn_executorch_option(compile_specs) - ) - + backend_type, target_soc_model, option = _get_target_option( + compile_specs, soc_model + ) # Re-applied even though the manager already exists, because a caller may have turned the # setting back on since it was built, and this is a lowering about to run. disable_mkldnn_on_amd() - return active_registry._registry[backend_type] + return _get_current_registry().get_or_create_qnn_manager( + backend_type, option, target_soc_model + ) diff --git a/backends/qualcomm/utils/utils.py b/backends/qualcomm/utils/utils.py index 75ef9c1f128..f341430d605 100644 --- a/backends/qualcomm/utils/utils.py +++ b/backends/qualcomm/utils/utils.py @@ -33,6 +33,8 @@ QcomChipset, QnnExecuTorchBackendOptions, QnnExecuTorchBackendType, + QnnExecuTorchFcbOptions, + QnnExecuTorchFcbTarget, QnnExecuTorchGpuBackendOptions, QnnExecuTorchGpuPrecision, QnnExecuTorchHtpBackendOptions, @@ -1176,8 +1178,8 @@ def generate_lpai_compiler_spec( def generate_qnn_executorch_compiler_spec( # noqa: C901 - soc_model: QcomChipset, - backend_options: QnnExecuTorchBackendOptions, + soc_model: Union[QcomChipset | List[QcomChipset]], + backend_options: QnnExecuTorchBackendOptions | List[QnnExecuTorchBackendOptions], debug: bool = False, saver: bool = False, online_prepare: bool = False, @@ -1187,48 +1189,91 @@ def generate_qnn_executorch_compiler_spec( # noqa: C901 is_from_context_binary: bool = False, op_package_options: QnnExecuTorchOpPackageOptions = None, use_mha2sha: bool = False, + fcb_reference_weight_sharing: bool = True, ) -> List[CompileSpec]: """ - Helper function generating compiler specs for Qualcomm AI Engine Direct + Helper function generating compiler specs for Qualcomm AI Engine Direct. Args: - soc_model: The SoC you plan to run the compiled model. Please check - QcomChipset for supported SoC. - SM8450 (Snapdragon 8 Gen 1) - SM8475(Snapdragon 8 Gen 1+) - SM8550(Snapdragon 8 Gen 2) - SM8650(Snapdragon 8 Gen 3) - SM8750(Snapdragon 8 Elite) - SM8850(Snapdragon 8 Elite Gen 5) - backend_options: Options required by different backends. + soc_model: The SoC you plan to run the compiled model. Pass one + ``QcomChipset`` with scalar ``backend_options`` for a normal target. + Check a connected Android device with ``adb shell getprop ro.soc.model`` + (Refer to qc_schema.py for a complete list of support soc). + Pass an ordered list with an equal-length ``backend_options`` list to + create an Flexible Context Binary (FCB). Each position + supplies the SoC and HTP options for one context in the output DLC. + Use FCB to ship one artifact to a known set of HTP SoCs; it is not a + runtime backend fallback. + backend_options: Options required by different backends. In FCB mode, + each entry must select HTP and provide HTP options for its paired SoC. debug: Enable verbose logging. Disclaimer: this option must change in the near future. - online_prepare: Compose QNN graph on device if set to True + online_prepare: Compose QNN graph on device if set to True. FCB requires + offline preparation. saver: Instead of compiling the model, run QNN Saver. Please check documents of Qualcomm AI Engine Direct SDK. This feature is usually for debugging purpose. - dump_intermediate_outputs: If tensor dump is enabled, all intermediate tensors output will be dumped. - This option exists for debugging accuracy issues - profile_level: Enable profiling the performance of per operator. - Note that for now only support kProfileDetailed and kProfileOptrace. - shared_buffer: Enables usage of shared buffer between application - and backend for graph I/O. - is_from_context_binary: True if current graph comes from pre-built context binary. - op_package_options: Optional structure to specify op packages - loaded and used by the backend. - use_mha2sha: This experimental parameter is used to decide whether to enable multi-head attention to single-head attention pass, aiming to reduce time consumption in AOT and improve performance on HTP. + dump_intermediate_outputs: If tensor dump is enabled, all intermediate + tensors output will be dumped. This option exists for debugging + accuracy issues. + profile_level: Enable profiling the performance of per operator. Note + that for now only kProfileDetailed and kProfileOptrace are supported. + shared_buffer: Enables usage of shared buffer between application and + backend for graph I/O. + is_from_context_binary: True if current graph comes from pre-built + context binary. + op_package_options: Optional structure to specify op packages loaded and + used by the backend. + use_mha2sha: Experimental switch for the multi-head-attention to + single-head-attention pass. + fcb_reference_weight_sharing: Enable reference-weight sharing while the + host AOT flow appends FCB contexts to its DLC. Enabled by default; + has no effect for a scalar target. It cannot be combined with DLBC. + This differs from ``use_weight_sharing``, which shares weights + between multiple graphs in one HTP context. Returns: List[CompileSpec]: Compiler specs for Qualcomm AI Engine Direct. Raises: - ValueError: The value QcomChipset is currently not supported. - ValueError: Confliction between compiler specs. + ValueError: The requested SoC, target pairing, or backend configuration + is unsupported. FCB requires at least two unique HTP SoCs, no online + preparation, and no DLBC when reference-weight sharing is enabled. """ - _supported_soc_models = {soc_model.value for soc_model in QcomChipset} - if soc_model not in _supported_soc_models: - raise ValueError(f"unknown SoC model for QNN: {soc_model}") + # Normalize inputs + soc_models = soc_model if isinstance(soc_model, list) else [soc_model] + target_backend_options = ( + backend_options if isinstance(backend_options, list) else [backend_options] + ) + if len(soc_models) != len(target_backend_options): + raise ValueError( + "soc_model and backend_options must have the same number of entries" + ) + is_fcb = len(soc_models) > 1 + if len(set(soc_models)) != len(soc_models): + raise ValueError("soc_model must contain unique QcomChipset values") + if any(model not in _soc_info_table for model in soc_models): + raise ValueError(f"unknown SoC model for QNN: {soc_models}") + if is_fcb: + if online_prepare: + raise ValueError("FCB requires offline_prepare") + if any( + option.backend_type != QnnExecuTorchBackendType.kHtpBackend + or option.htp_options is None + for option in target_backend_options + ): + raise ValueError("FCB requires HTP backend options") + if fcb_reference_weight_sharing and any( + option.htp_options.use_dlbc for option in target_backend_options + ): + raise ValueError("FCB reference weight sharing does not support DLBC") + if is_qnn_sdk_version_less_than("2.48"): + raise ValueError( + "FCB requires QNN SDK version >= 2.48; " + f"current QNN SDK version: {describe_sdk_build_id()}" + ) + backend_options = target_backend_options[0] if profile_level and dump_intermediate_outputs: warnings.warn( "It is not recommended to turn on both profiling and dump_intermediate_outputs the same time" @@ -1237,8 +1282,16 @@ def generate_qnn_executorch_compiler_spec( # noqa: C901 ) qnn_executorch_options = QnnExecuTorchOptions( - _soc_info_table[soc_model], backend_options + _soc_info_table[soc_models[0]], backend_options ) + if is_fcb: + qnn_executorch_options.fcb_options = QnnExecuTorchFcbOptions( + targets=[ + QnnExecuTorchFcbTarget(_soc_info_table[model], option.htp_options) + for model, option in zip(soc_models, target_backend_options) + ], + fcb_reference_weight_sharing=fcb_reference_weight_sharing, + ) qnn_executorch_options.log_level = ( QnnExecuTorchLogLevel.kLogLevelDebug if debug @@ -1284,9 +1337,9 @@ def generate_qnn_executorch_compiler_spec( # noqa: C901 raise ValueError("LPAI does not support online prepare.") if backend_options.backend_type == QnnExecuTorchBackendType.kLpaiBackend: - if soc_model.name not in get_soc_to_lpai_hw_ver_map(): + if soc_models[0].name not in get_soc_to_lpai_hw_ver_map(): raise ValueError( - f"Target soc_model({soc_model.name}) doesn't support LPAI backend. \n" + f"Target soc_model({soc_models[0].name}) doesn't support LPAI backend. \n" "Please choose the following SOC: " f"{list(get_soc_to_lpai_hw_ver_map().keys())}" ) @@ -1296,10 +1349,10 @@ def generate_qnn_executorch_compiler_spec( # noqa: C901 # through QNN_SDK_ROOT. setup_qnn_sdk() if get_soc_to_lpai_hw_ver_map()[ - soc_model.name + soc_models[0].name ] == LpaiHardwareVersion.V6 and is_qnn_sdk_version_less_than("2.39"): raise ValueError( - f"Target soc_model({soc_model.name}) with LPAI backend v6 requires QNN SDK version >= 2.39. \n" + f"Target soc_model({soc_models[0].name}) with LPAI backend v6 requires QNN SDK version >= 2.39. \n" f"Current QNN SDK version: {describe_sdk_build_id()}" ) diff --git a/docs/source/backends-qualcomm.md b/docs/source/backends-qualcomm.md index cf9369cb214..3dcfbc517ba 100644 --- a/docs/source/backends-qualcomm.md +++ b/docs/source/backends-qualcomm.md @@ -189,6 +189,113 @@ cd $env:EXECUTORCH_ROOT > The script supports building both x64 and cross-compiling ARM64 target artifacts on Windows x64 host. After the build completes, the ARM64 libraries and executables can be copied to a Windows on Snapdragon (WoS) device using `scp`. > This allows a `.pte` generated on Windows x64 host to be executed on WoS device. +## Flexible Context Binary for multiple HTP SoCs + +A normal offline QNN context binary is specific to one SoC. A Flexible Context +Binary (FCB) is one offline-prepared HTP DLC containing one context for each +specified SoC. Use FCB when the same model must ship as one artifact to a known +set of HTP devices. The QNN runtime selects the device-compatible context from +the DLC. FCB requires QNN SDK version 2.48 or later. + +FCB is created by passing paired lists to +`generate_qnn_executorch_compiler_spec`. Each position describes one target: + +```python +from executorch.backends.qualcomm.serialization.qc_schema import QcomChipset +from executorch.backends.qualcomm.utils.utils import ( + generate_htp_compiler_spec, + generate_qnn_executorch_compiler_spec, +) + +compiler_specs = generate_qnn_executorch_compiler_spec( + soc_model=[QcomChipset.SM8650, QcomChipset.SM8750], + backend_options=[ + generate_htp_compiler_spec(use_fp16=False), + generate_htp_compiler_spec(use_fp16=False), + ], +) +``` + +The resulting compiler specs are used with the ordinary QNN lowering flow. For +an end-to-end real-model example, including optional ADB execution and output +comparison, run: + +```bash +python -m examples.qualcomm.util_scripts.fcb_resnet50 \ + --soc_models SM8650 SM8750 \ + --devices \ + --build_folder build-android +``` + +### Quantized FCB + +`QnnQuantizer` and `make_quantizer` accept either a single SoC or the FCB +target list. For an FCB list, the quantizer validates annotations and loads +backend operator constraints for the target with the lowest HTP architecture. +This produces quantization valid for every requested target under HTP's +backward-compatibility guarantee. The compiler still receives the complete list +and creates one context per target. + +```python +from executorch.backends.qualcomm.export_utils import make_quantizer +from executorch.backends.qualcomm.quantizer.quantizer import QuantDtype +from executorch.backends.qualcomm.serialization.qc_schema import QcomChipset +from executorch.backends.qualcomm.utils.utils import ( + generate_htp_compiler_spec, + generate_qnn_executorch_compiler_spec, +) +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + +soc_models = [QcomChipset.SM8550, QcomChipset.SM8750] +quantizer = make_quantizer( + quant_dtype=QuantDtype.use_8a8w, + per_channel_conv=True, + soc_model=soc_models, +) +prepared_model = prepare_pt2e( + torch.export.export(model, example_inputs, strict=True).module(), quantizer +) +prepared_model(*example_inputs) # Replace with representative calibration data. +quantized_model = convert_pt2e(prepared_model) + +compiler_specs = generate_qnn_executorch_compiler_spec( + soc_model=soc_models, + backend_options=[ + generate_htp_compiler_spec(use_fp16=False) for _ in soc_models + ], +) +``` + +An end-to-end ResNet50 example uses this flow: + +```bash +python -m examples.qualcomm.util_scripts.fcb_resnet50_quantized \ + --soc_models SM8550 SM8750 \ + --dataset /path/to/imagenet-mini-val/val \ + --devices \ + --build_folder build-android +``` + +### Restrictions + +- FCB requires at least two distinct targets; `soc_model` and `backend_options` + must be equal-length lists. +- Every target must use the HTP backend. GPU, LPAI, and online preparation are + unsupported. +- `fcb_reference_weight_sharing` is enabled by default. It deduplicates weights + only while host AOT appends contexts to the FCB DLC; it is not a runtime + setting. (Note that `use_weight_sharing` is a separate option for HTP, addressing + different sharing scenarios. When FCB reference-weight sharing is enabled, the + incremental benefit of enabling `use_weight_sharing` may be limited.) +- Reference-weight sharing might conflict with DLBC. Set + `fcb_reference_weight_sharing=False` when any target enables `use_dlbc`. +- A FCB DLC can contain only one context binary per target SoC. Including multiple + backend option for same SoC in FCB is not supported + +FCB has device coverage only for the requested SoCs. Deploying the artifact to +an SoC omitted from the target list is unsupported. + + ## Deploying and running on device ### AOT compile a model diff --git a/examples/qualcomm/util_scripts/fcb_multi_soc_weight_sharing_demo.py b/examples/qualcomm/util_scripts/fcb_multi_soc_weight_sharing_demo.py new file mode 100644 index 00000000000..df1372ff14a --- /dev/null +++ b/examples/qualcomm/util_scripts/fcb_multi_soc_weight_sharing_demo.py @@ -0,0 +1,250 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Measure FCB multi-SoC reference-weight sharing during QNN AOT. + +Examples of Multi-Device and Multi-Host execution: + +1. Standard single device / single host setup: + python -m examples.qualcomm.util_scripts.fcb_multi_soc_weight_sharing_demo \ + --soc_models SM8650 SM8750 SM8850 SM8550 \ + --host adb-host1 --devices adb-device1 + +2. Advanced multi-device / multi-host setup: + We can target multiple devices across different adb servers by passing + the devices in the format '[host:]device_id'. This enables testing models + across various HTP architectures concurrently. + + python -m examples.qualcomm.util_scripts.fcb_multi_soc_weight_sharing_demo \ + --soc_models SM8650 SM8750 SM8850 SM8550 \ + --devices adb-host1:adb-device1 adb-host2:adb-device2 +""" + +import argparse +import shutil +import subprocess +from pathlib import Path +from typing import List + +import torch +from executorch.backends.qualcomm.export_utils import QnnConfig, SimpleADB + +from executorch.backends.qualcomm.serialization.qc_schema import ( + _soc_info_table, + QcomChipset, +) +from executorch.backends.qualcomm.utils.utils import ( + generate_htp_compiler_spec, + generate_qnn_executorch_compiler_spec, + to_edge_transform_and_lower_to_qnn, +) + + +class TwoConvs(torch.nn.Module): + def __init__(self): + super().__init__() + self.first = torch.nn.Conv2d(1, 3, 10) + self.second = torch.nn.Conv2d(3, 2, 10) + + def forward(self, x): + return self.second(self.first(x)) + + +model = TwoConvs().eval() +modules = {"two_convs": model, "second": model.second} + +inputs = { + "two_convs": (torch.randn(1, 1, 80, 80),), + "second": (torch.randn(1, 3, 60, 60),), +} + + +def export_fcb( + soc_models: List[QcomChipset], + fcb_reference_weight_sharing: bool, + use_weight_sharing: bool, +): + compiler_specs = generate_qnn_executorch_compiler_spec( + soc_model=soc_models, + backend_options=[ + generate_htp_compiler_spec( + use_fp16=False, use_weight_sharing=use_weight_sharing + ) + for _ in soc_models + ], + fcb_reference_weight_sharing=fcb_reference_weight_sharing, + ) + program = to_edge_transform_and_lower_to_qnn( + module=modules, + inputs=inputs, + compiler_specs={name: compiler_specs for name in modules}, + ).to_executorch() + return program.buffer + + +def get_device_soc_model(host, device): + command = ["adb"] + if host: + command.extend(["-H", host]) + command.extend(["-s", device, "shell", "getprop", "ro.soc.model"]) + soc_model = subprocess.run( + command, check=True, capture_output=True, text=True + ).stdout.strip() + if soc_model not in QcomChipset.__members__: + raise RuntimeError(f"device {device} reported unsupported SoC {soc_model!r}") + return soc_model + + +def get_htp_arch(soc_model: str) -> str: + try: + chipset = QcomChipset[soc_model] + if chipset in _soc_info_table: + return _soc_info_table[chipset].htp_info.htp_arch.name + except Exception: + pass + return "UNKNOWN" + + +def run_on_device(pte_path, output_dir, host, device, build_folder, requested_socs): + soc_model = get_device_soc_model(host, device) + if soc_model not in requested_socs: + raise RuntimeError( + f"device {device} has {soc_model}, not one of the prepared SoCs {requested_socs}" + ) + adb = SimpleADB( + qnn_config=QnnConfig( + soc_model=soc_model, + build_folder=build_folder, + device=device, + host=host, + ), + pte_path=str(pte_path), + workspace=f"/data/local/tmp/qnn_fcb_weight_sharing/{device}", + ) + for method_index, (name, module) in enumerate(sorted(modules.items())): + expected = module(*inputs[name]).detach() + adb.push(inputs=[inputs[name]], init_env=method_index == 0) + adb.execute(custom_runner_cmd=f"rm -rf {adb.output_folder}") + adb.execute(method_index=method_index) + device_outputs = output_dir / device / name + shutil.rmtree(device_outputs, ignore_errors=True) + device_outputs.parent.mkdir(parents=True, exist_ok=True) + adb.pull(str(device_outputs), device_output_path=adb.output_folder) + raw_output = next(device_outputs.rglob("*.raw")) + actual = torch.from_file( + str(raw_output), dtype=expected.dtype, size=expected.numel() + ).reshape(expected.shape) + torch.testing.assert_close(actual, expected, rtol=1, atol=1e-1) + shutil.move(raw_output, output_dir / f"{device}_{name}.raw") + print(f"device {device} ({soc_model}) method {method_index} ({name}): PASS") + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--soc_models", + nargs="+", + required=True, + choices=QcomChipset.__members__, + help="Target Qualcomm SoCs to compile context binaries for", + ) + parser.add_argument( + "--output_dir", + type=Path, + default=Path("/tmp/qnn_fcb_weight_sharing"), + help="Directory to save compile and runtime artifacts", + ) + parser.add_argument( + "--host", + help="Fallback ADB server host name or IP address if no host prefix is specified in --devices", + ) + parser.add_argument( + "--devices", + nargs="+", + default=[], + help="List of target devices to test. Can be specified in the format '[host:]device_id' to run across multiple ADB servers concurrently.", + ) + parser.add_argument( + "--build_folder", + default="build-android", + help="The android compilation folder containing target executables and runner binaries", + ) + args = parser.parse_args() + + if len(args.soc_models) < 2: + parser.error("FCB requires at least two SoCs") + + devices_with_hosts = [] + for d in args.devices: + if ":" in d: + parts = d.split(":", 1) + devices_with_hosts.append((parts[0], parts[1])) + else: + devices_with_hosts.append((args.host, d)) + + print("=" * 80) + print(" DETECTED DEVICES REPORT ".center(80, "=")) + print("=" * 80) + print(f"{'Host':<20} | {'Device ID':<16} | {'SoC Model':<12} | {'HTP Arch':<10}") + print("-" * 80) + for host, device in devices_with_hosts: + try: + soc_model = get_device_soc_model(host, device) + htp_arch = get_htp_arch(soc_model) + print( + f"{str(host or 'localhost'):<20} | {device:<16} | {soc_model:<12} | {htp_arch:<10}" + ) + except Exception as e: + print(f"{str(host or 'localhost'):<20} | {device:<16} | ERROR: {e}") + print("=" * 80 + "\n") + + soc_models = [QcomChipset[model] for model in args.soc_models] + args.output_dir.mkdir(parents=True, exist_ok=True) + torch.manual_seed(0) + + print(f"socs: { args.soc_models}") + results = [] + for ws in [True, False]: + for rws in [True, False]: + pte_bytes = export_fcb(soc_models, rws, ws) + pte_path = args.output_dir / f"ws={ws}_rws={rws}.pte" + pte_path.write_bytes(pte_bytes) + results.append( + { + "ws": ws, + "rws": rws, + "pte_size": len(pte_bytes), + } + ) + print("\n" + "=" * 80) + print(" FCB MULTI-SOC WEIGHT SHARING SUMMARY REPORT ".center(80, "=")) + print("=" * 80) + print(f"Target SoCs: {args.soc_models}") + print("-" * 80) + print(f"{'Weight Share':<14} | {'Ref Weight Share':<18} | {'PTE Size (Bytes)':<16}") + print("-" * 80) + for res in results: + ws_str = str(res["ws"]) + rws_str = str(res["rws"]) + size_str = f"{res['pte_size']:,}" + print(f"{ws_str:<14} | {rws_str:<18} | {size_str:>16}") + print("-" * 80) + for host, device in devices_with_hosts: + run_on_device( + pte_path, + args.output_dir, + host, + device, + args.build_folder, + args.soc_models, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/qualcomm/util_scripts/fcb_resnet50.py b/examples/qualcomm/util_scripts/fcb_resnet50.py new file mode 100644 index 00000000000..3ccf2a08fe7 --- /dev/null +++ b/examples/qualcomm/util_scripts/fcb_resnet50.py @@ -0,0 +1,136 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Export an offline HTP FCB ResNet50 and optionally verify it on devices.""" + +import argparse +import shutil +import subprocess +from pathlib import Path + +import torch +from executorch.backends.qualcomm.export_utils import QnnConfig, SimpleADB +from executorch.backends.qualcomm.serialization.qc_schema import QcomChipset +from executorch.backends.qualcomm.utils.utils import ( + generate_htp_compiler_spec, + generate_qnn_executorch_compiler_spec, + to_edge_transform_and_lower_to_qnn, +) +from executorch.examples.models.resnet import ResNet50Model + + +def get_device_soc_model(host: str | None, device: str) -> str: + command = ["adb"] + if host: + command.extend(["-H", host]) + command.extend(["-s", device, "shell", "getprop", "ro.soc.model"]) + soc_model = subprocess.run( + command, check=True, capture_output=True, text=True + ).stdout.strip() + if soc_model not in QcomChipset.__members__: + raise RuntimeError(f"device {device} reported unsupported SoC {soc_model!r}") + return soc_model + + +def export_fcb(model: torch.nn.Module, soc_models: list[QcomChipset], sample_input): + compiler_specs = generate_qnn_executorch_compiler_spec( + soc_model=soc_models, + backend_options=[generate_htp_compiler_spec(use_fp16=True) for _ in soc_models], + ) + return ( + to_edge_transform_and_lower_to_qnn( + module={"forward": model}, + inputs={"forward": sample_input}, + compiler_specs={"forward": compiler_specs}, + ) + .to_executorch() + .buffer + ) + + +def run_on_device( + pte_path: Path, + output_dir: Path, + host: str | None, + device: str, + build_folder: str, + requested_socs: list[str], + sample_input, + expected: torch.Tensor, +): + soc_model = get_device_soc_model(host, device) + if soc_model not in requested_socs: + raise RuntimeError( + f"device {device} has {soc_model}, not one of the prepared SoCs {requested_socs}" + ) + adb = SimpleADB( + qnn_config=QnnConfig( + soc_model=soc_model, + build_folder=build_folder, + device=device, + host=host, + ), + pte_path=str(pte_path), + workspace=f"/data/local/tmp/qnn_fcb_resnet50/{device}", + ) + adb.push(inputs=[sample_input], init_env=True) + adb.execute(custom_runner_cmd=f"rm -rf {adb.output_folder}") + adb.execute(method_index=0) + device_outputs = output_dir / device + shutil.rmtree(device_outputs, ignore_errors=True) + adb.pull(str(device_outputs), device_output_path=adb.output_folder) + raw_output = next(device_outputs.rglob("*.raw")) + actual = torch.from_file( + str(raw_output), dtype=expected.dtype, size=expected.numel() + ).reshape(expected.shape) + torch.testing.assert_close(actual, expected, rtol=1e-1, atol=1e-1) + print(f"device {device} ({soc_model}): PASS") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--soc_models", nargs="+", required=True, choices=QcomChipset.__members__ + ) + parser.add_argument( + "--output_dir", type=Path, default=Path("/tmp/qnn_fcb_resnet50") + ) + parser.add_argument("--host") + parser.add_argument("--devices", nargs="+", default=[]) + parser.add_argument("--build_folder", default="build-android") + args = parser.parse_args() + if len(args.soc_models) < 2: + parser.error("FCB requires at least two SoCs") + + torch.manual_seed(0) + model = ResNet50Model().get_eager_model().eval() + sample_input = (torch.randn(1, 3, 224, 224),) + expected = model(*sample_input).detach() + args.output_dir.mkdir(parents=True, exist_ok=True) + pte_path = args.output_dir / "resnet50_fcb.pte" + pte_path.write_bytes( + export_fcb(model, [QcomChipset[name] for name in args.soc_models], sample_input) + ) + + for device_with_host in args.devices: + if ":" in device_with_host: + host, device = device_with_host.split(":", 1) + else: + host, device = args.host, device_with_host + run_on_device( + pte_path, + args.output_dir, + host, + device, + args.build_folder, + args.soc_models, + sample_input, + expected, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/qualcomm/util_scripts/fcb_resnet50_quantized.py b/examples/qualcomm/util_scripts/fcb_resnet50_quantized.py new file mode 100644 index 00000000000..d63df6a41a6 --- /dev/null +++ b/examples/qualcomm/util_scripts/fcb_resnet50_quantized.py @@ -0,0 +1,171 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Export a PTQ HTP FCB ResNet50 and optionally verify it on devices.""" + +import argparse +import shutil +from pathlib import Path + +import numpy as np +import torch +from executorch.backends.qualcomm.export_utils import ( + make_quantizer, + QnnConfig, + SimpleADB, +) +from executorch.backends.qualcomm.quantizer.quantizer import QuantDtype +from executorch.backends.qualcomm.serialization.qc_schema import QcomChipset +from executorch.backends.qualcomm.utils.utils import ( + generate_htp_compiler_spec, + generate_qnn_executorch_compiler_spec, + to_edge_transform_and_lower_to_qnn, +) +from executorch.examples.models.resnet import ResNet50Model +from executorch.examples.qualcomm.util_scripts.fcb_resnet50 import get_device_soc_model +from executorch.examples.qualcomm.utils import get_imagenet_dataset, topk_accuracy +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + +def quantize_model( + model: torch.nn.Module, + soc_models: list[QcomChipset], + calibration_inputs, +) -> torch.nn.Module: + if not calibration_inputs: + raise ValueError("calibration_inputs must not be empty") + exported_model = torch.export.export( + model, calibration_inputs[0], strict=True + ).module() + quantizer = make_quantizer( + quant_dtype=QuantDtype.use_8a8w, + per_channel_conv=True, + soc_model=soc_models, + ) + prepared_model = prepare_pt2e(exported_model, quantizer) + for calibration_input in calibration_inputs: + prepared_model(*calibration_input) + return convert_pt2e(prepared_model) + + +def export_fcb(model: torch.nn.Module, soc_models: list[QcomChipset], sample_input): + compiler_specs = generate_qnn_executorch_compiler_spec( + soc_model=soc_models, + backend_options=[ + generate_htp_compiler_spec(use_fp16=False) for _ in soc_models + ], + ) + return ( + to_edge_transform_and_lower_to_qnn( + module={"forward": model}, + inputs={"forward": sample_input}, + compiler_specs={"forward": compiler_specs}, + ) + .to_executorch() + .buffer + ) + + +def run_on_device( + pte_path: Path, + output_dir: Path, + host: str | None, + device: str, + build_folder: str, + requested_socs: list[str], + inputs, + targets, +): + soc_model = get_device_soc_model(host, device) + if soc_model not in requested_socs: + raise RuntimeError( + f"device {device} has {soc_model}, not one of the prepared SoCs {requested_socs}" + ) + adb = SimpleADB( + qnn_config=QnnConfig( + soc_model=soc_model, + build_folder=build_folder, + device=device, + host=host, + ), + pte_path=str(pte_path), + workspace=f"/data/local/tmp/qnn_fcb_resnet50_quantized/{device}", + ) + adb.push(inputs=inputs, init_env=True) + adb.execute(custom_runner_cmd=f"rm -rf {adb.output_folder}") + adb.execute(method_index=0) + device_outputs = output_dir / device + shutil.rmtree(device_outputs, ignore_errors=True) + adb.pull(str(device_outputs), device_output_path=adb.output_folder) + + predictions = [] + for index in range(len(inputs)): + raw_output = next(device_outputs.rglob(f"output_{index}_0.raw")) + predictions.append(np.fromfile(raw_output, dtype=np.float32)) + + top1 = topk_accuracy(predictions, targets, 1).item() + top5 = topk_accuracy(predictions, targets, 5).item() + print(f"device {device} ({soc_model}): top_1={top1}% top_5={top5}%") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--soc_models", nargs="+", required=True, choices=QcomChipset.__members__ + ) + parser.add_argument( + "--output_dir", type=Path, default=Path("/tmp/qnn_fcb_resnet50_quantized") + ) + parser.add_argument( + "--dataset", + required=True, + help="path to the ImageNet validation folder used for PTQ calibration", + ) + parser.add_argument("--calibration_samples", type=int, default=100) + parser.add_argument("--host") + parser.add_argument("--devices", nargs="+", default=[]) + parser.add_argument("--build_folder", default="build-android") + args = parser.parse_args() + if len(args.soc_models) < 2: + parser.error("FCB requires at least two SoCs") + + model = ResNet50Model().get_eager_model().eval() + calibration_inputs, targets = get_imagenet_dataset( + dataset_path=args.dataset, + data_size=args.calibration_samples, + image_shape=(256, 256), + crop_size=224, + shuffle=False, + ) + if not calibration_inputs: + parser.error("no calibration images found in --dataset") + sample_input = calibration_inputs[0] + soc_models = [QcomChipset[name] for name in args.soc_models] + quantized_model = quantize_model(model, soc_models, calibration_inputs) + + args.output_dir.mkdir(parents=True, exist_ok=True) + pte_path = args.output_dir / "resnet50_fcb_quantized.pte" + pte_path.write_bytes(export_fcb(quantized_model, soc_models, sample_input)) + + for device_with_host in args.devices: + if ":" in device_with_host: + host, device = device_with_host.split(":", 1) + else: + host, device = args.host, device_with_host + run_on_device( + pte_path, + args.output_dir, + host, + device, + args.build_folder, + args.soc_models, + calibration_inputs, + targets, + ) + + +if __name__ == "__main__": + main()