From 664a7a3cfa1015a1fc317a2c6c042bcf8033beb5 Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:24:51 +0000 Subject: [PATCH 01/10] [ONNX] Quantize ResNet residual adds Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- examples/torch_onnx/README.md | 1 + examples/torch_onnx/torch_quant_to_onnx.py | 61 +++++++++++++++++++ .../torch_onnx/test_torch_quant_to_onnx.py | 29 ++++++++- 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/examples/torch_onnx/README.md b/examples/torch_onnx/README.md index 06ad11164dd..e4a9b1319fe 100644 --- a/examples/torch_onnx/README.md +++ b/examples/torch_onnx/README.md @@ -54,6 +54,7 @@ The `torch_quant_to_onnx.py` script quantizes [timm](https://github.com/huggingf - Loads a pretrained timm torch model (default: ViT-Base). - Quantizes the torch model to FP8, MXFP8, INT8, NVFP4, or INT4_AWQ using ModelOpt. - For models with Conv2d layers (e.g., SwinTransformer), automatically overrides Conv2d quantization to FP8 (for MXFP8/NVFP4 modes) or INT8 (for INT4_AWQ mode) for TensorRT compatibility. +- Quantizes ResNet residual-add outputs before activation for activation-quantized modes. - Exports the quantized model to ONNX. - Postprocesses the ONNX model to be compatible with TensorRT. - Saves the final ONNX model. diff --git a/examples/torch_onnx/torch_quant_to_onnx.py b/examples/torch_onnx/torch_quant_to_onnx.py index 97bd0b60c18..0c092b32134 100644 --- a/examples/torch_onnx/torch_quant_to_onnx.py +++ b/examples/torch_onnx/torch_quant_to_onnx.py @@ -34,6 +34,8 @@ from evaluation import evaluate import modelopt.torch.quantization as mtq +from modelopt.torch.quantization.config import QuantizerAttributeConfig +from modelopt.torch.quantization.nn import TensorQuantizer """ Quantize a timm vision model and export to ONNX for TensorRT deployment. @@ -218,6 +220,58 @@ def _disable_low_channel_conv_input_quantizers(model): q.disable() +def _quantize_residual_input(module, inputs): + return (module.residual_quantizer(inputs[0]),) + + +def _add_resnet_residual_quantizers(model, quantize_mode, auto_quantization_formats, data_loader): + if quantize_mode == "int8": + num_bits = 8 + elif quantize_mode in ("fp8", "mxfp8", "nvfp4"): + # Dynamic block quantizers do not support the residual path's 4D tensors. + num_bits = (4, 3) + elif quantize_mode == "auto": + activation_formats = set(auto_quantization_formats) - {"INT4_AWQ_CFG"} + if not activation_formats: + return + num_bits = 8 if activation_formats == {"INT8_DEFAULT_CFG"} else (4, 3) + else: + return + + residual_quantizers = [] + block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) + for block in model.modules(): + if not isinstance(block, block_types): + continue + activation = block.act3 if isinstance(block, timm.models.resnet.Bottleneck) else block.act2 + activation.residual_quantizer = TensorQuantizer( + QuantizerAttributeConfig(num_bits=num_bits, axis=None) + ).to(next(block.parameters()).device) + activation.register_forward_pre_hook(_quantize_residual_input) + residual_quantizers.append(activation.residual_quantizer) + + if not residual_quantizers: + return + + for quantizer in residual_quantizers: + quantizer.disable_quant() + quantizer.enable_calib() + + was_training = model.training + model.eval() + try: + with torch.no_grad(): + for batch in data_loader: + model(batch["image"] if isinstance(batch, dict) else batch) + finally: + model.train(was_training) + + for quantizer in residual_quantizers: + quantizer.load_calib_amax() + quantizer.disable_calib() + quantizer.enable_quant() + + def load_calibration_data(model, data_size, batch_size, device, with_labels=False): """Load and prepare calibration data. @@ -580,6 +634,13 @@ def main(): quantized_model = quantize_model(model, config, data_loader) + _add_resnet_residual_quantizers( + quantized_model, + args.quantize_mode, + args.auto_quantization_formats, + data_loader, + ) + # MXFP8/NVFP4 lower their input quantizers to TRT DynamicQuantize (2D/3D only). # Disable quantizers on 4D-input layers (Swin's norm1 / downsample.norm / top-level norm). # Auto mode also needs this when an MXFP8/NVFP4 candidate format is in the search set. diff --git a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py index 6d6e0d9de57..01ea931ca01 100644 --- a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py +++ b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py @@ -14,6 +14,9 @@ # limitations under the License. +from collections import defaultdict + +import onnx import pytest from _test_utils.examples.run_command import extend_cmd_parts, run_example_command @@ -29,11 +32,30 @@ } +def _assert_residual_adds_are_quantized(onnx_save_path): + model = onnx.load(onnx_save_path) + consumers = defaultdict(list) + for node in model.graph.node: + for input_name in node.input: + consumers[input_name].append(node) + + residual_adds = [node for node in model.graph.node if node.op_type == "Add"] + assert len(residual_adds) == 16 + for add in residual_adds: + add_consumers = consumers[add.output[0]] + assert len(add_consumers) == 1 + quantizer_input = add_consumers[0] + if quantizer_input.op_type == "Cast": + add_consumers = consumers[quantizer_input.output[0]] + assert len(add_consumers) == 1 + assert add_consumers[0].op_type.endswith("QuantizeLinear") + + @pytest.mark.parametrize("quantize_mode", _QUANT_MODES) @pytest.mark.parametrize("model_key", list(_MODELS)) -def test_torch_onnx(model_key, quantize_mode): +def test_torch_onnx(tmp_path, model_key, quantize_mode): timm_model_name, model_kwargs = _MODELS[model_key] - onnx_save_path = f"{model_key}.{quantize_mode}.onnx" + onnx_save_path = tmp_path / f"{model_key}.{quantize_mode}.onnx" cmd_parts = extend_cmd_parts( ["python", "torch_quant_to_onnx.py"], @@ -46,3 +68,6 @@ def test_torch_onnx(model_key, quantize_mode): ) cmd_parts.extend(["--no_pretrained", "--trt_build"]) run_example_command(cmd_parts, "torch_onnx") + + if model_key == "resnet50": + _assert_residual_adds_are_quantized(onnx_save_path) From 28b29126491b69ea3cce93803265502abdae481f Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:29:46 +0000 Subject: [PATCH 02/10] Update changelog Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 329e5d21f05..b1e77b720e5 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ Changelog **Bug Fixes** +- Quantize residual-add outputs in the torch ONNX ResNet example. + 0.46 (2026-08-17) ^^^^^^^^^^^^^^^^^ From c15755ca3578f2951b33751ff1cf890b4ba6e157 Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:04:32 +0000 Subject: [PATCH 03/10] Move changelog entry to 0.46 Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- CHANGELOG.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b1e77b720e5..90c446f5d13 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,8 +12,6 @@ Changelog **Bug Fixes** -- Quantize residual-add outputs in the torch ONNX ResNet example. - 0.46 (2026-08-17) ^^^^^^^^^^^^^^^^^ @@ -83,6 +81,7 @@ Changelog **Bug Fixes** +- Quantize residual-add outputs in the torch ONNX ResNet example. - Fix ``ShapeInferenceError`` during ONNX INT8 + FP16 quantization (``--high_precision_dtype fp16``) of weakly-typed models (e.g. TensorFlow exports) that carry stale rank-0 ``graph.output`` shapes or ops such as ``TopK`` that ONNX's static shape inference cannot resolve. ``clear_stale_value_info`` now reconciles stale output shapes via symbolic shape inference (keeping every output's shape field populated), and AutoCast runs ONNX shape inference in strict mode and falls back to schema-based standalone type inference when it fails, so unresolved ops no longer leave tensors untyped. - Fused MoE expert auto-detection (``register_fused_experts_on_the_fly``) no longer requires an ``act_fn`` attribute. Some fused-expert modules (e.g. ``MiniMaxM3VLExperts``) apply a custom gated activation between the two ``F.linear`` calls instead of exposing ``act_fn``; they were silently skipped, leaving routed experts unquantized (an experts-only recipe matched nothing) and failing HF export with ``NotImplementedError``. ``_QuantFusedExperts`` is activation-agnostic (it only intercepts the two ``F.linear`` calls), so the requirement was unnecessary. This enables NVFP4/FP8 quantization and export for MiniMax-M2 / MiniMax-M3. - Fix unified HF export emitting transformers' *in-memory* (post-``conversion_mapping``) tensor names instead of the original model-hub names, breaking the unified-checkpoint contract (observed on MiniMax-M3: exported ``model.language_model.*`` / ``mlp.experts.*.gate_proj`` instead of hub ``language_model.model.*`` / ``block_sparse_moe.experts.*.w{1,2,3}``). transformers' own save-side ``revert_weight_conversion`` is disabled by ModelOpt because it raises ``RuntimeError`` on 0-d scalar scale tensors, so a new quant-aware reverse conversion (``modelopt/torch/export/quant_aware_conversion.py``) derives rename/split rules from the model's conversion mapping via transformers' ``reverse_transform()`` and carries each weight's companion scale tensors (``weight_scale``, ``weight_scale_2``, ``input_scale``, ``weight_scale_inv``, ``bias``) through the renames and un-fusions, so quantized exports round-trip to the hub names. Any mapping op that cannot be reversed quant-aware yet (e.g. still-stacked fused experts) falls back to the previous in-memory names instead of aborting the export. From d2a78de56f7e1b6bc0935a76546744296b55680c Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:06:02 +0000 Subject: [PATCH 04/10] Place residual QDQ before Add Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- CHANGELOG.rst | 2 +- examples/torch_onnx/README.md | 2 +- examples/torch_onnx/torch_quant_to_onnx.py | 19 +++++++++--------- .../torch_onnx/test_torch_quant_to_onnx.py | 20 ++++++++++++------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 90c446f5d13..bc7e010dff1 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -81,7 +81,7 @@ Changelog **Bug Fixes** -- Quantize residual-add outputs in the torch ONNX ResNet example. +- Quantize shortcut inputs before residual addition in the torch ONNX ResNet example. - Fix ``ShapeInferenceError`` during ONNX INT8 + FP16 quantization (``--high_precision_dtype fp16``) of weakly-typed models (e.g. TensorFlow exports) that carry stale rank-0 ``graph.output`` shapes or ops such as ``TopK`` that ONNX's static shape inference cannot resolve. ``clear_stale_value_info`` now reconciles stale output shapes via symbolic shape inference (keeping every output's shape field populated), and AutoCast runs ONNX shape inference in strict mode and falls back to schema-based standalone type inference when it fails, so unresolved ops no longer leave tensors untyped. - Fused MoE expert auto-detection (``register_fused_experts_on_the_fly``) no longer requires an ``act_fn`` attribute. Some fused-expert modules (e.g. ``MiniMaxM3VLExperts``) apply a custom gated activation between the two ``F.linear`` calls instead of exposing ``act_fn``; they were silently skipped, leaving routed experts unquantized (an experts-only recipe matched nothing) and failing HF export with ``NotImplementedError``. ``_QuantFusedExperts`` is activation-agnostic (it only intercepts the two ``F.linear`` calls), so the requirement was unnecessary. This enables NVFP4/FP8 quantization and export for MiniMax-M2 / MiniMax-M3. - Fix unified HF export emitting transformers' *in-memory* (post-``conversion_mapping``) tensor names instead of the original model-hub names, breaking the unified-checkpoint contract (observed on MiniMax-M3: exported ``model.language_model.*`` / ``mlp.experts.*.gate_proj`` instead of hub ``language_model.model.*`` / ``block_sparse_moe.experts.*.w{1,2,3}``). transformers' own save-side ``revert_weight_conversion`` is disabled by ModelOpt because it raises ``RuntimeError`` on 0-d scalar scale tensors, so a new quant-aware reverse conversion (``modelopt/torch/export/quant_aware_conversion.py``) derives rename/split rules from the model's conversion mapping via transformers' ``reverse_transform()`` and carries each weight's companion scale tensors (``weight_scale``, ``weight_scale_2``, ``input_scale``, ``weight_scale_inv``, ``bias``) through the renames and un-fusions, so quantized exports round-trip to the hub names. Any mapping op that cannot be reversed quant-aware yet (e.g. still-stacked fused experts) falls back to the previous in-memory names instead of aborting the export. diff --git a/examples/torch_onnx/README.md b/examples/torch_onnx/README.md index e4a9b1319fe..a5f517e0356 100644 --- a/examples/torch_onnx/README.md +++ b/examples/torch_onnx/README.md @@ -54,7 +54,7 @@ The `torch_quant_to_onnx.py` script quantizes [timm](https://github.com/huggingf - Loads a pretrained timm torch model (default: ViT-Base). - Quantizes the torch model to FP8, MXFP8, INT8, NVFP4, or INT4_AWQ using ModelOpt. - For models with Conv2d layers (e.g., SwinTransformer), automatically overrides Conv2d quantization to FP8 (for MXFP8/NVFP4 modes) or INT8 (for INT4_AWQ mode) for TensorRT compatibility. -- Quantizes ResNet residual-add outputs before activation for activation-quantized modes. +- Quantizes ResNet shortcut inputs before residual addition for activation-quantized modes. - Exports the quantized model to ONNX. - Postprocesses the ONNX model to be compatible with TensorRT. - Saves the final ONNX model. diff --git a/examples/torch_onnx/torch_quant_to_onnx.py b/examples/torch_onnx/torch_quant_to_onnx.py index 0c092b32134..aa0d4a36ffb 100644 --- a/examples/torch_onnx/torch_quant_to_onnx.py +++ b/examples/torch_onnx/torch_quant_to_onnx.py @@ -220,10 +220,6 @@ def _disable_low_channel_conv_input_quantizers(model): q.disable() -def _quantize_residual_input(module, inputs): - return (module.residual_quantizer(inputs[0]),) - - def _add_resnet_residual_quantizers(model, quantize_mode, auto_quantization_formats, data_loader): if quantize_mode == "int8": num_bits = 8 @@ -243,12 +239,15 @@ def _add_resnet_residual_quantizers(model, quantize_mode, auto_quantization_form for block in model.modules(): if not isinstance(block, block_types): continue - activation = block.act3 if isinstance(block, timm.models.resnet.Bottleneck) else block.act2 - activation.residual_quantizer = TensorQuantizer( - QuantizerAttributeConfig(num_bits=num_bits, axis=None) - ).to(next(block.parameters()).device) - activation.register_forward_pre_hook(_quantize_residual_input) - residual_quantizers.append(activation.residual_quantizer) + quantizer = TensorQuantizer(QuantizerAttributeConfig(num_bits=num_bits, axis=None)).to( + next(block.parameters()).device + ) + if block.downsample is None: + block.downsample = torch.nn.Sequential() + elif not isinstance(block.downsample, torch.nn.Sequential): + block.downsample = torch.nn.Sequential(block.downsample) + block.downsample.add_module("residual_quantizer", quantizer) + residual_quantizers.append(quantizer) if not residual_quantizers: return diff --git a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py index 01ea931ca01..2a4fc9c4de0 100644 --- a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py +++ b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py @@ -35,20 +35,26 @@ def _assert_residual_adds_are_quantized(onnx_save_path): model = onnx.load(onnx_save_path) consumers = defaultdict(list) + producers = {} for node in model.graph.node: for input_name in node.input: consumers[input_name].append(node) + for output_name in node.output: + producers[output_name] = node residual_adds = [node for node in model.graph.node if node.op_type == "Add"] assert len(residual_adds) == 16 for add in residual_adds: - add_consumers = consumers[add.output[0]] - assert len(add_consumers) == 1 - quantizer_input = add_consumers[0] - if quantizer_input.op_type == "Cast": - add_consumers = consumers[quantizer_input.output[0]] - assert len(add_consumers) == 1 - assert add_consumers[0].op_type.endswith("QuantizeLinear") + input_producers = [producers[input_name] for input_name in add.input] + input_producers = [ + producers[node.input[0]] if node.op_type == "Cast" else node for node in input_producers + ] + assert any( + node.op_type.endswith("DequantizeLinear") + and producers[node.input[0]].op_type.endswith("QuantizeLinear") + for node in input_producers + ) + assert [node.op_type for node in consumers[add.output[0]]] == ["Relu"] @pytest.mark.parametrize("quantize_mode", _QUANT_MODES) From 77f2433a73ce64f873161216ebd4ece84fbc8706 Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:04:03 +0000 Subject: [PATCH 05/10] Address ResNet residual quantization review feedback Co-Authored-By: Codex Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- CHANGELOG.rst | 3 +- examples/torch_onnx/torch_quant_to_onnx.py | 224 ++++++++++++++---- modelopt/onnx/export/fp8_exporter.py | 5 + modelopt/onnx/utils.py | 54 ++++- modelopt/torch/_deploy/utils/torch_onnx.py | 5 +- .../torch_onnx/test_torch_quant_to_onnx.py | 67 +++++- .../quantization/test_fp8_mha_exporter.py | 17 ++ tests/unit/onnx/test_fold_casts.py | 100 +++++++- 8 files changed, 419 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bc7e010dff1..9aedad8172f 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -81,7 +81,8 @@ Changelog **Bug Fixes** -- Quantize shortcut inputs before residual addition in the torch ONNX ResNet example. +- Share each ResNet block-input Q/DQ between the main and shortcut paths in the torch ONNX + example, restoring TensorRT residual fusion for INT8 and FP8 exports. - Fix ``ShapeInferenceError`` during ONNX INT8 + FP16 quantization (``--high_precision_dtype fp16``) of weakly-typed models (e.g. TensorFlow exports) that carry stale rank-0 ``graph.output`` shapes or ops such as ``TopK`` that ONNX's static shape inference cannot resolve. ``clear_stale_value_info`` now reconciles stale output shapes via symbolic shape inference (keeping every output's shape field populated), and AutoCast runs ONNX shape inference in strict mode and falls back to schema-based standalone type inference when it fails, so unresolved ops no longer leave tensors untyped. - Fused MoE expert auto-detection (``register_fused_experts_on_the_fly``) no longer requires an ``act_fn`` attribute. Some fused-expert modules (e.g. ``MiniMaxM3VLExperts``) apply a custom gated activation between the two ``F.linear`` calls instead of exposing ``act_fn``; they were silently skipped, leaving routed experts unquantized (an experts-only recipe matched nothing) and failing HF export with ``NotImplementedError``. ``_QuantFusedExperts`` is activation-agnostic (it only intercepts the two ``F.linear`` calls), so the requirement was unnecessary. This enables NVFP4/FP8 quantization and export for MiniMax-M2 / MiniMax-M3. - Fix unified HF export emitting transformers' *in-memory* (post-``conversion_mapping``) tensor names instead of the original model-hub names, breaking the unified-checkpoint contract (observed on MiniMax-M3: exported ``model.language_model.*`` / ``mlp.experts.*.gate_proj`` instead of hub ``language_model.model.*`` / ``block_sparse_moe.experts.*.w{1,2,3}``). transformers' own save-side ``revert_weight_conversion`` is disabled by ModelOpt because it raises ``RuntimeError`` on 0-d scalar scale tensors, so a new quant-aware reverse conversion (``modelopt/torch/export/quant_aware_conversion.py``) derives rename/split rules from the model's conversion mapping via transformers' ``reverse_transform()`` and carries each weight's companion scale tensors (``weight_scale``, ``weight_scale_2``, ``input_scale``, ``weight_scale_inv``, ``bias``) through the renames and un-fusions, so quantized exports round-trip to the hub names. Any mapping op that cannot be reversed quant-aware yet (e.g. still-stacked fused experts) falls back to the previous in-memory names instead of aborting the export. diff --git a/examples/torch_onnx/torch_quant_to_onnx.py b/examples/torch_onnx/torch_quant_to_onnx.py index aa0d4a36ffb..d7720e3156d 100644 --- a/examples/torch_onnx/torch_quant_to_onnx.py +++ b/examples/torch_onnx/torch_quant_to_onnx.py @@ -90,6 +90,13 @@ }, ] +_FP8_RESIDUAL_OVERRIDE: list = [ + { + "quantizer_name": "*residual_quantizer.input_quantizer", + "cfg": {"num_bits": (4, 3), "axis": None}, + }, +] + # FP8 MHA-aware config entries: quantize LayerNorm output so TRT can fuse the shared # Q/DQ across all downstream Q/K/V/FC consumers. Softmax-output Q/DQ is handled by the # FP8 ONNX exporter's post-processing pass (fixed 1/448 scale, data-independent). @@ -135,6 +142,7 @@ def get_quant_config(quantize_mode): f"Overriding Conv2d quantization to FP8 for '{quantize_mode}' mode." ) config["quant_cfg"].extend(_FP8_CONV_OVERRIDE) + config["quant_cfg"].extend(_FP8_RESIDUAL_OVERRIDE) elif quantize_mode == "int4_awq": warnings.warn( "TensorRT only supports FP8/INT8 for Conv layers. " @@ -197,63 +205,39 @@ def hook(m, inp, out, _n=name): def _disable_low_channel_conv_input_quantizers(model): - """Disable ``input_quantizer`` on Conv2d modules whose ``in_channels <= 3``. + """Disable input quantization on Conv2d modules with at most 16 input or output channels. The first Conv2d of an image backbone (e.g. ResNet50's ``conv1``) consumes raw - RGB input, so ``in_channels == 3``. On Blackwell (compute capability 12.0) TRT - fails to find an FP8/MXFP8/NVFP4 tactic for this first-layer Q→Conv fusion: + RGB input. TensorRT does not reliably accelerate FP8 convolutions with such small + channel dimensions, so ONNX PTQ leaves the entire Conv in high precision. On + Blackwell, TRT can also fail to find a tactic for the first-layer Q→Conv fusion: Error Code 10: Could not find any implementation for node /conv1/input_quantizer/TRT_FP8QuantizeLinear ... [ElementWise] - Ada (8.9) happens to have a tactic, which is why local runs pass. Disabling the - input quantizer on the raw-RGB conv is also standard quantization practice — - first/last layers are typically left in higher precision. Weight quantization - still applies. Swin/ViT's ``patch_embed.proj`` is already excluded via - ``filter_func``'s ``patch_embed`` pattern, so this helper is effectively the - ResNet-shaped analogue. + Ada (8.9) happens to have a tactic, which is why local runs pass. Swin/ViT's + ``patch_embed.proj`` is already excluded via ``filter_func``. """ for _, mod in model.named_modules(): - if isinstance(mod, torch.nn.Conv2d) and mod.in_channels <= 3: + if isinstance(mod, torch.nn.Conv2d) and min(mod.in_channels, mod.out_channels) <= 16: q = getattr(mod, "input_quantizer", None) if q is not None and q.is_enabled: q.disable() -def _add_resnet_residual_quantizers(model, quantize_mode, auto_quantization_formats, data_loader): - if quantize_mode == "int8": - num_bits = 8 - elif quantize_mode in ("fp8", "mxfp8", "nvfp4"): - # Dynamic block quantizers do not support the residual path's 4D tensors. - num_bits = (4, 3) - elif quantize_mode == "auto": - activation_formats = set(auto_quantization_formats) - {"INT4_AWQ_CFG"} - if not activation_formats: - return - num_bits = 8 if activation_formats == {"INT8_DEFAULT_CFG"} else (4, 3) - else: - return +def _quantize_module_input(module, inputs): + return (module.input_quantizer(inputs[0]), *inputs[1:]) - residual_quantizers = [] - block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) - for block in model.modules(): - if not isinstance(block, block_types): - continue - quantizer = TensorQuantizer(QuantizerAttributeConfig(num_bits=num_bits, axis=None)).to( - next(block.parameters()).device - ) - if block.downsample is None: - block.downsample = torch.nn.Sequential() - elif not isinstance(block.downsample, torch.nn.Sequential): - block.downsample = torch.nn.Sequential(block.downsample) - block.downsample.add_module("residual_quantizer", quantizer) - residual_quantizers.append(quantizer) - if not residual_quantizers: - return - - for quantizer in residual_quantizers: +def _calibrate_new_quantizers(model, quantizers, data_loader): + enabled_quantizers = [ + module + for module in model.modules() + if isinstance(module, TensorQuantizer) and module.is_enabled + ] + for quantizer in enabled_quantizers: quantizer.disable_quant() + for quantizer in quantizers: quantizer.enable_calib() was_training = model.training @@ -262,13 +246,150 @@ def _add_resnet_residual_quantizers(model, quantize_mode, auto_quantization_form with torch.no_grad(): for batch in data_loader: model(batch["image"] if isinstance(batch, dict) else batch) + for quantizer in quantizers: + quantizer.load_calib_amax(strict=False) finally: model.train(was_training) + for quantizer in quantizers: + quantizer.disable_calib() + for quantizer in enabled_quantizers: + quantizer.enable_quant() + + _disable_invalid_quantizers(quantizers) + + +def _append_resnet_residual_quantizer(block, num_bits): + device = next(block.parameters()).device + quantizer = TensorQuantizer(QuantizerAttributeConfig(num_bits=num_bits, axis=None)).to(device) + residual_quantizer = torch.nn.Sequential() + residual_quantizer.add_module("input_quantizer", quantizer) + if block.downsample is None: + block.downsample = torch.nn.Sequential() + elif not isinstance(block.downsample, torch.nn.Sequential): + block.downsample = torch.nn.Sequential(block.downsample) + block.downsample.add_module("residual_quantizer", residual_quantizer) + return quantizer + + +def _prepare_resnet_quantizers(model, quantize_mode): + """Install ResNet shortcut quantizers before the standard calibration pass. + + INT8 and FP8 share one block-input Q/DQ between ``conv1`` and an identity shortcut; + projection blocks additionally quantize the downsample output immediately before ``Add``. + MXFP8 and NVFP4 use per-tensor FP8 on every 4D shortcut because their dynamic block + quantizers only support 2D/3D tensors. INT4-AWQ is skipped because it is weight-only. + Auto mode is configured after its per-block format search. + """ + block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) + blocks = [block for block in model.modules() if isinstance(block, block_types)] + if ( + not blocks + or quantize_mode in ("auto", "int4_awq") + or any( + hasattr(block, "input_quantizer") + or (block.downsample is not None and hasattr(block.downsample, "residual_quantizer")) + for block in blocks + ) + ): + return [] - for quantizer in residual_quantizers: - quantizer.load_calib_amax() - quantizer.disable_calib() - quantizer.enable_quant() + num_bits = 8 if quantize_mode == "int8" else (4, 3) + device = next(model.parameters()).device + residual_quantizers = [] + for block in blocks: + if quantize_mode in ("int8", "fp8"): + quantizer = TensorQuantizer(QuantizerAttributeConfig(num_bits=num_bits, axis=None)).to( + device + ) + block.add_module("input_quantizer", quantizer) + block.register_forward_pre_hook(_quantize_module_input) + residual_quantizers.append(quantizer) + + if block.downsample is None: + continue + + residual_quantizers.append(_append_resnet_residual_quantizer(block, num_bits)) + + if quantize_mode == "int8": + quantizer = TensorQuantizer(QuantizerAttributeConfig(num_bits=num_bits, axis=None)).to( + device + ) + model.global_pool.add_module("input_quantizer", quantizer) + model.global_pool.register_forward_pre_hook(_quantize_module_input) + residual_quantizers.append(quantizer) + + return residual_quantizers + + +def _disable_invalid_quantizers(quantizers): + for quantizer in quantizers: + if not quantizer.is_enabled: + continue + amax = quantizer.amax + if ( + amax is None + or not torch.is_tensor(amax) + or torch.any(torch.isnan(amax)) + or torch.all(amax <= 0) + ): + quantizer.disable() + + +def _add_auto_resnet_residual_quantizers(model, auto_quantization_formats, data_loader): + """Calibrate shortcuts with each block's AutoQuantize-selected activation format.""" + activation_formats = set(auto_quantization_formats) - {"INT4_AWQ_CFG"} + if not activation_formats: + return + fallback_num_bits = 8 if activation_formats == {"INT8_DEFAULT_CFG"} else (4, 3) + residual_quantizers = [] + block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) + for block in model.modules(): + if not isinstance(block, block_types): + continue + last_conv = block.conv3 if isinstance(block, timm.models.resnet.Bottleneck) else block.conv2 + selected_quantizer = getattr(last_conv, "input_quantizer", None) + num_bits = ( + selected_quantizer.num_bits + if selected_quantizer is not None + and selected_quantizer.is_enabled + and selected_quantizer.num_bits in (8, (4, 3)) + else fallback_num_bits + ) + residual_quantizers.append(_append_resnet_residual_quantizer(block, num_bits)) + + _calibrate_new_quantizers(model, residual_quantizers, data_loader) + + +def _finalize_resnet_quantizers( + model, quantize_mode, auto_quantization_formats, data_loader, residual_quantizers +): + block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) + blocks = [block for block in model.modules() if isinstance(block, block_types)] + if not blocks: + return + + if quantize_mode in ("int8", "fp8"): + for block in blocks: + block.conv1.input_quantizer.disable() + if block.downsample is not None: + downsample_conv = next( + ( + module + for module in block.downsample.modules() + if isinstance(module, torch.nn.Conv2d) + ), + None, + ) + if downsample_conv is not None: + downsample_conv.input_quantizer.disable() + model.fc.input_quantizer.disable() + model.fc.weight_quantizer.disable() + if quantize_mode == "int8": + model.global_pool.input_quantizer.enable() + elif quantize_mode == "auto": + _add_auto_resnet_residual_quantizers(model, auto_quantization_formats, data_loader) + + _disable_invalid_quantizers(residual_quantizers) def load_calibration_data(model, data_size, batch_size, device, with_labels=False): @@ -597,6 +718,8 @@ def main(): ) print(f"Base Model - Top-1 Accuracy: {top1:.2f}%, Top-5 Accuracy: {top5:.2f}%") + residual_quantizers = [] + # Quantize model based on mode if args.quantize_mode == "auto": # Auto quantization requires labels for loss computation @@ -622,6 +745,14 @@ def main(): # Conv2d layers are overridden to FP8 (for TRT compatibility), those FP8 # quantizers require calibration data. config = get_quant_config(args.quantize_mode) + block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) + if args.quantize_mode != "int4_awq" and any( + isinstance(module, block_types) for module in model.modules() + ): + conversion_config = copy.deepcopy(config) + conversion_config["algorithm"] = None + model = mtq.quantize(model, conversion_config) + residual_quantizers = _prepare_resnet_quantizers(model, args.quantize_mode) data_loader = load_calibration_data( model, @@ -633,11 +764,12 @@ def main(): quantized_model = quantize_model(model, config, data_loader) - _add_resnet_residual_quantizers( + _finalize_resnet_quantizers( quantized_model, args.quantize_mode, args.auto_quantization_formats, data_loader, + residual_quantizers, ) # MXFP8/NVFP4 lower their input quantizers to TRT DynamicQuantize (2D/3D only). diff --git a/modelopt/onnx/export/fp8_exporter.py b/modelopt/onnx/export/fp8_exporter.py index 427a7791f3b..fcad21d331e 100644 --- a/modelopt/onnx/export/fp8_exporter.py +++ b/modelopt/onnx/export/fp8_exporter.py @@ -31,6 +31,7 @@ # when using 1/448 as the Q scale (single fixed value — softmax range is data-independent). _FP8_E4M3_MAX = 448.0 _FP8_E4M3_SOFTMAX_SCALE = 1.0 / _FP8_E4M3_MAX +_FP8_MIN_CONV_CHANNELS = 16 class FP8QuantExporter(ONNXQuantExporter): @@ -189,6 +190,10 @@ def _quantize_conv_weights_to_fp8(graph: gs.Graph) -> int: weight_input = node.inputs[1] if not isinstance(weight_input, gs.Constant): continue + if any( + channels <= _FP8_MIN_CONV_CHANNELS for channels in weight_input.values.shape[:2] + ): + continue # Skip if weight already has a DQ producer if any(out.op == "DequantizeLinear" for out in weight_input.outputs): diff --git a/modelopt/onnx/utils.py b/modelopt/onnx/utils.py index bc37c4a9333..0619c3ec3a9 100644 --- a/modelopt/onnx/utils.py +++ b/modelopt/onnx/utils.py @@ -28,6 +28,7 @@ import onnx_graphsurgeon as gs from onnx.helper import get_attribute_value from onnx_graphsurgeon import Constant, Node, Variable +from onnxconverter_common.float16 import convert_np_to_float16 from modelopt.onnx.logging_config import logger @@ -1467,13 +1468,17 @@ def fold_q_fp16_to_fp32_casts(onnx_model: onnx.ModelProto) -> onnx.ModelProto: for inp in node.input: consumer_map.setdefault(inp, []).append(node) initializers = {init.name: init for init in onnx_model.graph.initializer} + tensor_types = _build_tensor_type_map(onnx_model) to_remove = [] for node in onnx_model.graph.node: if node.op_type != "Cast": continue cast_to = next((a.i for a in node.attribute if a.name == "to"), None) - if cast_to != onnx.TensorProto.FLOAT: + if ( + cast_to != onnx.TensorProto.FLOAT + or tensor_types.get(node.input[0]) != onnx.TensorProto.FLOAT16 + ): continue consumers = consumer_map.get(node.output[0], []) if not consumers or not all(c.op_type in _Q_OPS for c in consumers): @@ -1492,6 +1497,53 @@ def fold_q_fp16_to_fp32_casts(onnx_model: onnx.ModelProto) -> onnx.ModelProto: return onnx_model +def _convert_q_data_initializers_to_fp16(onnx_model: onnx.ModelProto) -> onnx.ModelProto: + """Convert FP32 initializer data inputs after Q/DQ scales have been normalized to FP16.""" + if get_opset_version(onnx_model) < BASE_MIN_OPSET: + return onnx_model + + consumers: dict[str, list[tuple[onnx.NodeProto, int]]] = defaultdict(list) + for node in onnx_model.graph.node: + for index, input_name in enumerate(node.input): + consumers[input_name].append((node, index)) + + initializers = {initializer.name: initializer for initializer in onnx_model.graph.initializer} + tensor_types = _build_tensor_type_map(onnx_model) + + for name, initializer in list(initializers.items()): + if initializer.data_type != onnx.TensorProto.FLOAT: + continue + + initializer_consumers = consumers.get(name, []) + q_consumers = [ + node for node, index in initializer_consumers if index == 0 and node.op_type in _Q_OPS + ] + if not q_consumers: + continue + + for q_node in q_consumers: + scale_type = tensor_types.get(q_node.input[1]) if len(q_node.input) >= 2 else None + if scale_type != onnx.TensorProto.FLOAT16: + raise ValueError("Q scales must be FP16 before converting Q data initializers") + + fp16_initializer = onnx.numpy_helper.from_array( + convert_np_to_float16(onnx.numpy_helper.to_array(initializer)), initializer.name + ) + if len(q_consumers) == len(initializer_consumers): + initializer.CopyFrom(fp16_initializer) + continue + + fp16_initializer.name = f"{initializer.name}_fp16_q" + while fp16_initializer.name in initializers: + fp16_initializer.name += "_" + onnx_model.graph.initializer.append(fp16_initializer) + initializers[fp16_initializer.name] = fp16_initializer + for node in q_consumers: + node.input[0] = fp16_initializer.name + + return onnx_model + + def _is_foldable_constant_cast_pattern(model: onnx.ModelProto, node: onnx.NodeProto) -> bool: """Check if a Constant -> Cast pattern can be folded.""" assert node.op_type == "Cast" diff --git a/modelopt/torch/_deploy/utils/torch_onnx.py b/modelopt/torch/_deploy/utils/torch_onnx.py index 01fb754bbae..6ef53bf33dd 100644 --- a/modelopt/torch/_deploy/utils/torch_onnx.py +++ b/modelopt/torch/_deploy/utils/torch_onnx.py @@ -45,6 +45,7 @@ ) from modelopt.onnx.quantization.qdq_utils import qdq_to_dq, replace_zero_scale_with_smallest_nonzero from modelopt.onnx.utils import ( + _convert_q_data_initializers_to_fp16, change_casts_to_fp16, check_model_uses_external_data, fold_dq_fp32_to_fp16_casts, @@ -665,9 +666,11 @@ def get_onnx_bytes_and_metadata( onnx_opt_graph = remove_redundant_casts(onnx_opt_graph) # Remove Cast nodes around Q/DQ for optimal TRT fusion - if is_fp8_quantized(model): + if is_fp8_quantized(model) or (is_int8_quantized(model) and weights_dtype == "fp16"): onnx_opt_graph = fold_q_fp16_to_fp32_casts(onnx_opt_graph) onnx_opt_graph = fold_dq_fp32_to_fp16_casts(onnx_opt_graph) + if is_int8_quantized(model) and weights_dtype == "fp16": + onnx_opt_graph = _convert_q_data_initializers_to_fp16(onnx_opt_graph) # TensorRT expects all scales to be postive onnx_opt_graph = replace_zero_scale_with_smallest_nonzero(onnx_opt_graph) diff --git a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py index 2a4fc9c4de0..7896e341c27 100644 --- a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py +++ b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py @@ -32,8 +32,9 @@ } -def _assert_residual_adds_are_quantized(onnx_save_path): +def _assert_residual_adds_are_quantized(onnx_save_path, quantize_mode): model = onnx.load(onnx_save_path) + initializers = {initializer.name: initializer for initializer in model.graph.initializer} consumers = defaultdict(list) producers = {} for node in model.graph.node: @@ -42,7 +43,12 @@ def _assert_residual_adds_are_quantized(onnx_save_path): for output_name in node.output: producers[output_name] = node - residual_adds = [node for node in model.graph.node if node.op_type == "Add"] + residual_adds = [ + node + for node in model.graph.node + if node.op_type == "Add" + and [consumer.op_type for consumer in consumers[node.output[0]]] == ["Relu"] + ] assert len(residual_adds) == 16 for add in residual_adds: input_producers = [producers[input_name] for input_name in add.input] @@ -54,7 +60,60 @@ def _assert_residual_adds_are_quantized(onnx_save_path): and producers[node.input[0]].op_type.endswith("QuantizeLinear") for node in input_producers ) - assert [node.op_type for node in consumers[add.output[0]]] == ["Relu"] + + if quantize_mode not in ("int8", "fp8"): + return + + activation_quantizers = [ + node + for node in model.graph.node + if node.op_type.endswith("QuantizeLinear") and node.input[0] not in initializers + ] + assert len(activation_quantizers) == (54 if quantize_mode == "int8" else 52) + assert len({node.input[0] for node in activation_quantizers}) == len(activation_quantizers) + assert all( + producers.get(node.input[0]) is None or producers[node.input[0]].op_type != "Cast" + for node in activation_quantizers + ) + + dq_fanouts = [ + sorted(consumer.op_type for consumer in consumers[node.output[0]]) + for node in model.graph.node + if node.op_type.endswith("DequantizeLinear") + ] + assert dq_fanouts.count(["Add", "Conv"]) == 12 + assert dq_fanouts.count(["Conv", "Conv"]) == 4 + assert dq_fanouts.count(["Add"]) == 4 + + gemm = next(node for node in model.graph.node if node.op_type == "Gemm") + assert all( + producers.get(input_name) is None + or not producers[input_name].op_type.endswith("DequantizeLinear") + for input_name in gemm.input + ) + + global_pool = next(node for node in model.graph.node if node.op_type == "GlobalAveragePool") + pool_input_producer = producers[global_pool.input[0]] + if quantize_mode == "int8": + assert pool_input_producer.op_type.endswith("DequantizeLinear") + weight_quantizers = [ + node + for node in model.graph.node + if node.op_type.endswith("QuantizeLinear") and node.input[0] in initializers + ] + assert len(weight_quantizers) == 53 + assert all( + initializers[node.input[0]].data_type == onnx.TensorProto.FLOAT16 + for node in weight_quantizers + ) + else: + assert pool_input_producer.op_type == "Relu" + first_conv = next(node for node in model.graph.node if node.op_type == "Conv") + assert all( + producers.get(input_name) is None + or not producers[input_name].op_type.endswith("DequantizeLinear") + for input_name in first_conv.input + ) @pytest.mark.parametrize("quantize_mode", _QUANT_MODES) @@ -76,4 +135,4 @@ def test_torch_onnx(tmp_path, model_key, quantize_mode): run_example_command(cmd_parts, "torch_onnx") if model_key == "resnet50": - _assert_residual_adds_are_quantized(onnx_save_path) + _assert_residual_adds_are_quantized(onnx_save_path, quantize_mode) diff --git a/tests/unit/onnx/quantization/test_fp8_mha_exporter.py b/tests/unit/onnx/quantization/test_fp8_mha_exporter.py index 1f7251a9ad9..391595033f0 100644 --- a/tests/unit/onnx/quantization/test_fp8_mha_exporter.py +++ b/tests/unit/onnx/quantization/test_fp8_mha_exporter.py @@ -40,6 +40,23 @@ def _graph(nodes, inputs, outputs): return gs.Graph(nodes=nodes, inputs=inputs, outputs=outputs, opset=19) +@pytest.mark.parametrize( + ("output_channels", "input_channels", "expected_count"), + [(64, 3, 0), (16, 64, 0), (64, 16, 0), (64, 17, 1)], +) +def test_quantize_conv_weights_to_fp8_skips_small_channels( + output_channels, input_channels, expected_count +): + x = gs.Variable("x", dtype=np.float16, shape=[1, input_channels, 32, 32]) + weight = gs.Constant( + "weight", np.ones((output_channels, input_channels, 3, 3), dtype=np.float16) + ) + y = gs.Variable("y", dtype=np.float16) + graph = _graph([gs.Node(op="Conv", inputs=[x, weight], outputs=[y])], [x], [y]) + + assert FP8QuantExporter._quantize_conv_weights_to_fp8(graph) == expected_count + + def test_move_mul_before_qdq_rewrites_dq_mul_matmul_pattern(): """``DQ → Mul(const) → MatMul`` collapses to ``Mul → Q → DQ → MatMul``.""" x, k, y, mul_out = _var("x"), _var("k"), _var("y"), _var("mul_out") diff --git a/tests/unit/onnx/test_fold_casts.py b/tests/unit/onnx/test_fold_casts.py index 59a434d1206..717ee5889b8 100644 --- a/tests/unit/onnx/test_fold_casts.py +++ b/tests/unit/onnx/test_fold_casts.py @@ -16,10 +16,15 @@ """Tests for the FP16 Q/DQ scale cast-folding helpers in ``modelopt.onnx.utils``.""" import numpy as np +import onnx import pytest from onnx import TensorProto, helper, numpy_helper -from modelopt.onnx.utils import fold_dq_fp32_to_fp16_casts, fold_q_fp16_to_fp32_casts +from modelopt.onnx.utils import ( + _convert_q_data_initializers_to_fp16, + fold_dq_fp32_to_fp16_casts, + fold_q_fp16_to_fp32_casts, +) def _dq_cast_model(opset): @@ -46,7 +51,7 @@ def _dq_cast_model(opset): ) -def _cast_q_model(opset): +def _cast_q_model(opset, input_dtype=TensorProto.FLOAT16): """``Cast(FP16→FP32) → Q → DQ → MatMul`` with FP32 scale.""" nodes = [ helper.make_node("Cast", ["x"], ["c_out"], "cast", to=TensorProto.FLOAT), @@ -63,7 +68,7 @@ def _cast_q_model(opset): helper.make_graph( nodes, "g", - [helper.make_tensor_value_info("x", TensorProto.FLOAT16, [None, 4])], + [helper.make_tensor_value_info("x", input_dtype, [None, 4])], [helper.make_tensor_value_info("y", TensorProto.FLOAT, [None, 4])], initializer=inits, ), @@ -71,6 +76,37 @@ def _cast_q_model(opset): ) +def _initializer_q_model(opset, shared=False): + scale_dtype = np.float16 if opset >= 19 else np.float32 + nodes = [ + helper.make_node("QuantizeLinear", ["w", "scale", "zp"], ["q_out"], "q"), + helper.make_node("DequantizeLinear", ["q_out", "scale", "zp"], ["dq_out"], "dq"), + helper.make_node("MatMul", ["x", "dq_out"], ["y"], "matmul"), + ] + outputs = [helper.make_tensor_value_info("y", TensorProto.FLOAT16, [None, 4])] + if shared: + nodes.append(helper.make_node("Identity", ["w"], ["w_out"], "identity")) + outputs.append(helper.make_tensor_value_info("w_out", TensorProto.FLOAT, [4, 4])) + inits = [ + numpy_helper.from_array(np.ones((4, 4), dtype=np.float32), "w"), + numpy_helper.from_array(np.array(0.1, dtype=scale_dtype), "scale"), + numpy_helper.from_array(np.array(0, dtype=np.int8), "zp"), + ] + input_dtype = TensorProto.FLOAT16 if opset >= 19 else TensorProto.FLOAT + output_dtype = TensorProto.FLOAT16 if opset >= 19 else TensorProto.FLOAT + outputs[0].type.tensor_type.elem_type = output_dtype + return helper.make_model( + helper.make_graph( + nodes, + "g", + [helper.make_tensor_value_info("x", input_dtype, [None, 4])], + outputs, + initializer=inits, + ), + opset_imports=[helper.make_opsetid("", opset)], + ) + + @pytest.mark.parametrize( ("fold_fn", "build_model", "scale_name"), [ @@ -97,3 +133,61 @@ def test_fold_is_noop_below_min_opset(fold_fn, build_model, scale_name): assert "Cast" in {n.op_type for n in folded.graph.node} scale = next(i for i in folded.graph.initializer if i.name == scale_name) assert scale.data_type == TensorProto.FLOAT + + +def test_fold_q_preserves_cast_with_non_fp16_source(): + model = _cast_q_model(opset=19, input_dtype=TensorProto.FLOAT) + folded = fold_q_fp16_to_fp32_casts(model) + assert "Cast" in {node.op_type for node in folded.graph.node} + scale = next( + initializer for initializer in folded.graph.initializer if initializer.name == "scale" + ) + assert scale.data_type == TensorProto.FLOAT + onnx.checker.check_model(folded) + + +@pytest.mark.parametrize("shared", [False, True]) +def test_convert_q_data_initializers_to_fp16(shared): + converted = _convert_q_data_initializers_to_fp16(_initializer_q_model(opset=19, shared=shared)) + initializers = {initializer.name: initializer for initializer in converted.graph.initializer} + q_node = next(node for node in converted.graph.node if node.op_type == "QuantizeLinear") + + assert initializers[q_node.input[0]].data_type == TensorProto.FLOAT16 + assert initializers[q_node.input[1]].data_type == TensorProto.FLOAT16 + if shared: + identity = next(node for node in converted.graph.node if node.op_type == "Identity") + assert identity.input[0] == "w" + assert initializers["w"].data_type == TensorProto.FLOAT + else: + assert q_node.input[0] == "w" + onnx.checker.check_model(converted) + + +def test_convert_q_data_initializers_is_noop_below_min_opset(): + model = _initializer_q_model(opset=18) + converted = _convert_q_data_initializers_to_fp16(model) + weight = next( + initializer for initializer in converted.graph.initializer if initializer.name == "w" + ) + assert weight.data_type == TensorProto.FLOAT + + +def test_convert_q_data_initializers_requires_fp16_scale(): + model = _initializer_q_model(opset=19) + scale = next( + initializer for initializer in model.graph.initializer if initializer.name == "scale" + ) + scale.CopyFrom(numpy_helper.from_array(np.array(0.1, dtype=np.float32), "scale")) + with pytest.raises(ValueError, match="Q scales must be FP16"): + _convert_q_data_initializers_to_fp16(model) + + +def test_convert_q_data_initializers_rejects_fp32_graph_input_scale(): + model = _initializer_q_model(opset=19) + scale = next( + initializer for initializer in model.graph.initializer if initializer.name == "scale" + ) + model.graph.initializer.remove(scale) + model.graph.input.append(helper.make_tensor_value_info("scale", TensorProto.FLOAT, [])) + with pytest.raises(ValueError, match="Q scales must be FP16"): + _convert_q_data_initializers_to_fp16(model) From 7cf4bf5443cd472e4a264b1fbb27aebdff01221e Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:03:24 +0000 Subject: [PATCH 06/10] Scope input stem exclusion to ResNet Co-Authored-By: Codex Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- examples/torch_onnx/torch_quant_to_onnx.py | 40 ++++--------------- modelopt/onnx/export/fp8_exporter.py | 10 ++--- .../torch_onnx/test_torch_quant_to_onnx.py | 17 ++++---- .../quantization/test_fp8_mha_exporter.py | 21 ++++++---- 4 files changed, 34 insertions(+), 54 deletions(-) diff --git a/examples/torch_onnx/torch_quant_to_onnx.py b/examples/torch_onnx/torch_quant_to_onnx.py index d7720e3156d..6c4a8de0a88 100644 --- a/examples/torch_onnx/torch_quant_to_onnx.py +++ b/examples/torch_onnx/torch_quant_to_onnx.py @@ -152,15 +152,19 @@ def get_quant_config(quantize_mode): return config -def filter_func(name): +def filter_func(name, model=None): """Filter function to exclude certain layers from quantization. + A ResNet's top-level ``conv1`` consumes three-channel image input, for which TensorRT + cannot find an FP8 Q→Conv tactic on Blackwell. ``downsample.reduction`` (Swin/SwinV2) is excluded because it operates on 4D tensors and TRT's DynamicQuantize layer (used for MXFP8/NVFP4) requires 2D/3D input. Other 4D-input layers (e.g. Swin's ``norm1``, ``downsample.norm``, top-level ``norm``) are handled dynamically by ``_disable_high_rank_input_quantizers`` via a forward-pass rank probe — that avoids false positives on ViT, whose same-named ``norm`` sees 3D input. """ + if isinstance(model, timm.models.resnet.ResNet) and name.startswith("conv1."): + return True pattern = re.compile( r".*(time_emb_proj|time_embedding|conv_in|conv_out|conv_shortcut|add_embedding|" r"pos_embed|time_text_embed|context_embedder|norm_out|x_embedder|patch_embed|cpb_mlp|" @@ -204,27 +208,6 @@ def hook(m, inp, out, _n=name): mtq.disable_quantizer(model, lambda n: n.startswith(prefixes)) -def _disable_low_channel_conv_input_quantizers(model): - """Disable input quantization on Conv2d modules with at most 16 input or output channels. - - The first Conv2d of an image backbone (e.g. ResNet50's ``conv1``) consumes raw - RGB input. TensorRT does not reliably accelerate FP8 convolutions with such small - channel dimensions, so ONNX PTQ leaves the entire Conv in high precision. On - Blackwell, TRT can also fail to find a tactic for the first-layer Q→Conv fusion: - - Error Code 10: Could not find any implementation for node - /conv1/input_quantizer/TRT_FP8QuantizeLinear ... [ElementWise] - - Ada (8.9) happens to have a tactic, which is why local runs pass. Swin/ViT's - ``patch_embed.proj`` is already excluded via ``filter_func``. - """ - for _, mod in model.named_modules(): - if isinstance(mod, torch.nn.Conv2d) and min(mod.in_channels, mod.out_channels) <= 16: - q = getattr(mod, "input_quantizer", None) - if q is not None and q.is_enabled: - q.disable() - - def _quantize_module_input(module, inputs): return (module.input_quantizer(inputs[0]), *inputs[1:]) @@ -492,7 +475,7 @@ def forward_loop(model): # Disable filtered quantizers BEFORE calibrating override quantizers so we don't # waste time calibrating quantizers that are about to be turned off. - mtq.disable_quantizer(quantized_model, filter_func) + mtq.disable_quantizer(quantized_model, lambda name: filter_func(name, quantized_model)) # Calibrate any FP8 override quantizers that weren't calibrated by mtq.quantize(). if data_loader is not None: @@ -584,7 +567,7 @@ def auto_quantize_model( ) # Disable quantization for specified layers - mtq.disable_quantizer(quantized_model, filter_func) + mtq.disable_quantizer(quantized_model, lambda name: filter_func(name, quantized_model)) _disable_dead_quantizers(quantized_model) @@ -782,15 +765,6 @@ def main(): if uses_dynamic_quantize: _disable_high_rank_input_quantizers(quantized_model, input_shape, device) - # FP8-family modes emit TRT_FP8QuantizeLinear on the first-layer conv; Blackwell has - # no tactic for that 3-channel Q→Conv fusion. Skip for pure INT8 (unaffected). - uses_fp8_conv_input = args.quantize_mode in ("fp8", "mxfp8", "nvfp4") or ( - args.quantize_mode == "auto" - and any(fmt != "INT8_DEFAULT_CFG" for fmt in args.auto_quantization_formats) - ) - if uses_fp8_conv_input: - _disable_low_channel_conv_input_quantizers(quantized_model) - # Print quantization summary print("\nQuantization Summary:") mtq.print_quant_summary(quantized_model) diff --git a/modelopt/onnx/export/fp8_exporter.py b/modelopt/onnx/export/fp8_exporter.py index fcad21d331e..a80ad3dbb78 100644 --- a/modelopt/onnx/export/fp8_exporter.py +++ b/modelopt/onnx/export/fp8_exporter.py @@ -31,7 +31,6 @@ # when using 1/448 as the Q scale (single fixed value — softmax range is data-independent). _FP8_E4M3_MAX = 448.0 _FP8_E4M3_SOFTMAX_SCALE = 1.0 / _FP8_E4M3_MAX -_FP8_MIN_CONV_CHANNELS = 16 class FP8QuantExporter(ONNXQuantExporter): @@ -173,6 +172,9 @@ def _quantize_conv_weights_to_fp8(graph: gs.Graph) -> int: 2. Quantize weights to FP8E4M3FN 3. Insert a DequantizeLinear(fp8_weights, scale) before the Conv weight input + An RGB Conv that directly consumes an unquantized graph input is treated as a + filtered input stem and left entirely in high precision. + Args: graph: The onnx-graphsurgeon graph to modify in-place. @@ -180,6 +182,7 @@ def _quantize_conv_weights_to_fp8(graph: gs.Graph) -> int: Number of Conv weight DQ nodes inserted. """ count = 0 + graph_inputs = {tensor.name for tensor in graph.inputs} for node in list(graph.nodes): if node.op != "Conv": @@ -190,11 +193,8 @@ def _quantize_conv_weights_to_fp8(graph: gs.Graph) -> int: weight_input = node.inputs[1] if not isinstance(weight_input, gs.Constant): continue - if any( - channels <= _FP8_MIN_CONV_CHANNELS for channels in weight_input.values.shape[:2] - ): + if node.inputs[0].name in graph_inputs and weight_input.values.shape[1] == 3: continue - # Skip if weight already has a DQ producer if any(out.op == "DequantizeLinear" for out in weight_input.outputs): continue diff --git a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py index 7896e341c27..15785bc4fb4 100644 --- a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py +++ b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py @@ -61,6 +61,13 @@ def _assert_residual_adds_are_quantized(onnx_save_path, quantize_mode): for node in input_producers ) + first_conv = next(node for node in model.graph.node if node.op_type == "Conv") + assert all( + producers.get(input_name) is None + or not producers[input_name].op_type.endswith("DequantizeLinear") + for input_name in first_conv.input + ) + if quantize_mode not in ("int8", "fp8"): return @@ -69,7 +76,7 @@ def _assert_residual_adds_are_quantized(onnx_save_path, quantize_mode): for node in model.graph.node if node.op_type.endswith("QuantizeLinear") and node.input[0] not in initializers ] - assert len(activation_quantizers) == (54 if quantize_mode == "int8" else 52) + assert len(activation_quantizers) == (53 if quantize_mode == "int8" else 52) assert len({node.input[0] for node in activation_quantizers}) == len(activation_quantizers) assert all( producers.get(node.input[0]) is None or producers[node.input[0]].op_type != "Cast" @@ -101,19 +108,13 @@ def _assert_residual_adds_are_quantized(onnx_save_path, quantize_mode): for node in model.graph.node if node.op_type.endswith("QuantizeLinear") and node.input[0] in initializers ] - assert len(weight_quantizers) == 53 + assert len(weight_quantizers) == 52 assert all( initializers[node.input[0]].data_type == onnx.TensorProto.FLOAT16 for node in weight_quantizers ) else: assert pool_input_producer.op_type == "Relu" - first_conv = next(node for node in model.graph.node if node.op_type == "Conv") - assert all( - producers.get(input_name) is None - or not producers[input_name].op_type.endswith("DequantizeLinear") - for input_name in first_conv.input - ) @pytest.mark.parametrize("quantize_mode", _QUANT_MODES) diff --git a/tests/unit/onnx/quantization/test_fp8_mha_exporter.py b/tests/unit/onnx/quantization/test_fp8_mha_exporter.py index 391595033f0..7ed6011265b 100644 --- a/tests/unit/onnx/quantization/test_fp8_mha_exporter.py +++ b/tests/unit/onnx/quantization/test_fp8_mha_exporter.py @@ -41,18 +41,23 @@ def _graph(nodes, inputs, outputs): @pytest.mark.parametrize( - ("output_channels", "input_channels", "expected_count"), - [(64, 3, 0), (16, 64, 0), (64, 16, 0), (64, 17, 1)], + ("direct_input", "input_channels", "expected_count"), + [(True, 3, 0), (True, 17, 1), (False, 3, 1)], ) -def test_quantize_conv_weights_to_fp8_skips_small_channels( - output_channels, input_channels, expected_count +def test_quantize_conv_weights_to_fp8_skips_unquantized_rgb_graph_input( + direct_input, input_channels, expected_count ): x = gs.Variable("x", dtype=np.float16, shape=[1, input_channels, 32, 32]) - weight = gs.Constant( - "weight", np.ones((output_channels, input_channels, 3, 3), dtype=np.float16) - ) + conv_input = x + nodes = [] + if not direct_input: + conv_input = gs.Variable("conv_input", dtype=np.float16, shape=x.shape) + nodes.append(gs.Node(op="Identity", inputs=[x], outputs=[conv_input])) + + weight = gs.Constant("weight", np.ones((64, input_channels, 3, 3), dtype=np.float16)) y = gs.Variable("y", dtype=np.float16) - graph = _graph([gs.Node(op="Conv", inputs=[x, weight], outputs=[y])], [x], [y]) + nodes.append(gs.Node(op="Conv", inputs=[conv_input, weight], outputs=[y])) + graph = _graph(nodes, [x], [y]) assert FP8QuantExporter._quantize_conv_weights_to_fp8(graph) == expected_count From 9cbcbbdc6da50d1055874051b571385cfbd2bb43 Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:09:31 +0000 Subject: [PATCH 07/10] Address follow-up export review feedback Co-Authored-By: Codex Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- examples/torch_onnx/torch_quant_to_onnx.py | 20 +++++++++++++++----- modelopt/onnx/utils.py | 14 +++++++++++++- tests/unit/onnx/test_fold_casts.py | 20 ++++++++++++++------ 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/examples/torch_onnx/torch_quant_to_onnx.py b/examples/torch_onnx/torch_quant_to_onnx.py index 6c4a8de0a88..9b7414197bb 100644 --- a/examples/torch_onnx/torch_quant_to_onnx.py +++ b/examples/torch_onnx/torch_quant_to_onnx.py @@ -343,6 +343,15 @@ def _add_auto_resnet_residual_quantizers(model, auto_quantization_formats, data_ _calibrate_new_quantizers(model, residual_quantizers, data_loader) +def _set_quantizer_enabled(module, name, enabled): + quantizer = getattr(module, name, None) if module is not None else None + if quantizer is not None: + if enabled: + quantizer.enable() + else: + quantizer.disable() + + def _finalize_resnet_quantizers( model, quantize_mode, auto_quantization_formats, data_loader, residual_quantizers ): @@ -353,7 +362,7 @@ def _finalize_resnet_quantizers( if quantize_mode in ("int8", "fp8"): for block in blocks: - block.conv1.input_quantizer.disable() + _set_quantizer_enabled(getattr(block, "conv1", None), "input_quantizer", False) if block.downsample is not None: downsample_conv = next( ( @@ -364,11 +373,12 @@ def _finalize_resnet_quantizers( None, ) if downsample_conv is not None: - downsample_conv.input_quantizer.disable() - model.fc.input_quantizer.disable() - model.fc.weight_quantizer.disable() + _set_quantizer_enabled(downsample_conv, "input_quantizer", False) + fc = getattr(model, "fc", None) + _set_quantizer_enabled(fc, "input_quantizer", False) + _set_quantizer_enabled(fc, "weight_quantizer", False) if quantize_mode == "int8": - model.global_pool.input_quantizer.enable() + _set_quantizer_enabled(getattr(model, "global_pool", None), "input_quantizer", True) elif quantize_mode == "auto": _add_auto_resnet_residual_quantizers(model, auto_quantization_formats, data_loader) diff --git a/modelopt/onnx/utils.py b/modelopt/onnx/utils.py index 0619c3ec3a9..1d2c260a7d3 100644 --- a/modelopt/onnx/utils.py +++ b/modelopt/onnx/utils.py @@ -1521,10 +1521,22 @@ def _convert_q_data_initializers_to_fp16(onnx_model: onnx.ModelProto) -> onnx.Mo if not q_consumers: continue + invalid_scale = None for q_node in q_consumers: scale_type = tensor_types.get(q_node.input[1]) if len(q_node.input) >= 2 else None if scale_type != onnx.TensorProto.FLOAT16: - raise ValueError("Q scales must be FP16 before converting Q data initializers") + invalid_scale = (q_node, scale_type) + break + if invalid_scale is not None: + q_node, scale_type = invalid_scale + scale_type_name = ( + "unknown" if scale_type is None else onnx.TensorProto.DataType.Name(scale_type) + ) + logger.warning( + f"Skipping FP16 conversion for Q data initializer '{name}': " + f"node '{q_node.name}' has {scale_type_name} scale" + ) + continue fp16_initializer = onnx.numpy_helper.from_array( convert_np_to_float16(onnx.numpy_helper.to_array(initializer)), initializer.name diff --git a/tests/unit/onnx/test_fold_casts.py b/tests/unit/onnx/test_fold_casts.py index 717ee5889b8..cc60435b1ce 100644 --- a/tests/unit/onnx/test_fold_casts.py +++ b/tests/unit/onnx/test_fold_casts.py @@ -172,22 +172,30 @@ def test_convert_q_data_initializers_is_noop_below_min_opset(): assert weight.data_type == TensorProto.FLOAT -def test_convert_q_data_initializers_requires_fp16_scale(): +def test_convert_q_data_initializers_skips_non_fp16_scale(caplog): model = _initializer_q_model(opset=19) scale = next( initializer for initializer in model.graph.initializer if initializer.name == "scale" ) scale.CopyFrom(numpy_helper.from_array(np.array(0.1, dtype=np.float32), "scale")) - with pytest.raises(ValueError, match="Q scales must be FP16"): - _convert_q_data_initializers_to_fp16(model) + converted = _convert_q_data_initializers_to_fp16(model) + weight = next( + initializer for initializer in converted.graph.initializer if initializer.name == "w" + ) + assert weight.data_type == TensorProto.FLOAT + assert "node 'q' has FLOAT scale" in caplog.text -def test_convert_q_data_initializers_rejects_fp32_graph_input_scale(): +def test_convert_q_data_initializers_skips_fp32_graph_input_scale(caplog): model = _initializer_q_model(opset=19) scale = next( initializer for initializer in model.graph.initializer if initializer.name == "scale" ) model.graph.initializer.remove(scale) model.graph.input.append(helper.make_tensor_value_info("scale", TensorProto.FLOAT, [])) - with pytest.raises(ValueError, match="Q scales must be FP16"): - _convert_q_data_initializers_to_fp16(model) + converted = _convert_q_data_initializers_to_fp16(model) + weight = next( + initializer for initializer in converted.graph.initializer if initializer.name == "w" + ) + assert weight.data_type == TensorProto.FLOAT + assert "node 'q' has FLOAT scale" in caplog.text From 552ac54054f733fde6ec3836feef32f62aca5ab2 Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:10:22 +0000 Subject: [PATCH 08/10] Revert follow-up review fixes Co-Authored-By: Codex Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- examples/torch_onnx/torch_quant_to_onnx.py | 20 +++++--------------- modelopt/onnx/utils.py | 14 +------------- tests/unit/onnx/test_fold_casts.py | 20 ++++++-------------- 3 files changed, 12 insertions(+), 42 deletions(-) diff --git a/examples/torch_onnx/torch_quant_to_onnx.py b/examples/torch_onnx/torch_quant_to_onnx.py index 9b7414197bb..6c4a8de0a88 100644 --- a/examples/torch_onnx/torch_quant_to_onnx.py +++ b/examples/torch_onnx/torch_quant_to_onnx.py @@ -343,15 +343,6 @@ def _add_auto_resnet_residual_quantizers(model, auto_quantization_formats, data_ _calibrate_new_quantizers(model, residual_quantizers, data_loader) -def _set_quantizer_enabled(module, name, enabled): - quantizer = getattr(module, name, None) if module is not None else None - if quantizer is not None: - if enabled: - quantizer.enable() - else: - quantizer.disable() - - def _finalize_resnet_quantizers( model, quantize_mode, auto_quantization_formats, data_loader, residual_quantizers ): @@ -362,7 +353,7 @@ def _finalize_resnet_quantizers( if quantize_mode in ("int8", "fp8"): for block in blocks: - _set_quantizer_enabled(getattr(block, "conv1", None), "input_quantizer", False) + block.conv1.input_quantizer.disable() if block.downsample is not None: downsample_conv = next( ( @@ -373,12 +364,11 @@ def _finalize_resnet_quantizers( None, ) if downsample_conv is not None: - _set_quantizer_enabled(downsample_conv, "input_quantizer", False) - fc = getattr(model, "fc", None) - _set_quantizer_enabled(fc, "input_quantizer", False) - _set_quantizer_enabled(fc, "weight_quantizer", False) + downsample_conv.input_quantizer.disable() + model.fc.input_quantizer.disable() + model.fc.weight_quantizer.disable() if quantize_mode == "int8": - _set_quantizer_enabled(getattr(model, "global_pool", None), "input_quantizer", True) + model.global_pool.input_quantizer.enable() elif quantize_mode == "auto": _add_auto_resnet_residual_quantizers(model, auto_quantization_formats, data_loader) diff --git a/modelopt/onnx/utils.py b/modelopt/onnx/utils.py index 1d2c260a7d3..0619c3ec3a9 100644 --- a/modelopt/onnx/utils.py +++ b/modelopt/onnx/utils.py @@ -1521,22 +1521,10 @@ def _convert_q_data_initializers_to_fp16(onnx_model: onnx.ModelProto) -> onnx.Mo if not q_consumers: continue - invalid_scale = None for q_node in q_consumers: scale_type = tensor_types.get(q_node.input[1]) if len(q_node.input) >= 2 else None if scale_type != onnx.TensorProto.FLOAT16: - invalid_scale = (q_node, scale_type) - break - if invalid_scale is not None: - q_node, scale_type = invalid_scale - scale_type_name = ( - "unknown" if scale_type is None else onnx.TensorProto.DataType.Name(scale_type) - ) - logger.warning( - f"Skipping FP16 conversion for Q data initializer '{name}': " - f"node '{q_node.name}' has {scale_type_name} scale" - ) - continue + raise ValueError("Q scales must be FP16 before converting Q data initializers") fp16_initializer = onnx.numpy_helper.from_array( convert_np_to_float16(onnx.numpy_helper.to_array(initializer)), initializer.name diff --git a/tests/unit/onnx/test_fold_casts.py b/tests/unit/onnx/test_fold_casts.py index cc60435b1ce..717ee5889b8 100644 --- a/tests/unit/onnx/test_fold_casts.py +++ b/tests/unit/onnx/test_fold_casts.py @@ -172,30 +172,22 @@ def test_convert_q_data_initializers_is_noop_below_min_opset(): assert weight.data_type == TensorProto.FLOAT -def test_convert_q_data_initializers_skips_non_fp16_scale(caplog): +def test_convert_q_data_initializers_requires_fp16_scale(): model = _initializer_q_model(opset=19) scale = next( initializer for initializer in model.graph.initializer if initializer.name == "scale" ) scale.CopyFrom(numpy_helper.from_array(np.array(0.1, dtype=np.float32), "scale")) - converted = _convert_q_data_initializers_to_fp16(model) - weight = next( - initializer for initializer in converted.graph.initializer if initializer.name == "w" - ) - assert weight.data_type == TensorProto.FLOAT - assert "node 'q' has FLOAT scale" in caplog.text + with pytest.raises(ValueError, match="Q scales must be FP16"): + _convert_q_data_initializers_to_fp16(model) -def test_convert_q_data_initializers_skips_fp32_graph_input_scale(caplog): +def test_convert_q_data_initializers_rejects_fp32_graph_input_scale(): model = _initializer_q_model(opset=19) scale = next( initializer for initializer in model.graph.initializer if initializer.name == "scale" ) model.graph.initializer.remove(scale) model.graph.input.append(helper.make_tensor_value_info("scale", TensorProto.FLOAT, [])) - converted = _convert_q_data_initializers_to_fp16(model) - weight = next( - initializer for initializer in converted.graph.initializer if initializer.name == "w" - ) - assert weight.data_type == TensorProto.FLOAT - assert "node 'q' has FLOAT scale" in caplog.text + with pytest.raises(ValueError, match="Q scales must be FP16"): + _convert_q_data_initializers_to_fp16(model) From edc43d46ca83ccd0a0bb122d3972721b0e352f73 Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:52:19 +0000 Subject: [PATCH 09/10] [OMNIML-5613] Add ResNet quantization recipes Co-Authored-By: Codex Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- CHANGELOG.rst | 6 +- examples/torch_onnx/README.md | 5 +- examples/torch_onnx/torch_quant_to_onnx.py | 304 +++--------- modelopt/torch/opt/dynamic.py | 7 + modelopt/torch/quantization/algorithms.py | 170 +++++-- modelopt/torch/quantization/conversion.py | 20 +- .../torch/quantization/plugins/__init__.py | 3 + modelopt/torch/quantization/plugins/timm.py | 287 +++++++++++ modelopt_recipes/README.md | 8 + modelopt_recipes/timm/resnet/ptq/README.md | 24 + modelopt_recipes/timm/resnet/ptq/fp8.yaml | 22 + modelopt_recipes/timm/resnet/ptq/int8.yaml | 28 ++ modelopt_recipes/timm/resnet/ptq/mxfp8.yaml | 25 + modelopt_recipes/timm/resnet/ptq/nvfp4.yaml | 20 + .../timm/resnet/ptq/nvfp4_awq_lite.yaml | 20 + .../timm/resnet/ptq/static_fp8.quant_cfg.yaml | 36 ++ .../resnet/ptq/static_int8.quant_cfg.yaml | 40 ++ tests/unit/torch/nas/test_registry.py | 3 + .../torch/quantization/plugins/test_timm.py | 469 ++++++++++++++++++ .../unit/torch/quantization/test_autoquant.py | 43 +- .../quantization/test_config_validation.py | 25 +- .../torch/quantization/test_quantize_cpu.py | 29 ++ 22 files changed, 1294 insertions(+), 300 deletions(-) create mode 100644 modelopt/torch/quantization/plugins/timm.py create mode 100644 modelopt_recipes/timm/resnet/ptq/README.md create mode 100644 modelopt_recipes/timm/resnet/ptq/fp8.yaml create mode 100644 modelopt_recipes/timm/resnet/ptq/int8.yaml create mode 100644 modelopt_recipes/timm/resnet/ptq/mxfp8.yaml create mode 100644 modelopt_recipes/timm/resnet/ptq/nvfp4.yaml create mode 100644 modelopt_recipes/timm/resnet/ptq/nvfp4_awq_lite.yaml create mode 100644 modelopt_recipes/timm/resnet/ptq/static_fp8.quant_cfg.yaml create mode 100644 modelopt_recipes/timm/resnet/ptq/static_int8.quant_cfg.yaml create mode 100644 tests/unit/torch/quantization/plugins/test_timm.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9aedad8172f..cad42607819 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -81,8 +81,10 @@ Changelog **Bug Fixes** -- Share each ResNet block-input Q/DQ between the main and shortcut paths in the torch ONNX - example, restoring TensorRT residual fusion for INT8 and FP8 exports. +- Add timm ResNet PTQ recipes for FP8, INT8, MXFP8, and NVFP4. The recipes + keep the input stem convolution unquantized, select TensorRT-compatible convolution formats, + and place shared block-input and projection-shortcut Q/DQ before residual adds, including + during AutoQuantize. - Fix ``ShapeInferenceError`` during ONNX INT8 + FP16 quantization (``--high_precision_dtype fp16``) of weakly-typed models (e.g. TensorFlow exports) that carry stale rank-0 ``graph.output`` shapes or ops such as ``TopK`` that ONNX's static shape inference cannot resolve. ``clear_stale_value_info`` now reconciles stale output shapes via symbolic shape inference (keeping every output's shape field populated), and AutoCast runs ONNX shape inference in strict mode and falls back to schema-based standalone type inference when it fails, so unresolved ops no longer leave tensors untyped. - Fused MoE expert auto-detection (``register_fused_experts_on_the_fly``) no longer requires an ``act_fn`` attribute. Some fused-expert modules (e.g. ``MiniMaxM3VLExperts``) apply a custom gated activation between the two ``F.linear`` calls instead of exposing ``act_fn``; they were silently skipped, leaving routed experts unquantized (an experts-only recipe matched nothing) and failing HF export with ``NotImplementedError``. ``_QuantFusedExperts`` is activation-agnostic (it only intercepts the two ``F.linear`` calls), so the requirement was unnecessary. This enables NVFP4/FP8 quantization and export for MiniMax-M2 / MiniMax-M3. - Fix unified HF export emitting transformers' *in-memory* (post-``conversion_mapping``) tensor names instead of the original model-hub names, breaking the unified-checkpoint contract (observed on MiniMax-M3: exported ``model.language_model.*`` / ``mlp.experts.*.gate_proj`` instead of hub ``language_model.model.*`` / ``block_sparse_moe.experts.*.w{1,2,3}``). transformers' own save-side ``revert_weight_conversion`` is disabled by ModelOpt because it raises ``RuntimeError`` on 0-d scalar scale tensors, so a new quant-aware reverse conversion (``modelopt/torch/export/quant_aware_conversion.py``) derives rename/split rules from the model's conversion mapping via transformers' ``reverse_transform()`` and carries each weight's companion scale tensors (``weight_scale``, ``weight_scale_2``, ``input_scale``, ``weight_scale_inv``, ``bias``) through the renames and un-fusions, so quantized exports round-trip to the hub names. Any mapping op that cannot be reversed quant-aware yet (e.g. still-stacked fused experts) falls back to the previous in-memory names instead of aborting the export. diff --git a/examples/torch_onnx/README.md b/examples/torch_onnx/README.md index a5f517e0356..8e1e3f1ac25 100644 --- a/examples/torch_onnx/README.md +++ b/examples/torch_onnx/README.md @@ -54,7 +54,8 @@ The `torch_quant_to_onnx.py` script quantizes [timm](https://github.com/huggingf - Loads a pretrained timm torch model (default: ViT-Base). - Quantizes the torch model to FP8, MXFP8, INT8, NVFP4, or INT4_AWQ using ModelOpt. - For models with Conv2d layers (e.g., SwinTransformer), automatically overrides Conv2d quantization to FP8 (for MXFP8/NVFP4 modes) or INT8 (for INT4_AWQ mode) for TensorRT compatibility. -- Quantizes ResNet shortcut inputs before residual addition for activation-quantized modes. +- Uses the [timm ResNet PTQ recipes](../../modelopt_recipes/timm/resnet/ptq/) to + quantize shortcut inputs before residual addition. - Exports the quantized model to ONNX. - Postprocesses the ONNX model to be compatible with TensorRT. - Saves the final ONNX model. @@ -274,7 +275,7 @@ The `auto` mode enables mixed precision quantization by searching for the optima | Parameter | Default | Description | | :--- | :---: | :--- | -| `--effective_bits` | 4.8 | Target average bits per weight across the model. Lower values = more compression but potentially lower accuracy. The search algorithm finds the optimal per-layer format assignment that meets this constraint while minimizing accuracy loss. For example, 4.8 means an average of 4.8 bits per weight (mix of FP4 and FP8 layers). | +| `--effective_bits` | 4.8 (8.0 for ResNet) | Target average bits per weight across the model. Lower values = more compression but potentially lower accuracy. The ResNet default remains feasible when Conv2d candidates use FP8 or INT8. | | `--num_score_steps` | 128 | Number of forward/backward passes used to compute per-layer sensitivity scores via gradient-based analysis. Higher values provide more accurate sensitivity estimates but increase search time. Recommended range: 64-256. | | `--calibration_data_size` | 512 | Number of calibration samples used for both sensitivity scoring and calibration. For auto mode, labels are required for loss computation. | diff --git a/examples/torch_onnx/torch_quant_to_onnx.py b/examples/torch_onnx/torch_quant_to_onnx.py index 6c4a8de0a88..a83bffc5ae3 100644 --- a/examples/torch_onnx/torch_quant_to_onnx.py +++ b/examples/torch_onnx/torch_quant_to_onnx.py @@ -34,8 +34,9 @@ from evaluation import evaluate import modelopt.torch.quantization as mtq -from modelopt.torch.quantization.config import QuantizerAttributeConfig +from modelopt.recipe import load_recipe from modelopt.torch.quantization.nn import TensorQuantizer +from modelopt.torch.quantization.plugins.timm import is_resnet_quantization_supported """ Quantize a timm vision model and export to ONNX for TensorRT deployment. @@ -63,6 +64,15 @@ "nvfp4": mtq.NVFP4_DEFAULT_CFG, "int4_awq": mtq.INT4_AWQ_CFG, } +_RESNET_RECIPE_DIR = "timm/resnet/ptq" +_RESNET_QUANTIZE_MODES = {"fp8", "int8", "mxfp8", "nvfp4"} +_RESNET_AUTO_RECIPE_NAMES = { + "FP8_DEFAULT_CFG": "fp8", + "INT8_DEFAULT_CFG": "int8", + "MXFP8_DEFAULT_CFG": "mxfp8", + "NVFP4_DEFAULT_CFG": "nvfp4", + "NVFP4_AWQ_LITE_CFG": "nvfp4_awq_lite", +} _FP8_CONV_OVERRIDE: list = [ { @@ -90,13 +100,6 @@ }, ] -_FP8_RESIDUAL_OVERRIDE: list = [ - { - "quantizer_name": "*residual_quantizer.input_quantizer", - "cfg": {"num_bits": (4, 3), "axis": None}, - }, -] - # FP8 MHA-aware config entries: quantize LayerNorm output so TRT can fuse the shared # Q/DQ across all downstream Q/K/V/FC consumers. Softmax-output Q/DQ is handled by the # FP8 ONNX exporter's post-processing pass (fixed 1/448 scale, data-independent). @@ -123,7 +126,7 @@ _NEEDS_INT8_CONV_OVERRIDE: set[str] = {"INT4_AWQ_CFG"} -def get_quant_config(quantize_mode): +def get_quant_config(quantize_mode, model=None): """Get quantization config, overriding Conv2d for TRT compatibility. TensorRT only supports FP8 and INT8 for Conv layers. @@ -133,6 +136,9 @@ def get_quant_config(quantize_mode): - For MXFP8, NVFP4: override Conv2d to FP8 - For INT4_AWQ: override Conv2d to INT8 """ + if is_resnet_quantization_supported(model) and quantize_mode in _RESNET_QUANTIZE_MODES: + return load_recipe(f"{_RESNET_RECIPE_DIR}/{quantize_mode}").quantize.model_dump() + config: dict = copy.deepcopy(QUANT_CONFIG_DICT[quantize_mode]) if quantize_mode == "fp8": config["quant_cfg"].extend(_FP8_MHA_OVERRIDE) @@ -142,7 +148,6 @@ def get_quant_config(quantize_mode): f"Overriding Conv2d quantization to FP8 for '{quantize_mode}' mode." ) config["quant_cfg"].extend(_FP8_CONV_OVERRIDE) - config["quant_cfg"].extend(_FP8_RESIDUAL_OVERRIDE) elif quantize_mode == "int4_awq": warnings.warn( "TensorRT only supports FP8/INT8 for Conv layers. " @@ -152,19 +157,28 @@ def get_quant_config(quantize_mode): return config -def filter_func(name, model=None): +def get_auto_quant_config(format_name, model): + recipe_name = _RESNET_AUTO_RECIPE_NAMES.get(format_name) + if is_resnet_quantization_supported(model) and recipe_name is not None: + return load_recipe(f"{_RESNET_RECIPE_DIR}/{recipe_name}").quantize.model_dump() + + config = copy.deepcopy(getattr(mtq, format_name)) + if format_name in _NEEDS_FP8_CONV_OVERRIDE: + config["quant_cfg"].extend(_FP8_CONV_OVERRIDE) + elif format_name in _NEEDS_INT8_CONV_OVERRIDE: + config["quant_cfg"].extend(_INT8_CONV_OVERRIDE) + return config + + +def filter_func(name): """Filter function to exclude certain layers from quantization. - A ResNet's top-level ``conv1`` consumes three-channel image input, for which TensorRT - cannot find an FP8 Q→Conv tactic on Blackwell. ``downsample.reduction`` (Swin/SwinV2) is excluded because it operates on 4D tensors and TRT's DynamicQuantize layer (used for MXFP8/NVFP4) requires 2D/3D input. Other 4D-input layers (e.g. Swin's ``norm1``, ``downsample.norm``, top-level ``norm``) are handled dynamically by ``_disable_high_rank_input_quantizers`` via a forward-pass rank probe — that avoids false positives on ViT, whose same-named ``norm`` sees 3D input. """ - if isinstance(model, timm.models.resnet.ResNet) and name.startswith("conv1."): - return True pattern = re.compile( r".*(time_emb_proj|time_embedding|conv_in|conv_out|conv_shortcut|add_embedding|" r"pos_embed|time_text_embed|context_embedder|norm_out|x_embedder|patch_embed|cpb_mlp|" @@ -208,173 +222,6 @@ def hook(m, inp, out, _n=name): mtq.disable_quantizer(model, lambda n: n.startswith(prefixes)) -def _quantize_module_input(module, inputs): - return (module.input_quantizer(inputs[0]), *inputs[1:]) - - -def _calibrate_new_quantizers(model, quantizers, data_loader): - enabled_quantizers = [ - module - for module in model.modules() - if isinstance(module, TensorQuantizer) and module.is_enabled - ] - for quantizer in enabled_quantizers: - quantizer.disable_quant() - for quantizer in quantizers: - quantizer.enable_calib() - - was_training = model.training - model.eval() - try: - with torch.no_grad(): - for batch in data_loader: - model(batch["image"] if isinstance(batch, dict) else batch) - for quantizer in quantizers: - quantizer.load_calib_amax(strict=False) - finally: - model.train(was_training) - for quantizer in quantizers: - quantizer.disable_calib() - for quantizer in enabled_quantizers: - quantizer.enable_quant() - - _disable_invalid_quantizers(quantizers) - - -def _append_resnet_residual_quantizer(block, num_bits): - device = next(block.parameters()).device - quantizer = TensorQuantizer(QuantizerAttributeConfig(num_bits=num_bits, axis=None)).to(device) - residual_quantizer = torch.nn.Sequential() - residual_quantizer.add_module("input_quantizer", quantizer) - if block.downsample is None: - block.downsample = torch.nn.Sequential() - elif not isinstance(block.downsample, torch.nn.Sequential): - block.downsample = torch.nn.Sequential(block.downsample) - block.downsample.add_module("residual_quantizer", residual_quantizer) - return quantizer - - -def _prepare_resnet_quantizers(model, quantize_mode): - """Install ResNet shortcut quantizers before the standard calibration pass. - - INT8 and FP8 share one block-input Q/DQ between ``conv1`` and an identity shortcut; - projection blocks additionally quantize the downsample output immediately before ``Add``. - MXFP8 and NVFP4 use per-tensor FP8 on every 4D shortcut because their dynamic block - quantizers only support 2D/3D tensors. INT4-AWQ is skipped because it is weight-only. - Auto mode is configured after its per-block format search. - """ - block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) - blocks = [block for block in model.modules() if isinstance(block, block_types)] - if ( - not blocks - or quantize_mode in ("auto", "int4_awq") - or any( - hasattr(block, "input_quantizer") - or (block.downsample is not None and hasattr(block.downsample, "residual_quantizer")) - for block in blocks - ) - ): - return [] - - num_bits = 8 if quantize_mode == "int8" else (4, 3) - device = next(model.parameters()).device - residual_quantizers = [] - for block in blocks: - if quantize_mode in ("int8", "fp8"): - quantizer = TensorQuantizer(QuantizerAttributeConfig(num_bits=num_bits, axis=None)).to( - device - ) - block.add_module("input_quantizer", quantizer) - block.register_forward_pre_hook(_quantize_module_input) - residual_quantizers.append(quantizer) - - if block.downsample is None: - continue - - residual_quantizers.append(_append_resnet_residual_quantizer(block, num_bits)) - - if quantize_mode == "int8": - quantizer = TensorQuantizer(QuantizerAttributeConfig(num_bits=num_bits, axis=None)).to( - device - ) - model.global_pool.add_module("input_quantizer", quantizer) - model.global_pool.register_forward_pre_hook(_quantize_module_input) - residual_quantizers.append(quantizer) - - return residual_quantizers - - -def _disable_invalid_quantizers(quantizers): - for quantizer in quantizers: - if not quantizer.is_enabled: - continue - amax = quantizer.amax - if ( - amax is None - or not torch.is_tensor(amax) - or torch.any(torch.isnan(amax)) - or torch.all(amax <= 0) - ): - quantizer.disable() - - -def _add_auto_resnet_residual_quantizers(model, auto_quantization_formats, data_loader): - """Calibrate shortcuts with each block's AutoQuantize-selected activation format.""" - activation_formats = set(auto_quantization_formats) - {"INT4_AWQ_CFG"} - if not activation_formats: - return - fallback_num_bits = 8 if activation_formats == {"INT8_DEFAULT_CFG"} else (4, 3) - residual_quantizers = [] - block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) - for block in model.modules(): - if not isinstance(block, block_types): - continue - last_conv = block.conv3 if isinstance(block, timm.models.resnet.Bottleneck) else block.conv2 - selected_quantizer = getattr(last_conv, "input_quantizer", None) - num_bits = ( - selected_quantizer.num_bits - if selected_quantizer is not None - and selected_quantizer.is_enabled - and selected_quantizer.num_bits in (8, (4, 3)) - else fallback_num_bits - ) - residual_quantizers.append(_append_resnet_residual_quantizer(block, num_bits)) - - _calibrate_new_quantizers(model, residual_quantizers, data_loader) - - -def _finalize_resnet_quantizers( - model, quantize_mode, auto_quantization_formats, data_loader, residual_quantizers -): - block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) - blocks = [block for block in model.modules() if isinstance(block, block_types)] - if not blocks: - return - - if quantize_mode in ("int8", "fp8"): - for block in blocks: - block.conv1.input_quantizer.disable() - if block.downsample is not None: - downsample_conv = next( - ( - module - for module in block.downsample.modules() - if isinstance(module, torch.nn.Conv2d) - ), - None, - ) - if downsample_conv is not None: - downsample_conv.input_quantizer.disable() - model.fc.input_quantizer.disable() - model.fc.weight_quantizer.disable() - if quantize_mode == "int8": - model.global_pool.input_quantizer.enable() - elif quantize_mode == "auto": - _add_auto_resnet_residual_quantizers(model, auto_quantization_formats, data_loader) - - _disable_invalid_quantizers(residual_quantizers) - - def load_calibration_data(model, data_size, batch_size, device, with_labels=False): """Load and prepare calibration data. @@ -419,16 +266,14 @@ def _disable_dead_quantizers(model): calibrates to ``amax == 0``. Disable such dead quantizers — they have nothing meaningful to quantize and would otherwise break ONNX export. """ - for _, mod in model.named_modules(): - for attr in ("input_quantizer", "output_quantizer", "weight_quantizer"): - q = getattr(mod, attr, None) - if q is None or not q.is_enabled: - continue - amax = q.amax - if amax is None or not torch.is_tensor(amax): - continue - if torch.any(torch.isnan(amax)) or torch.all(amax <= 0): - q.disable() + for quantizer in model.modules(): + if not isinstance(quantizer, TensorQuantizer) or not quantizer.is_enabled: + continue + amax = quantizer.amax + if amax is None or not torch.is_tensor(amax): + continue + if torch.any(torch.isnan(amax)) or torch.all(amax <= 0): + quantizer.disable() def _calibrate_uncalibrated_quantizers(model, data_loader): @@ -438,15 +283,16 @@ def _calibrate_uncalibrated_quantizers(model, data_loader): be calibrated because the MXFP8/NVFP4 quantization pipeline skips standard calibration. This function explicitly calibrates those uncalibrated quantizers. """ - uncalibrated = [] - for _, module in model.named_modules(): - for attr_name in ("input_quantizer", "weight_quantizer"): - if not hasattr(module, attr_name): - continue - quantizer = getattr(module, attr_name) - if quantizer.is_enabled and not quantizer.block_sizes and quantizer.amax is None: - quantizer.enable_calib() - uncalibrated.append(quantizer) + uncalibrated = [ + quantizer + for quantizer in model.modules() + if isinstance(quantizer, TensorQuantizer) + and quantizer.is_enabled + and not quantizer.block_sizes + and quantizer.amax is None + ] + for quantizer in uncalibrated: + quantizer.enable_calib() if not uncalibrated: return @@ -473,9 +319,8 @@ def forward_loop(model): else: quantized_model = mtq.quantize(model, config) - # Disable filtered quantizers BEFORE calibrating override quantizers so we don't - # waste time calibrating quantizers that are about to be turned off. - mtq.disable_quantizer(quantized_model, lambda name: filter_func(name, quantized_model)) + if not is_resnet_quantization_supported(quantized_model): + mtq.disable_quantizer(quantized_model, filter_func) # Calibrate any FP8 override quantizers that weren't calibrated by mtq.quantize(). if data_loader is not None: @@ -514,7 +359,7 @@ def auto_quantize_model( model, data_loader, quantization_formats, - effective_bits=4.8, + effective_bits=None, num_calib_steps=512, num_score_steps=128, ): @@ -532,23 +377,18 @@ def auto_quantize_model( Tuple of (quantized_model, search_state_dict) """ _disable_inplace_relu(model) + if effective_bits is None: + effective_bits = 8.0 if is_resnet_quantization_supported(model) else 4.8 constraints = {"effective_bits": effective_bits} # Convert string format names to config objects, incorporating Conv2d TRT overrides. # TRT DynamicQuantize requires 2D/3D input, but Conv2d operates on 4D tensors. # By including the overrides in the format configs, the auto_quantize search # correctly accounts for Conv2d being FP8/INT8 in the effective_bits budget. - format_configs = [] - for fmt in quantization_formats: - if isinstance(fmt, str): - config = copy.deepcopy(getattr(mtq, fmt)) - if fmt in _NEEDS_FP8_CONV_OVERRIDE: - config["quant_cfg"].extend(_FP8_CONV_OVERRIDE) - elif fmt in _NEEDS_INT8_CONV_OVERRIDE: - config["quant_cfg"].extend(_INT8_CONV_OVERRIDE) - format_configs.append(config) - else: - format_configs.append(fmt) + format_configs = [ + get_auto_quant_config(fmt, model) if isinstance(fmt, str) else fmt + for fmt in quantization_formats + ] print(f"Starting auto-quantization search with {len(format_configs)} formats...") print(f"Effective bits constraint: {effective_bits}") @@ -566,8 +406,8 @@ def auto_quantize_model( verbose=True, ) - # Disable quantization for specified layers - mtq.disable_quantizer(quantized_model, lambda name: filter_func(name, quantized_model)) + if not is_resnet_quantization_supported(quantized_model): + mtq.disable_quantizer(quantized_model, filter_func) _disable_dead_quantizers(quantized_model) @@ -638,6 +478,7 @@ def main(): nargs="+", choices=[ "NVFP4_AWQ_LITE_CFG", + "NVFP4_DEFAULT_CFG", "FP8_DEFAULT_CFG", "MXFP8_DEFAULT_CFG", "INT8_DEFAULT_CFG", @@ -649,8 +490,11 @@ def main(): parser.add_argument( "--effective_bits", type=float, - default=4.8, - help="Target effective bits for auto quantization constraint. Default is 4.8.", + default=None, + help=( + "Target effective bits for auto quantization. Defaults to 8.0 for ResNet " + "and 4.8 for other models." + ), ) parser.add_argument( "--num_score_steps", @@ -701,8 +545,6 @@ def main(): ) print(f"Base Model - Top-1 Accuracy: {top1:.2f}%, Top-5 Accuracy: {top5:.2f}%") - residual_quantizers = [] - # Quantize model based on mode if args.quantize_mode == "auto": # Auto quantization requires labels for loss computation @@ -727,15 +569,7 @@ def main(): # Note: MXFP8 is dynamic and does not need calibration itself, but when # Conv2d layers are overridden to FP8 (for TRT compatibility), those FP8 # quantizers require calibration data. - config = get_quant_config(args.quantize_mode) - block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) - if args.quantize_mode != "int4_awq" and any( - isinstance(module, block_types) for module in model.modules() - ): - conversion_config = copy.deepcopy(config) - conversion_config["algorithm"] = None - model = mtq.quantize(model, conversion_config) - residual_quantizers = _prepare_resnet_quantizers(model, args.quantize_mode) + config = get_quant_config(args.quantize_mode, model) data_loader = load_calibration_data( model, @@ -747,14 +581,6 @@ def main(): quantized_model = quantize_model(model, config, data_loader) - _finalize_resnet_quantizers( - quantized_model, - args.quantize_mode, - args.auto_quantization_formats, - data_loader, - residual_quantizers, - ) - # MXFP8/NVFP4 lower their input quantizers to TRT DynamicQuantize (2D/3D only). # Disable quantizers on 4D-input layers (Swin's norm1 / downsample.norm / top-level norm). # Auto mode also needs this when an MXFP8/NVFP4 candidate format is in the search set. diff --git a/modelopt/torch/opt/dynamic.py b/modelopt/torch/opt/dynamic.py index 7988f9f970a..4acb3bade88 100644 --- a/modelopt/torch/opt/dynamic.py +++ b/modelopt/torch/opt/dynamic.py @@ -1015,6 +1015,13 @@ def get_key(self, nn_cls: type[nn.Module] | str) -> str: assert nn_cls_ is not None return self._key_registry[nn_cls_] + def get_registered_class(self, key: str) -> type[nn.Module]: + """Retrieve the registered nn.Module class for a string key.""" + for nn_cls, registered_key in self._key_registry.items(): + if registered_key == key: + return nn_cls + raise KeyError(f"{key} is not registered for a dynamic module!") + def get_rule_class(self, nn_cls: type[nn.Module] | str) -> type[ModeloptBaseRule]: """Retrieve the rule config class that is registered for a given nn module class.""" dm_cls = self.get(nn_cls) diff --git a/modelopt/torch/quantization/algorithms.py b/modelopt/torch/quantization/algorithms.py index 7beeef6ad7f..9dd27866ae3 100644 --- a/modelopt/torch/quantization/algorithms.py +++ b/modelopt/torch/quantization/algorithms.py @@ -52,7 +52,13 @@ ) from .config import QuantizeConfig, QuantizerAttributeConfig, QuantizerCfgEntry from .conversion import set_quantizer_by_cfg -from .nn import QuantLinearConvBase, QuantModule, SequentialQuantizer, TensorQuantizer +from .nn import ( + QuantLinearConvBase, + QuantModule, + QuantModuleRegistry, + SequentialQuantizer, + TensorQuantizer, +) from .utils import is_quantized_linear @@ -101,8 +107,17 @@ def _get_quantizer_attrs(module: nn.Module) -> tuple[str, ...]: For fused MoE experts, this returns the four plural quantizer attrs (two shared input quantizers + two ``ModuleList`` of per-expert weight quantizers). - For standard Linear-derived QuantModules, returns the canonical trio. + Modules can override the canonical trio with ``_auto_quantize_quantizer_attrs``. """ + attr_names = getattr(module, "_auto_quantize_quantizer_attrs", None) + if attr_names is not None: + missing = [attr_name for attr_name in attr_names if not hasattr(module, attr_name)] + if missing: + raise AttributeError( + f"{type(module).__name__} declares missing AutoQuantize quantizer attributes: " + f"{missing}." + ) + return tuple(attr_names) if _is_hf_quant_fused_experts_module(module): try: from .plugins.huggingface import _get_fused_experts_quantizer_attr_names @@ -112,6 +127,13 @@ def _get_quantizer_attrs(module: nn.Module) -> tuple[str, ...]: return _STD_QUANTIZER_ATTRS +def _get_quant_module_parent_class(module: nn.Module) -> str | None: + try: + return QuantModuleRegistry.get_key_from_dm(module) + except KeyError: + return None + + def _make_fresh_quantizer_for_attr(module: nn.Module, attr_name: str) -> nn.Module: """Return a fresh, default quantizer object suitable to overwrite ``module.``. @@ -161,45 +183,64 @@ def _fixed_module_format_signature(module: nn.Module) -> tuple: ) -def _fixed_module_weight_compression( - module: nn.Module, effective_bits_override: float | None = None +def _module_weight_compression( + quantizer_attrs: dict[str, nn.Module], + effective_bits_override: float | None = None, ) -> float: - weight_quantizers = [] - for attr_name in _get_quantizer_attrs(module): - if "weight_quantizer" not in attr_name: - continue - weight_quantizers.extend(_iter_tensor_quantizers(getattr(module, attr_name))) - - if not weight_quantizers or all(not quantizer.is_enabled for quantizer in weight_quantizers): - return 1.0 - if any(not quantizer.is_enabled for quantizer in weight_quantizers): - raise ValueError( - "The fixed quantize baseline enables only some weight quantizers within one " - "quantizable module. Move that module into an explicit AutoQuantize " - "module_search_spaces entry." - ) - if effective_bits_override is not None: - return effective_bits_override / 16 - - compressions = [] - for quantizer in weight_quantizers: + def tensor_compression(quantizer): + if not quantizer.is_enabled: + return 1.0 effective_bits = getattr(quantizer, "_effective_bits", None) num_bits = quantizer.num_bits if effective_bits is not None: - compressions.append(effective_bits / 16) - elif isinstance(num_bits, tuple): - compressions.append((sum(num_bits) + 1) / 16) - elif isinstance(num_bits, int): - compressions.append(num_bits / 16) - else: - raise ValueError(f"Cannot infer AutoQuantize cost from num_bits={num_bits!r}.") + return effective_bits / 16 + if isinstance(num_bits, tuple): + return (sum(num_bits) + 1) / 16 + if isinstance(num_bits, int): + return num_bits / 16 + raise ValueError(f"Cannot infer AutoQuantize cost from num_bits={num_bits!r}.") + + def weight_quantizer_compression(quantizer): + if isinstance(quantizer, TensorQuantizer): + return tensor_compression(quantizer) + if isinstance(quantizer, SequentialQuantizer): + stage_compressions = [weight_quantizer_compression(stage) for stage in quantizer] + return min(stage_compressions, default=1.0) + if isinstance(quantizer, nn.ModuleList): + parallel_compressions = [weight_quantizer_compression(child) for child in quantizer] + if any( + abs(value - parallel_compressions[0]) > 1e-12 for value in parallel_compressions[1:] + ): + raise ValueError( + "A quantization recipe assigns different weight formats within one " + "quantizable module. Use one weight format for the entire module." + ) + return parallel_compressions[0] if parallel_compressions else 1.0 + raise TypeError(f"Unsupported weight quantizer type {type(quantizer)}.") + + compressions = [ + weight_quantizer_compression(quantizer) + for attr_name, quantizer in quantizer_attrs.items() + if "weight_quantizer" in attr_name + ] + if not compressions or all(value == 1.0 for value in compressions): + return 1.0 if any(abs(value - compressions[0]) > 1e-12 for value in compressions[1:]): raise ValueError( - "The fixed quantize baseline assigns different weight formats within one quantizable " - "module. Move that module into an explicit AutoQuantize module_search_spaces entry." + "A quantization recipe assigns different weight formats within one quantizable " + "module. Use one weight format for the entire module." ) - return compressions[0] + return effective_bits_override / 16 if effective_bits_override is not None else compressions[0] + + +def _fixed_module_weight_compression( + module: nn.Module, effective_bits_override: float | None = None +) -> float: + return _module_weight_compression( + {attr_name: getattr(module, attr_name) for attr_name in _get_quantizer_attrs(module)}, + effective_bits_override, + ) def estimate_quant_compression(quant_cfg: QuantizeConfig) -> float: @@ -408,6 +449,10 @@ def __init__( name: tuple(_get_replay_quantizer_attr(attr) for attr in _get_quantizer_attrs(module)) for module, name in zip(quant_modules or [], self.quant_module_names) } + self.quant_module_parent_classes = { + name: _get_quant_module_parent_class(module) + for module, name in zip(quant_modules or [], self.quant_module_names) + } assert cost_weight >= 0.0, "cost_weight must be non-negative." self.cost_weight = cost_weight self.allow_no_quant = allow_no_quant @@ -558,14 +603,23 @@ def get_cost(self, recipe: QuantRecipe, cost_weight: float | None = None) -> flo """ cost_weight = self.cost_weight if cost_weight is None else cost_weight cost = 0 + quantizer_choices = self._all_quantizer_choices.get(recipe) for quant_module in self.quant_modules: weight_size = ( _AutoQuantizeBaseSearcher._get_total_weight_size([quant_module]) * cost_weight ) + compression = ( + _module_weight_compression( + quantizer_choices[quant_module], + recipe.config.effective_bits, + ) + if quantizer_choices is not None + else recipe.compression + ) parallel_state = getattr(quant_module, "parallel_state", None) if parallel_state is None: - cost += weight_size * recipe.compression + cost += weight_size * compression continue weight_size = DistributedProcessGroup.get_dist_syncd_obj( @@ -583,7 +637,7 @@ def get_cost(self, recipe: QuantRecipe, cost_weight: float | None = None) -> flo [parallel_state.data_parallel_group], lambda a: a[0], ) - cost += weight_size * recipe.compression + cost += weight_size * compression return cost @@ -714,6 +768,8 @@ def load_search_checkpoint(self) -> bool: @staticmethod def _is_auto_quantize_module(module): + if getattr(module, "_auto_quantize_disabled", False): + return False if (is_quantized_linear(module) or isinstance(module, QuantLinearConvBase)) and isinstance( module, QuantModule ): @@ -1074,6 +1130,7 @@ def initialize_candidate_stats(self): self.candidate_stats[name]["costs"] = costs self.candidate_stats[name]["module_names"] = hparam.quant_module_names self.candidate_stats[name]["quantizer_attrs"] = hparam.quant_module_replay_attrs + self.candidate_stats[name]["parent_classes"] = hparam.quant_module_parent_classes self.candidate_stats[name]["cost_weight"] = hparam.cost_weight self.candidate_stats[name]["allow_no_quant"] = hparam.allow_no_quant self.candidate_stats[name]["is_fixed"] = hparam.is_fixed @@ -2012,11 +2069,16 @@ def _cfg_to_dict(v): for pattern in _as_list(search_state.get("disabled_layers")) ) per_module_entries: list[dict] = [] - _per_module_attrs = ( + per_module_attrs = { *_STD_QUANTIZER_ATTRS, *_FUSED_EXPERTS_REPLAY_QUANTIZER_ATTRS, *_NON_GATED_FUSED_EXPERTS_REPLAY_QUANTIZER_ATTRS, - ) + } + for candidate_stat in search_state["candidate_stats"].values(): + quantizer_attrs = candidate_stat.get("quantizer_attrs") + if isinstance(quantizer_attrs, dict): + for attrs in quantizer_attrs.values(): + per_module_attrs.update(attrs) # Track global (non per-module) recipe entries. Last recipe wins for each pattern. global_entries: dict[str, dict] = {} @@ -2027,10 +2089,16 @@ def _cfg_to_dict(v): if recipe == QuantRecipe(quant_cfg=None): continue module_names = candidate_stat["module_names"] + parent_classes = candidate_stat.get("parent_classes") for module_name in module_names: + parent_class = ( + parent_classes.get(module_name) if isinstance(parent_classes, dict) else None + ) for quantizer_attr in _get_replay_quantizer_attrs(candidate_stat, module_name): matched_cfg, matched_enable = _match_quantizer_cfg( - recipe.config.quant_cfg, quantizer_attr + recipe.config.quant_cfg, + quantizer_attr, + parent_class, ) if matched_enable is not None: entry: dict[str, Any] = { @@ -2044,10 +2112,7 @@ def _cfg_to_dict(v): # Collect non-per-module entries (e.g. *[kv]_bmm_quantizer) from winning recipes. for recipe_entry in recipe.config.quant_cfg: pattern = recipe_entry["quantizer_name"] - if pattern == "*" or any( - fnmatch.fnmatch(attr, pattern) or pattern.endswith(attr) - for attr in _per_module_attrs - ): + if pattern == "*" or any(fnmatch.fnmatch(attr, pattern) for attr in per_module_attrs): continue cfg = recipe_entry.get("cfg") enable = recipe_entry.get("enable", True) @@ -2115,22 +2180,29 @@ def _resolve_best_recipe(search_state, constraints, verbose=False): return best_recipe -def _match_quantizer_cfg(quant_cfg, quantizer_attr): +def _match_quantizer_cfg(quant_cfg, quantizer_attr, module_parent_class=None): # Last-match-wins to mirror set_quantizer_by_cfg behavior. - # Patterns may be path-scoped (e.g. "*mlp*weight_quantizer") while quantizer_attr - # is a bare name like "weight_quantizer". We match if the bare name matches directly - # OR if the pattern ends with the bare quantizer_attr (path-scoped match). + # AutoQuantize applies each candidate to an isolated module, so only patterns that match + # its bare quantizer attribute participate in the per-module candidate. matched = None matched_enable = None for entry in quant_cfg: parent_class = entry.get("parent_class") if hasattr(entry, "get") else entry.parent_class if parent_class is not None: - continue + if module_parent_class is None: + continue + if parent_class != module_parent_class: + try: + module_cls = QuantModuleRegistry.get_registered_class(module_parent_class) + parent_cls = QuantModuleRegistry.get_registered_class(parent_class) + except KeyError: + continue + if not issubclass(module_cls, parent_cls): + continue pattern = entry["quantizer_name"] cfg = entry.get("cfg") enable = entry.get("enable", True) - # Direct match: the bare quantizer_attr matches the whole pattern (e.g. "*weight_quantizer") - if fnmatch.fnmatch(quantizer_attr, pattern) or pattern.endswith(quantizer_attr): + if fnmatch.fnmatch(quantizer_attr, pattern): matched = cfg matched_enable = enable diff --git a/modelopt/torch/quantization/conversion.py b/modelopt/torch/quantization/conversion.py index 00187d291c0..2ece1f7d461 100644 --- a/modelopt/torch/quantization/conversion.py +++ b/modelopt/torch/quantization/conversion.py @@ -25,7 +25,7 @@ import torch.nn as nn from modelopt.torch.opt.conversion import ApplyModeError, ModelLikeModule, ModeloptStateManager -from modelopt.torch.opt.dynamic import _DMRegistryCls +from modelopt.torch.opt.dynamic import DynamicModule, _DMRegistryCls from modelopt.torch.opt.mode import ConvertReturnType, MetadataDict from modelopt.torch.utils import get_unwrapped_name @@ -364,9 +364,21 @@ def _match_quantizer( # Get the parent module of this quantizer. When name has no dots (root-level quantizer), # ".".join([]) == "" and get_submodule("") returns the model itself (PyTorch convention). - return parent_class is None or isinstance( - full_model.get_submodule(".".join(name.split(".")[:-1])), parent_class - ) + if parent_class is None: + return True + + parent_module = full_model.get_submodule(".".join(name.split(".")[:-1])) + if isinstance(parent_module, parent_class): + return True + if not isinstance(parent_module, DynamicModule): + return False + + try: + parent_class_key = QuantModuleRegistry.get_key_from_dm(parent_class) + registered_parent_class = QuantModuleRegistry.get_registered_class(parent_class_key) + except KeyError: + return False + return issubclass(parent_module.original_cls, registered_parent_class) def set_quantizer_attributes_full( diff --git a/modelopt/torch/quantization/plugins/__init__.py b/modelopt/torch/quantization/plugins/__init__.py index 22b4bc2e3cc..fe622ef5a3b 100644 --- a/modelopt/torch/quantization/plugins/__init__.py +++ b/modelopt/torch/quantization/plugins/__init__.py @@ -58,6 +58,9 @@ with import_plugin("torch_geometric"): from .pytorch_geometric import * +with import_plugin("timm"): + from .timm import * + with import_plugin("transformer_engine"): from .transformer_engine import * diff --git a/modelopt/torch/quantization/plugins/timm.py b/modelopt/torch/quantization/plugins/timm.py new file mode 100644 index 00000000000..f661833c2db --- /dev/null +++ b/modelopt/torch/quantization/plugins/timm.py @@ -0,0 +1,287 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Quantization support for timm modules.""" + +import torch.nn as nn +from timm.models.resnet import BasicBlock, Bottleneck, ResNet + +from ..algorithms import AutoQuantizeGradientSearcher, AutoQuantizeKLDivSearcher +from ..nn import QuantModule, QuantModuleRegistry, TensorQuantizer +from ..nn.modules.quant_conv import _QuantConv2d +from .custom import CUSTOM_MODEL_PLUGINS, CUSTOM_POST_CONVERSION_PLUGINS + + +# Forward overrides give marker subclasses distinct registry entries without changing computation. +class _ResNetInputConv2d(nn.Conv2d): + def forward(self, input): + return super().forward(input) + + +class _ResNetOutputConv2d(nn.Conv2d): + def forward(self, input): + return super().forward(input) + + +class _ResNetProjectionOutputConv2d(_ResNetOutputConv2d): + def forward(self, input): + return super().forward(input) + + +class _ResNetFinalOutputConv2d(_ResNetOutputConv2d): + def forward(self, input): + return super().forward(input) + + +class _ResNetFinalProjectionOutputConv2d(_ResNetProjectionOutputConv2d): + def forward(self, input): + return super().forward(input) + + +class _ResNetStemConv2d(nn.Conv2d): + def forward(self, input): + return super().forward(input) + + +class _ResNetBasicBlock(BasicBlock): + def forward(self, input): + return super().forward(input) + + +class _ResNetBottleneck(Bottleneck): + def forward(self, input): + return super().forward(input) + + +def is_resnet_quantization_supported(model): + """Return whether the model uses a supported timm ResNet block implementation.""" + if not isinstance(model, ResNet): + return False + blocks = [module for module in model.modules() if isinstance(module, (BasicBlock, Bottleneck))] + supported_types = (BasicBlock, Bottleneck, _ResNetBasicBlock, _ResNetBottleneck) + return bool(blocks) and all( + type(block) in supported_types or getattr(block, "original_cls", None) in supported_types + for block in blocks + ) + + +@QuantModuleRegistry.register( + { + _ResNetBasicBlock: "timm.ResNetBasicBlock", + _ResNetBottleneck: "timm.ResNetBottleneck", + } +) +class _QuantResNetBlock(QuantModule): + def _setup(self): + pass + + def forward(self, input): + output_conv = self.conv3 if isinstance(self, Bottleneck) else self.conv2 + return super().forward(output_conv.block_input_activation_quantizer(input)) + + +def _register_disabled_quantizer(module, name): + quantizer = TensorQuantizer() + quantizer.disable() + module._register_temp_attribute(name, quantizer) + + +@QuantModuleRegistry.register({_ResNetOutputConv2d: "timm.ResNetOutputConv2d"}) +class _QuantResNetOutputConv2d(_QuantConv2d): + _auto_quantize_quantizer_attrs = ( + "input_quantizer", + "weight_quantizer", + "output_quantizer", + "block_input_activation_quantizer", + ) + + def _setup(self): + super()._setup() + _register_disabled_quantizer(self, "block_input_activation_quantizer") + + +@QuantModuleRegistry.register({_ResNetProjectionOutputConv2d: "timm.ResNetProjectionOutputConv2d"}) +class _QuantResNetProjectionOutputConv2d(_QuantResNetOutputConv2d): + _auto_quantize_quantizer_attrs = ( + *_QuantResNetOutputConv2d._auto_quantize_quantizer_attrs, + "residual_quantizer", + ) + + def _setup(self): + super()._setup() + _register_disabled_quantizer(self, "residual_quantizer") + + +@QuantModuleRegistry.register({_ResNetFinalOutputConv2d: "timm.ResNetFinalOutputConv2d"}) +class _QuantResNetFinalOutputConv2d(_QuantResNetOutputConv2d): + _auto_quantize_quantizer_attrs = ( + *_QuantResNetOutputConv2d._auto_quantize_quantizer_attrs, + "model_output_activation_quantizer", + ) + + def _setup(self): + super()._setup() + _register_disabled_quantizer(self, "model_output_activation_quantizer") + + +@QuantModuleRegistry.register( + {_ResNetFinalProjectionOutputConv2d: "timm.ResNetFinalProjectionOutputConv2d"} +) +class _QuantResNetFinalProjectionOutputConv2d(_QuantResNetProjectionOutputConv2d): + _auto_quantize_quantizer_attrs = ( + *_QuantResNetProjectionOutputConv2d._auto_quantize_quantizer_attrs, + "model_output_activation_quantizer", + ) + + def _setup(self): + super()._setup() + _register_disabled_quantizer(self, "model_output_activation_quantizer") + + +QuantModuleRegistry.register({_ResNetInputConv2d: "timm.ResNetInputConv2d"})(_QuantConv2d) + + +@QuantModuleRegistry.register({_ResNetStemConv2d: "timm.ResNetStemConv2d"}) +class _QuantResNetStemConv2d(_QuantConv2d): + _auto_quantize_disabled = True + + +def _mark_resnet_convs(model): + for resnet in (module for module in model.modules() if isinstance(module, ResNet)): + if not is_resnet_quantization_supported(resnet): + continue + stem_conv = next( + (module for module in resnet.conv1.modules() if type(module) is nn.Conv2d), None + ) + if stem_conv is not None: + stem_conv.__class__ = _ResNetStemConv2d + blocks = [module for module in resnet.modules() if type(module) in (BasicBlock, Bottleneck)] + for index, block in enumerate(blocks): + is_bottleneck = isinstance(block, Bottleneck) + if type(block.conv1) is nn.Conv2d: + block.conv1.__class__ = _ResNetInputConv2d + if block.downsample is not None: + downsample_ops = list(block.downsample.children()) or [block.downsample] + for module in downsample_ops: + if type(module) is nn.Identity: + continue + if type(module) is nn.Conv2d: + module.__class__ = _ResNetInputConv2d + break + output_conv = block.conv3 if is_bottleneck else block.conv2 + if type(output_conv) is nn.Conv2d: + is_projection = block.downsample is not None + is_final = index == len(blocks) - 1 + if is_final: + output_conv.__class__ = ( + _ResNetFinalProjectionOutputConv2d + if is_projection + else _ResNetFinalOutputConv2d + ) + else: + output_conv.__class__ = ( + _ResNetProjectionOutputConv2d if is_projection else _ResNetOutputConv2d + ) + block.__class__ = _ResNetBottleneck if is_bottleneck else _ResNetBasicBlock + + +CUSTOM_MODEL_PLUGINS.add(_mark_resnet_convs) + + +def _register_resnet_quantizer_hooks(model): + for resnet in (module for module in model.modules() if isinstance(module, ResNet)): + blocks = [ + module for module in resnet.modules() if isinstance(module, (BasicBlock, Bottleneck)) + ] + for block in blocks: + if block.downsample is None: + continue + output_conv = block.conv3 if isinstance(block, Bottleneck) else block.conv2 + if not hasattr(output_conv, "residual_quantizer"): + continue + handle = block.downsample.register_forward_hook( + lambda _module, _inputs, output, conv=output_conv: conv.residual_quantizer(output) + ) + output_conv._register_temp_attribute( + "_residual_quantizer_hook", + handle, + del_hook=lambda module, name: getattr(module, name).remove(), + ) + + if not blocks: + continue + final_output_conv = ( + blocks[-1].conv3 if isinstance(blocks[-1], Bottleneck) else blocks[-1].conv2 + ) + if not hasattr(final_output_conv, "model_output_activation_quantizer"): + continue + handle = resnet.global_pool.register_forward_pre_hook( + lambda _module, inputs, conv=final_output_conv: ( + conv.model_output_activation_quantizer(inputs[0]), + *inputs[1:], + ) + ) + final_output_conv._register_temp_attribute( + "_model_output_quantizer_hook", + handle, + del_hook=lambda module, name: getattr(module, name).remove(), + ) + + +CUSTOM_POST_CONVERSION_PLUGINS.add(_register_resnet_quantizer_hooks) + + +def _resnet_block_context(model, name): + parts = name.split(".") + for index in range(len(parts) - 1, -1, -1): + block_name = ".".join(parts[:index]) + block = model.get_submodule(block_name) + if not isinstance(block, (BasicBlock, Bottleneck)): + continue + relative_name = ".".join(parts[index:]) + if relative_name not in ("conv1", "conv2", "conv3") and not relative_name.startswith( + "downsample." + ): + return None + output_conv = block.conv3 if isinstance(block, Bottleneck) else block.conv2 + if not hasattr(output_conv, "block_input_activation_quantizer"): + return None + return block_name, block, output_conv + return None + + +def _resnet_block_group(model, name): + context = _resnet_block_context(model, name) + return context[0] if context is not None else None + + +def _resnet_block_score(model, name): + context = _resnet_block_context(model, name) + if context is None: + return None + block_name, _, output_conv = context + if not hasattr(output_conv, "model_output_activation_quantizer"): + return block_name + parts = block_name.split(".") + for index in range(len(parts), -1, -1): + resnet_name = ".".join(parts[:index]) + if isinstance(model.get_submodule(resnet_name), ResNet): + return ".".join(filter(None, (resnet_name, "global_pool"))) + return block_name + + +AutoQuantizeGradientSearcher.quant_grouping_rules.append(_resnet_block_group) +AutoQuantizeGradientSearcher.score_module_rules.append(_resnet_block_score) # type: ignore[arg-type] +AutoQuantizeKLDivSearcher.score_module_rules.append(_resnet_block_score) diff --git a/modelopt_recipes/README.md b/modelopt_recipes/README.md index b366e4cc670..9affd30d3d4 100644 --- a/modelopt_recipes/README.md +++ b/modelopt_recipes/README.md @@ -43,6 +43,7 @@ huggingface/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast`. |-----------|-----------------| | `general/` | **Model-agnostic** recipes — a good starting point for any model. PTQ combos, speculative-decoding training, and distillation. | | `huggingface//` | **Model-specific** recipes keyed by a HF `model_type`, optionally nested by released checkpoint. Use these first if your model has an entry. | +| `timm//` | **timm architecture-specific** recipes, including deployment-aware vision PTQ choices. | | `models//` | **Instance-specific** recipes that mirror a particular published checkpoint's quantization config. | | `configs/` | Shared building blocks (`numerics/`, `ptq/units/`, `ptq/presets/`) that recipes compose from via `$import`. Not run directly. | @@ -75,6 +76,12 @@ exclusions are still inherited from `configs/`. Browse folder has a `README.md` describing the exact delta. See [`ptq.md`](ptq.md) for how the model-specific recipes compare to the general ones and why they deviate. +## `timm/` — architecture-specific recipes + +Recipes under `timm//` capture quantization choices required by +vision architectures and their deployment backends. See +[`timm/resnet/ptq/`](timm/resnet/ptq/) for ResNet recipes. + ## `models/` — checkpoint-specific recipes These mirror a single **published checkpoint's** quantization config exactly — @@ -90,6 +97,7 @@ a per-component mixed-precision scheme tuned to match a specific release. Browse - **Tuned for a HF architecture** → `huggingface///`, with a `README.md` documenting the delta from the generic preset. Verify the exact `model_type` against the checkpoint's `config.json` before placing it. +- **Tuned for a timm architecture** → `timm///`. - **Mirrors a specific released checkpoint** → `models//`. - Share reused bodies via a `# modelopt-schema:`-tagged snippet and `$import` it; keep recipe wrappers thin. diff --git a/modelopt_recipes/timm/resnet/ptq/README.md b/modelopt_recipes/timm/resnet/ptq/README.md new file mode 100644 index 00000000000..500eb88113c --- /dev/null +++ b/modelopt_recipes/timm/resnet/ptq/README.md @@ -0,0 +1,24 @@ +# timm ResNet PTQ recipes + +These recipes support timm ResNet models built from the standard `BasicBlock` +or `Bottleneck` and keep their quantizer placement and numeric choices out of +the torch ONNX example: + +- The three-channel stem convolution remains unquantized. +- Every block input is quantized once and shared by the main and identity + shortcut paths. +- Projection shortcuts are quantized immediately before the residual add. +- INT8 quantizes the final block output before global pooling. +- MXFP8 and NVFP4 recipes use FP8 for convolution and residual inputs, matching + TensorRT convolution support. + +| Recipe | Numerics | +|--------|----------| +| `fp8.yaml` | FP8 convolution and residual inputs; classifier unquantized. | +| `int8.yaml` | INT8 convolution and residual inputs; classifier unquantized. | +| `mxfp8.yaml` | MXFP8 with FP8 convolution and residual inputs. | +| `nvfp4.yaml` | NVFP4 with FP8 convolution and residual inputs. | +| `nvfp4_awq_lite.yaml` | AWQ-lite NVFP4 AutoQuantize candidate with FP8 convolution and residual inputs. | + +`static_fp8.quant_cfg.yaml` and `static_int8.quant_cfg.yaml` are shared recipe +snippets, not standalone recipes. diff --git a/modelopt_recipes/timm/resnet/ptq/fp8.yaml b/modelopt_recipes/timm/resnet/ptq/fp8.yaml new file mode 100644 index 00000000000..38609c27bff --- /dev/null +++ b/modelopt_recipes/timm/resnet/ptq/fp8.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + fp8: configs/ptq/units/w8a8_fp8_fp8 + resnet: timm/resnet/ptq/static_fp8.quant_cfg + +metadata: + recipe_type: ptq + description: FP8 ResNet PTQ with quantized residual inputs. + +quantize: + algorithm: max + quant_cfg: + - $import: base_disable_all + - $import: fp8 + - $import: default_disabled_quantizers + - $import: resnet + - quantizer_name: 'fc.*' + enable: false diff --git a/modelopt_recipes/timm/resnet/ptq/int8.yaml b/modelopt_recipes/timm/resnet/ptq/int8.yaml new file mode 100644 index 00000000000..cb313b58292 --- /dev/null +++ b/modelopt_recipes/timm/resnet/ptq/int8.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + int8: configs/numerics/int8 + int8_per_channel: configs/numerics/int8_per_channel + resnet: timm/resnet/ptq/static_int8.quant_cfg + +metadata: + recipe_type: ptq + description: INT8 ResNet PTQ with quantized residual inputs. + +quantize: + algorithm: max + quant_cfg: + - $import: base_disable_all + - quantizer_name: '*weight_quantizer' + cfg: + $import: int8_per_channel + - quantizer_name: '*input_quantizer' + cfg: + $import: int8 + - $import: default_disabled_quantizers + - $import: resnet + - quantizer_name: 'fc.*' + enable: false diff --git a/modelopt_recipes/timm/resnet/ptq/mxfp8.yaml b/modelopt_recipes/timm/resnet/ptq/mxfp8.yaml new file mode 100644 index 00000000000..875d49b5ae4 --- /dev/null +++ b/modelopt_recipes/timm/resnet/ptq/mxfp8.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + mxfp8: configs/numerics/mxfp8 + resnet: timm/resnet/ptq/static_fp8.quant_cfg + +metadata: + recipe_type: ptq + description: MXFP8 ResNet PTQ with FP8 convolution and residual inputs. + +quantize: + algorithm: max + quant_cfg: + - $import: base_disable_all + - quantizer_name: '*weight_quantizer' + cfg: + $import: mxfp8 + - quantizer_name: '*input_quantizer' + cfg: + $import: mxfp8 + - $import: default_disabled_quantizers + - $import: resnet diff --git a/modelopt_recipes/timm/resnet/ptq/nvfp4.yaml b/modelopt_recipes/timm/resnet/ptq/nvfp4.yaml new file mode 100644 index 00000000000..9f01ca857ef --- /dev/null +++ b/modelopt_recipes/timm/resnet/ptq/nvfp4.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + nvfp4: configs/ptq/units/w4a4_nvfp4_nvfp4 + resnet: timm/resnet/ptq/static_fp8.quant_cfg + +metadata: + recipe_type: ptq + description: NVFP4 ResNet PTQ with FP8 convolution and residual inputs. + +quantize: + algorithm: max + quant_cfg: + - $import: base_disable_all + - $import: nvfp4 + - $import: default_disabled_quantizers + - $import: resnet diff --git a/modelopt_recipes/timm/resnet/ptq/nvfp4_awq_lite.yaml b/modelopt_recipes/timm/resnet/ptq/nvfp4_awq_lite.yaml new file mode 100644 index 00000000000..33a9bbd8011 --- /dev/null +++ b/modelopt_recipes/timm/resnet/ptq/nvfp4_awq_lite.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + nvfp4: configs/ptq/units/w4a4_nvfp4_nvfp4 + resnet: timm/resnet/ptq/static_fp8.quant_cfg + +metadata: + recipe_type: ptq + description: NVFP4 AWQ-lite ResNet PTQ with FP8 convolution and residual inputs. + +quantize: + algorithm: awq_lite + quant_cfg: + - $import: base_disable_all + - $import: nvfp4 + - $import: default_disabled_quantizers + - $import: resnet diff --git a/modelopt_recipes/timm/resnet/ptq/static_fp8.quant_cfg.yaml b/modelopt_recipes/timm/resnet/ptq/static_fp8.quant_cfg.yaml new file mode 100644 index 00000000000..0902d15406a --- /dev/null +++ b/modelopt_recipes/timm/resnet/ptq/static_fp8.quant_cfg.yaml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# modelopt-schema: modelopt.torch.quantization.config.QuantizerCfgListConfig +imports: + fp8: configs/numerics/fp8 +--- + - parent_class: nn.Conv2d + quantizer_name: '*weight_quantizer' + cfg: + $import: fp8 + - parent_class: nn.Conv2d + quantizer_name: '*input_quantizer' + cfg: + $import: fp8 + - parent_class: timm.ResNetInputConv2d + quantizer_name: '*input_quantizer' + enable: false + - quantizer_name: '*block_input_activation_quantizer' + cfg: + $import: fp8 + - quantizer_name: '*residual_quantizer' + cfg: + $import: fp8 + - parent_class: timm.ResNetStemConv2d + quantizer_name: '*' + enable: false + - parent_class: nn.MaxPool2d + quantizer_name: '*input_quantizer' + enable: false + - parent_class: nn.AvgPool2d + quantizer_name: '*input_quantizer' + enable: false + - parent_class: nn.AdaptiveAvgPool2d + quantizer_name: '*input_quantizer' + enable: false diff --git a/modelopt_recipes/timm/resnet/ptq/static_int8.quant_cfg.yaml b/modelopt_recipes/timm/resnet/ptq/static_int8.quant_cfg.yaml new file mode 100644 index 00000000000..6150d96cef2 --- /dev/null +++ b/modelopt_recipes/timm/resnet/ptq/static_int8.quant_cfg.yaml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# modelopt-schema: modelopt.torch.quantization.config.QuantizerCfgListConfig +imports: + int8: configs/numerics/int8 + int8_per_channel: configs/numerics/int8_per_channel +--- + - parent_class: nn.Conv2d + quantizer_name: '*weight_quantizer' + cfg: + $import: int8_per_channel + - parent_class: nn.Conv2d + quantizer_name: '*input_quantizer' + cfg: + $import: int8 + - parent_class: timm.ResNetInputConv2d + quantizer_name: '*input_quantizer' + enable: false + - quantizer_name: '*block_input_activation_quantizer' + cfg: + $import: int8 + - quantizer_name: '*residual_quantizer' + cfg: + $import: int8 + - quantizer_name: '*model_output_activation_quantizer' + cfg: + $import: int8 + - parent_class: timm.ResNetStemConv2d + quantizer_name: '*' + enable: false + - parent_class: nn.MaxPool2d + quantizer_name: '*input_quantizer' + enable: false + - parent_class: nn.AvgPool2d + quantizer_name: '*input_quantizer' + enable: false + - parent_class: nn.AdaptiveAvgPool2d + quantizer_name: '*input_quantizer' + enable: false diff --git a/tests/unit/torch/nas/test_registry.py b/tests/unit/torch/nas/test_registry.py index a425abcf29c..b072bb30a00 100644 --- a/tests/unit/torch/nas/test_registry.py +++ b/tests/unit/torch/nas/test_registry.py @@ -67,6 +67,9 @@ def _test_new_cls(cls, name): assert DMRegistry2.get_key_from_dm(DMRegistry2[nn.Linear]) == "nn.Linear" assert DMRegistry2.get_key_from_dm(DMRegistry2[Linear]) == "nn.Linear" assert DMRegistry2.get_key_from_dm(DMRegistry2[Linear2]) == "nn.Linear" + assert DMRegistry2.get_registered_class("nn.Linear") is nn.Linear + with pytest.raises(KeyError): + DMRegistry2.get_registered_class("missing") # unregister nn.Linear (only works with truly registered class) with pytest.raises(KeyError): diff --git a/tests/unit/torch/quantization/plugins/test_timm.py b/tests/unit/torch/quantization/plugins/test_timm.py new file mode 100644 index 00000000000..9a03fcc2bc1 --- /dev/null +++ b/tests/unit/torch/quantization/plugins/test_timm.py @@ -0,0 +1,469 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +from copy import deepcopy + +import pytest +import torch + +timm = pytest.importorskip("timm") + +import modelopt.torch.opt as mto +import modelopt.torch.quantization as mtq +from modelopt.recipe import load_recipe +from modelopt.torch.quantization.algorithms import ( + AutoQuantizeGradientSearcher, + QuantRecipe, + QuantRecipeHparam, +) +from modelopt.torch.quantization.plugins.timm import is_resnet_quantization_supported + + +def _get_resnet(block=timm.models.resnet.BasicBlock): + return timm.models.resnet.ResNet( + block=block, + layers=[2, 1, 1, 1], + num_classes=8, + stem_width=8, + channels=(8, 16, 32, 64), + ).eval() + + +def _get_output_conv(block): + return block.conv3 if isinstance(block, timm.models.resnet.Bottleneck) else block.conv2 + + +class _UnsupportedBottleneck(timm.models.resnet.Bottleneck): + def forward(self, input): + return super().forward(input) + + +def test_resnet_recipe_support_is_limited_to_standard_blocks(): + model = timm.models.resnet.ResNet( + block=_UnsupportedBottleneck, + layers=[1, 1, 1, 1], + num_classes=8, + stem_width=8, + channels=(8, 16, 32, 64), + ).eval() + + assert not is_resnet_quantization_supported(model) + model = mtq.quantize(model, {**deepcopy(mtq.INT8_DEFAULT_CFG), "algorithm": None}) + assert not hasattr(_get_output_conv(model.layer1[0]), "block_input_activation_quantizer") + + +@pytest.mark.parametrize( + ("recipe_name", "conv_num_bits", "conv_weight_axis", "fc_num_bits", "fc_block_size"), + [ + ("fp8", (4, 3), None, (4, 3), None), + ("int8", 8, 0, 8, None), + ("mxfp8", (4, 3), None, (4, 3), 32), + ("nvfp4", (4, 3), None, (2, 1), 16), + ("nvfp4_awq_lite", (4, 3), None, (2, 1), 16), + ], +) +@pytest.mark.parametrize( + "block_type", [timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck] +) +def test_resnet_recipe_quantizer_choices( + recipe_name, conv_num_bits, conv_weight_axis, fc_num_bits, fc_block_size, block_type +): + recipe = load_recipe(f"timm/resnet/ptq/{recipe_name}") + config = recipe.quantize.model_dump() + config["algorithm"] = None + model = mtq.quantize(_get_resnet(block_type), config) + + assert is_resnet_quantization_supported(model) + assert not model.conv1.input_quantizer.is_enabled + assert not model.conv1.weight_quantizer.is_enabled + assert not model.maxpool.input_quantizer.is_enabled + + for block in (model.layer1[1], model.layer2[0]): + output_conv = _get_output_conv(block) + assert output_conv.block_input_activation_quantizer.is_enabled + assert output_conv.block_input_activation_quantizer.num_bits == conv_num_bits + assert output_conv.block_input_activation_quantizer.axis is None + assert output_conv.block_input_activation_quantizer.block_sizes is None + assert output_conv.input_quantizer.num_bits == conv_num_bits + assert output_conv.weight_quantizer.num_bits == conv_num_bits + assert output_conv.weight_quantizer.axis == conv_weight_axis + assert not output_conv.output_quantizer.is_enabled + assert not block.conv1.input_quantizer.is_enabled + + projection_block = model.layer2[0] + assert not projection_block.downsample[0].input_quantizer.is_enabled + assert _get_output_conv(projection_block).residual_quantizer.is_enabled + assert _get_output_conv(projection_block).residual_quantizer.num_bits == conv_num_bits + assert not hasattr(_get_output_conv(model.layer1[1]), "residual_quantizer") + + pool_input_quantizers = [ + module + for name, module in model.named_modules() + if name.startswith("global_pool.") and name.endswith("input_quantizer") + ] + assert not any(quantizer.is_enabled for quantizer in pool_input_quantizers) + + final_output_conv = _get_output_conv(model.layer4[-1]) + assert final_output_conv.model_output_activation_quantizer.is_enabled == (recipe_name == "int8") + if recipe_name == "int8": + assert final_output_conv.model_output_activation_quantizer.num_bits == 8 + + assert model.fc.weight_quantizer.num_bits == fc_num_bits + if fc_block_size is None: + assert model.fc.weight_quantizer.block_sizes is None + else: + assert model.fc.weight_quantizer.block_sizes[-1] == fc_block_size + if recipe_name in ("fp8", "int8"): + assert not model.fc.weight_quantizer.is_enabled + assert not model.fc.input_quantizer.is_enabled + else: + assert model.fc.weight_quantizer.is_enabled + assert model.fc.input_quantizer.is_enabled + + +@pytest.mark.parametrize( + "config_name", + ["FP8_DEFAULT_CFG", "INT8_DEFAULT_CFG", "MXFP8_DEFAULT_CFG", "NVFP4_DEFAULT_CFG"], +) +def test_stock_configs_do_not_enable_resnet_residual_quantizers(config_name): + config = deepcopy(getattr(mtq, config_name)) + config["algorithm"] = None + model = mtq.quantize(_get_resnet(), config) + + for block in (model.layer1[1], model.layer2[0]): + output_conv = _get_output_conv(block) + assert not output_conv.block_input_activation_quantizer.is_enabled + if block.downsample is not None: + assert not output_conv.residual_quantizer.is_enabled + assert block.conv1.input_quantizer.is_enabled + assert model.layer2[0].downsample[0].input_quantizer.is_enabled + assert not _get_output_conv(model.layer4[-1]).model_output_activation_quantizer.is_enabled + + +def test_parent_conv_rule_matches_resnet_conv_subclasses(): + model = mtq.quantize( + _get_resnet(), + { + "quant_cfg": [ + {"quantizer_name": "*", "enable": False}, + { + "parent_class": "nn.Conv2d", + "quantizer_name": "*weight_quantizer", + "cfg": {"num_bits": 7}, + }, + ], + "algorithm": None, + }, + ) + + for conv in ( + model.conv1, + model.layer1[0].conv1, + _get_output_conv(model.layer1[0]), + model.layer2[0].downsample[0], + _get_output_conv(model.layer2[0]), + ): + assert conv.weight_quantizer.is_enabled + assert conv.weight_quantizer.num_bits == 7 + + +def test_resnet_recipe_disables_only_first_deep_stem_convolution(): + model = timm.models.resnet.ResNet( + block=timm.models.resnet.Bottleneck, + layers=[1, 1, 1, 1], + num_classes=8, + stem_width=8, + stem_type="deep", + channels=(8, 16, 32, 64), + ).eval() + recipe = load_recipe("timm/resnet/ptq/int8") + config = recipe.quantize.model_dump() + config["algorithm"] = None + model = mtq.quantize(model, config) + + stem_convs = [module for module in model.conv1.modules() if hasattr(module, "weight_quantizer")] + assert len(stem_convs) == 3 + assert not stem_convs[0].input_quantizer.is_enabled + assert not stem_convs[0].weight_quantizer.is_enabled + assert all(conv.input_quantizer.is_enabled for conv in stem_convs[1:]) + assert all(conv.weight_quantizer.is_enabled for conv in stem_convs[1:]) + + +def test_resnet_recipe_quantizes_replacement_stem_pool_convolution(): + model = timm.models.resnet.ResNet( + block=timm.models.resnet.Bottleneck, + layers=[1, 1, 1, 1], + num_classes=8, + stem_width=8, + stem_type="deep", + replace_stem_pool=True, + channels=(8, 16, 32, 64), + ).eval() + recipe = load_recipe("timm/resnet/ptq/int8") + config = recipe.quantize.model_dump() + config["algorithm"] = None + model = mtq.quantize(model, config) + + assert model.maxpool[0].input_quantizer.is_enabled + assert model.maxpool[0].weight_quantizer.is_enabled + + +def test_resnet_recipe_handles_avg_down_and_antialias_pools(): + model = timm.models.resnet.ResNet( + block=timm.models.resnet.Bottleneck, + layers=[1, 1, 1, 1], + num_classes=8, + stem_width=8, + avg_down=True, + aa_layer=torch.nn.AvgPool2d, + channels=(8, 16, 32, 64), + ).eval() + recipe = load_recipe("timm/resnet/ptq/mxfp8") + config = recipe.quantize.model_dump() + config["algorithm"] = None + model = mtq.quantize(model, config) + + block = model.layer2[0] + assert not block.aa.input_quantizer.is_enabled + assert not block.downsample[0].input_quantizer.is_enabled + assert block.downsample[1].input_quantizer.is_enabled + assert block.downsample[1].input_quantizer.num_bits == (4, 3) + + +@pytest.mark.parametrize( + "block_type", [timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck] +) +def test_resnet_recipe_calibrates_residual_quantizers_in_one_pass(block_type): + model = _get_resnet(block_type) + recipe = load_recipe("timm/resnet/ptq/int8") + calibration_calls = 0 + + def forward_loop(quantized_model): + nonlocal calibration_calls + calibration_calls += 1 + quantized_model(torch.randn(2, 3, 32, 32)) + + model = mtq.quantize(model, recipe.quantize.model_dump(), forward_loop=forward_loop) + + assert calibration_calls == 1 + for block in (model.layer1[1], model.layer2[0]): + quantizers = [_get_output_conv(block).block_input_activation_quantizer] + if block.downsample is not None: + quantizers.append(_get_output_conv(block).residual_quantizer) + for quantizer in quantizers: + assert quantizer.amax is not None + assert torch.isfinite(quantizer.amax).all() + assert torch.all(quantizer.amax > 0) + + model_output_quantizer = _get_output_conv(model.layer4[-1]).model_output_activation_quantizer + assert model_output_quantizer.amax is not None + assert torch.isfinite(model_output_quantizer.amax).all() + assert torch.all(model_output_quantizer.amax > 0) + + +def test_auto_quantize_uses_resnet_conv_format_for_cost_and_replay(): + recipe = load_recipe("timm/resnet/ptq/nvfp4") + config = recipe.quantize.model_dump() + config["algorithm"] = None + model = mtq.quantize(_get_resnet(), config) + block = model.layer2[0] + output_conv = _get_output_conv(block) + quant_recipe = QuantRecipe(recipe.quantize.model_dump(), name="resnet_nvfp4") + hparam = QuantRecipeHparam( + [quant_recipe], + quant_modules=[output_conv], + score_modules=[block], + quant_module_names=["layer2.0.conv2"], + ) + + assert hparam.quant_module_replay_attrs["layer2.0.conv2"] == ( + "input_quantizer", + "weight_quantizer", + "output_quantizer", + "block_input_activation_quantizer", + "residual_quantizer", + ) + assert hparam.quant_module_parent_classes["layer2.0.conv2"] == ( + "timm.ResNetProjectionOutputConv2d" + ) + quantizer_choice = hparam._all_quantizer_choices[quant_recipe][output_conv] + for quantizer_name in ( + "input_quantizer", + "weight_quantizer", + "block_input_activation_quantizer", + "residual_quantizer", + ): + assert quantizer_choice[quantizer_name].num_bits == (4, 3) + assert hparam.get_cost(quant_recipe) == pytest.approx(output_conv.weight.numel() * 0.5) + + hparam_name = "layer2.0.conv2.quant_recipe" + search_state = { + "best": {"recipe": {hparam_name: quant_recipe}}, + "candidate_stats": { + hparam_name: { + "module_names": hparam.quant_module_names, + "quantizer_attrs": hparam.quant_module_replay_attrs, + "parent_classes": hparam.quant_module_parent_classes, + } + }, + } + with pytest.warns(UserWarning, match="algorithm='max'"): + replay_config = mtq.get_auto_quantize_config(search_state) + entries = {entry["quantizer_name"]: entry for entry in replay_config["quant_cfg"]} + for quantizer_name in hparam.quant_module_replay_attrs["layer2.0.conv2"]: + entry = entries[f"layer2.0.conv2.{quantizer_name}"] + if quantizer_name == "output_quantizer": + assert not entry["enable"] + else: + assert entry["enable"] + assert entry["cfg"]["num_bits"] == (4, 3) + + assert AutoQuantizeGradientSearcher._is_auto_quantize_module(output_conv) + assert not AutoQuantizeGradientSearcher._is_auto_quantize_module(model.conv1) + + +def test_auto_quantize_replay_keeps_resnet_block_input_convs_disabled(): + recipe = load_recipe("timm/resnet/ptq/int8") + config = recipe.quantize.model_dump() + config["algorithm"] = None + model = mtq.quantize(_get_resnet(), config) + conv = model.layer2[0].conv1 + quant_recipe = QuantRecipe(recipe.quantize.model_dump(), name="resnet_int8") + hparam = QuantRecipeHparam( + [quant_recipe], + quant_modules=[conv], + quant_module_names=["layer2.0.conv1"], + ) + hparam_name = "layer2.0.conv1.quant_recipe" + search_state = { + "best": {"recipe": {hparam_name: quant_recipe}}, + "candidate_stats": { + hparam_name: { + "module_names": hparam.quant_module_names, + "quantizer_attrs": hparam.quant_module_replay_attrs, + "parent_classes": hparam.quant_module_parent_classes, + } + }, + } + + with pytest.warns(UserWarning, match="algorithm='max'"): + replay_config = mtq.get_auto_quantize_config(search_state) + entries = {entry["quantizer_name"]: entry for entry in replay_config["quant_cfg"]} + + assert not entries["layer2.0.conv1.input_quantizer"]["enable"] + + +@pytest.mark.parametrize( + "block_type", [timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck] +) +def test_auto_quantize_calibrates_and_scores_resnet_residual_quantizers(block_type): + recipes = [load_recipe(f"timm/resnet/ptq/{name}") for name in ("int8", "fp8")] + data = [{"image": torch.randn(1, 3, 32, 32), "label": torch.tensor([1])}] + + model, search_state = mtq.auto_quantize( + _get_resnet(block_type), + constraints={"effective_bits": 8.0}, + quantization_formats=[recipe.quantize.model_dump() for recipe in recipes], + data_loader=data, + forward_step=lambda model, batch: model(batch["image"]), + loss_func=lambda output, batch: torch.nn.functional.cross_entropy(output, batch["label"]), + num_calib_steps=1, + num_score_steps=1, + ) + + block = model.layer2[0] + output_conv = _get_output_conv(block) + for quantizer in ( + output_conv.block_input_activation_quantizer, + output_conv.residual_quantizer, + ): + assert quantizer.is_enabled + assert quantizer.amax is not None + assert torch.isfinite(quantizer.amax).all() + assert torch.all(quantizer.amax > 0) + + hparam = output_conv.get_hparam("quant_recipe") + assert hparam.score_modules == [block] + block_convs = [block.conv1, block.conv2, block.downsample[0]] + if isinstance(block, timm.models.resnet.Bottleneck): + block_convs.append(block.conv3) + assert all(conv.get_hparam("quant_recipe") is hparam for conv in block_convs) + assert all( + conv.weight_quantizer.num_bits == output_conv.block_input_activation_quantizer.num_bits + for conv in block_convs + ) + output_conv_name = ( + "layer2.0.conv3" if isinstance(block, timm.models.resnet.Bottleneck) else "layer2.0.conv2" + ) + candidate_stat = next( + stat + for stat in search_state["candidate_stats"].values() + if output_conv_name in stat["module_names"] + ) + assert candidate_stat["quantizer_attrs"][output_conv_name][-2:] == ( + "block_input_activation_quantizer", + "residual_quantizer", + ) + assert candidate_stat["parent_classes"][output_conv_name] == ( + "timm.ResNetProjectionOutputConv2d" + ) + + final_block = model.layer4[-1] + final_output_conv = _get_output_conv(final_block) + final_hparam = final_output_conv.get_hparam("quant_recipe") + assert final_hparam.score_modules == [model.global_pool] + assert final_output_conv.model_output_activation_quantizer.is_enabled == ( + final_output_conv.block_input_activation_quantizer.num_bits == 8 + ) + for quantizer_choices in final_hparam._all_quantizer_choices.values(): + quantizer = quantizer_choices[final_output_conv]["model_output_activation_quantizer"] + if quantizer.is_enabled: + assert quantizer.amax is not None + assert torch.isfinite(quantizer.amax).all() + assert torch.all(quantizer.amax > 0) + + +def test_resnet_quantizer_hooks_survive_save_restore(): + inputs = torch.randn(1, 3, 32, 32) + recipe = load_recipe("timm/resnet/ptq/int8") + model = mtq.quantize( + _get_resnet(), + recipe.quantize.model_dump(), + forward_loop=lambda quantized_model: quantized_model(inputs), + ) + expected = model(inputs) + + buffer = io.BytesIO() + mto.save(model, buffer) + buffer.seek(0) + restored = mto.restore(_get_resnet(), buffer).eval() + + block = restored.layer2[0] + output_conv = _get_output_conv(block) + counts = {"block_input": 0, "projection": 0} + + def count_call(name): + def hook(_module, _inputs, _output): + counts[name] += 1 + + return hook + + output_conv.block_input_activation_quantizer.register_forward_hook(count_call("block_input")) + output_conv.residual_quantizer.register_forward_hook(count_call("projection")) + + assert torch.equal(restored(inputs), expected) + assert counts == {"block_input": 1, "projection": 1} + assert len(block.downsample._forward_hooks) == 1 diff --git a/tests/unit/torch/quantization/test_autoquant.py b/tests/unit/torch/quantization/test_autoquant.py index e83f7fa0a70..24e3daa4329 100644 --- a/tests/unit/torch/quantization/test_autoquant.py +++ b/tests/unit/torch/quantization/test_autoquant.py @@ -36,10 +36,17 @@ QuantRecipe, QuantRecipeHparam, _AutoQuantizeBaseSearcher, + _get_quantizer_attrs, _module_search_space_signature, + _module_weight_compression, estimate_quant_compression, ) -from modelopt.torch.quantization.config import _base_disable_all, _default_disabled_quantizer_cfg +from modelopt.torch.quantization.config import ( + QuantizerAttributeConfig, + _base_disable_all, + _default_disabled_quantizer_cfg, +) +from modelopt.torch.quantization.nn import TensorQuantizer from modelopt.torch.utils import safe_load, safe_save from modelopt.torch.utils.distributed import DistributedProcessGroup @@ -204,6 +211,40 @@ def test_quant_recipe_hparam_zero_cost_weight(): assert hparam.get_cost(QuantRecipe(mtq.INT8_DEFAULT_CFG)) == pytest.approx(0.0) +def test_custom_auto_quantize_attrs_are_explicit(): + module = torch.nn.Module() + module.custom_quantizer = TensorQuantizer() + module._auto_quantize_quantizer_attrs = () + assert _get_quantizer_attrs(module) == () + + module._auto_quantize_quantizer_attrs = ("custom_quantizer",) + assert _get_quantizer_attrs(module) == ("custom_quantizer",) + + module._auto_quantize_quantizer_attrs = ("missing_quantizer",) + with pytest.raises(AttributeError, match="missing_quantizer"): + _get_quantizer_attrs(module) + + +def test_candidate_cost_rejects_mixed_weight_formats(): + quantizers = { + "first_weight_quantizer": TensorQuantizer(QuantizerAttributeConfig(num_bits=8)), + "second_weight_quantizer": TensorQuantizer(QuantizerAttributeConfig(num_bits=4)), + } + with pytest.raises(ValueError, match="different weight formats"): + _module_weight_compression(quantizers) + + +def test_candidate_cost_supports_sequential_weight_quantization(): + recipe = QuantRecipe(mtq.W4A8_AWQ_BETA_CFG) + model = mtq.quantize( + torch.nn.Linear(4, 16), + {"quant_cfg": [{"quantizer_name": "*", "enable": False}], "algorithm": None}, + ) + hparam = QuantRecipeHparam([recipe], quant_modules=[model]) + + assert hparam.get_cost(recipe) == pytest.approx(model.weight.numel() * 0.25) + + def test_quant_recipe_hparam_cost_weight_and_effective_bits_compose(): """cost_weight (active_moe) and effective_bits stack multiplicatively in get_cost.""" model_test = mtq.quantize(torch.nn.Linear(4, 16), mtq.NVFP4_DEFAULT_CFG) diff --git a/tests/unit/torch/quantization/test_config_validation.py b/tests/unit/torch/quantization/test_config_validation.py index 4b969d3259c..c8f71c9adba 100644 --- a/tests/unit/torch/quantization/test_config_validation.py +++ b/tests/unit/torch/quantization/test_config_validation.py @@ -463,13 +463,13 @@ def test_star_matches_any_bare_name(self): assert matched is None # enable-only entry has cfg=None assert enable is False - def test_path_scoped_pattern_matches_matching_suffix(self): - """'*mlp*weight_quantizer' matches bare 'weight_quantizer' (suffix match).""" + def test_path_scoped_pattern_does_not_match_bare_name(self): quant_cfg = normalize_quant_cfg_list( [{"quantizer_name": "*mlp*weight_quantizer", "cfg": {"num_bits": 4}}] ) matched, enable = _match_quantizer_cfg(quant_cfg, "weight_quantizer") - assert matched.model_dump(exclude_unset=True) == {"num_bits": 4} + assert matched is None + assert enable is None def test_path_scoped_pattern_does_not_match_different_suffix(self): """'*mlp*weight_quantizer' does NOT match bare 'input_quantizer'.""" @@ -507,6 +507,25 @@ def test_parent_class_scoped_entries_are_ignored_for_bare_autoquant_lookup(self) assert matched.model_dump(exclude_unset=True) == {"num_bits": 8} assert enable is True + def test_parent_class_scoped_entry_matches_persisted_autoquant_parent(self): + quant_cfg = normalize_quant_cfg_list( + [ + {"quantizer_name": "*weight_quantizer", "cfg": {"num_bits": 4}}, + { + "parent_class": "nn.Conv2d", + "quantizer_name": "*weight_quantizer", + "cfg": {"num_bits": 8}, + }, + ] + ) + + matched, enable = _match_quantizer_cfg( + quant_cfg, "weight_quantizer", module_parent_class="nn.Conv2d" + ) + + assert matched.model_dump(exclude_unset=True) == {"num_bits": 8} + assert enable is True + def test_no_match_returns_none(self): """No matching entry returns (None, None).""" quant_cfg = normalize_quant_cfg_list( diff --git a/tests/unit/torch/quantization/test_quantize_cpu.py b/tests/unit/torch/quantization/test_quantize_cpu.py index 3e4925e7b63..433f4af150a 100644 --- a/tests/unit/torch/quantization/test_quantize_cpu.py +++ b/tests/unit/torch/quantization/test_quantize_cpu.py @@ -84,6 +84,35 @@ } +class _LinearSubclass(torch.nn.Linear): + pass + + +def test_parent_class_config_matches_registered_subclass(): + model = mtq.quantize( + _LinearSubclass(4, 4), + { + "quant_cfg": [ + {"quantizer_name": "*", "enable": False}, + { + "parent_class": "nn.Linear", + "quantizer_name": "*weight_quantizer", + "cfg": {"num_bits": 7}, + }, + { + "parent_class": "nn.Conv2d", + "quantizer_name": "*weight_quantizer", + "cfg": {"num_bits": 3}, + }, + ], + "algorithm": None, + }, + ) + + assert model.weight_quantizer.is_enabled + assert model.weight_quantizer.num_bits == 7 + + class NewMaxCalibrator(MaxCalibrator): def compute_amax(self): return 2 * self._calib_amax From 7f3a6d7bed98665bc1ee19f2b36b4471acf6bc49 Mon Sep 17 00:00:00 2001 From: ajrasane <131806219+ajrasane@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:34:26 +0000 Subject: [PATCH 10/10] [OMNIML-5613] Simplify ResNet residual quantization Keep the change focused on shortcut QDQ placement with FP8, INT8, and AutoQuantize recipes. Co-Authored-By: Codex Signed-off-by: ajrasane <131806219+ajrasane@users.noreply.github.com> --- CHANGELOG.rst | 6 +- examples/torch_onnx/README.md | 10 +- examples/torch_onnx/torch_quant_to_onnx.py | 206 +++++--- modelopt/onnx/export/fp8_exporter.py | 7 +- modelopt/onnx/utils.py | 54 +- modelopt/torch/_deploy/utils/torch_onnx.py | 5 +- modelopt/torch/opt/dynamic.py | 7 - modelopt/torch/quantization/algorithms.py | 170 ++----- modelopt/torch/quantization/conversion.py | 20 +- .../torch/quantization/plugins/__init__.py | 3 - modelopt/torch/quantization/plugins/timm.py | 287 ----------- modelopt_recipes/README.md | 8 - .../auto_quantize/fp8_int8_at_8p0bits.yaml | 34 ++ modelopt_recipes/timm/resnet/ptq/README.md | 24 - modelopt_recipes/timm/resnet/ptq/fp8.yaml | 15 +- modelopt_recipes/timm/resnet/ptq/int8.yaml | 10 +- modelopt_recipes/timm/resnet/ptq/mxfp8.yaml | 25 - modelopt_recipes/timm/resnet/ptq/nvfp4.yaml | 20 - .../timm/resnet/ptq/nvfp4_awq_lite.yaml | 20 - .../timm/resnet/ptq/static_fp8.quant_cfg.yaml | 36 -- .../resnet/ptq/static_int8.quant_cfg.yaml | 40 -- .../torch_onnx/test_torch_quant_to_onnx.py | 68 +-- .../quantization/test_fp8_mha_exporter.py | 22 - tests/unit/onnx/test_fold_casts.py | 100 +--- tests/unit/torch/nas/test_registry.py | 3 - .../torch/quantization/plugins/test_timm.py | 469 ------------------ .../unit/torch/quantization/test_autoquant.py | 43 +- .../quantization/test_config_validation.py | 25 +- .../torch/quantization/test_quantize_cpu.py | 29 -- 29 files changed, 268 insertions(+), 1498 deletions(-) delete mode 100644 modelopt/torch/quantization/plugins/timm.py create mode 100644 modelopt_recipes/timm/resnet/auto_quantize/fp8_int8_at_8p0bits.yaml delete mode 100644 modelopt_recipes/timm/resnet/ptq/README.md delete mode 100644 modelopt_recipes/timm/resnet/ptq/mxfp8.yaml delete mode 100644 modelopt_recipes/timm/resnet/ptq/nvfp4.yaml delete mode 100644 modelopt_recipes/timm/resnet/ptq/nvfp4_awq_lite.yaml delete mode 100644 modelopt_recipes/timm/resnet/ptq/static_fp8.quant_cfg.yaml delete mode 100644 modelopt_recipes/timm/resnet/ptq/static_int8.quant_cfg.yaml delete mode 100644 tests/unit/torch/quantization/plugins/test_timm.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cad42607819..3b65f1ce879 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -81,10 +81,8 @@ Changelog **Bug Fixes** -- Add timm ResNet PTQ recipes for FP8, INT8, MXFP8, and NVFP4. The recipes - keep the input stem convolution unquantized, select TensorRT-compatible convolution formats, - and place shared block-input and projection-shortcut Q/DQ before residual adds, including - during AutoQuantize. +- Add FP8, INT8, and AutoQuantize recipes that quantize timm ResNet shortcut inputs + immediately before residual adds. - Fix ``ShapeInferenceError`` during ONNX INT8 + FP16 quantization (``--high_precision_dtype fp16``) of weakly-typed models (e.g. TensorFlow exports) that carry stale rank-0 ``graph.output`` shapes or ops such as ``TopK`` that ONNX's static shape inference cannot resolve. ``clear_stale_value_info`` now reconciles stale output shapes via symbolic shape inference (keeping every output's shape field populated), and AutoCast runs ONNX shape inference in strict mode and falls back to schema-based standalone type inference when it fails, so unresolved ops no longer leave tensors untyped. - Fused MoE expert auto-detection (``register_fused_experts_on_the_fly``) no longer requires an ``act_fn`` attribute. Some fused-expert modules (e.g. ``MiniMaxM3VLExperts``) apply a custom gated activation between the two ``F.linear`` calls instead of exposing ``act_fn``; they were silently skipped, leaving routed experts unquantized (an experts-only recipe matched nothing) and failing HF export with ``NotImplementedError``. ``_QuantFusedExperts`` is activation-agnostic (it only intercepts the two ``F.linear`` calls), so the requirement was unnecessary. This enables NVFP4/FP8 quantization and export for MiniMax-M2 / MiniMax-M3. - Fix unified HF export emitting transformers' *in-memory* (post-``conversion_mapping``) tensor names instead of the original model-hub names, breaking the unified-checkpoint contract (observed on MiniMax-M3: exported ``model.language_model.*`` / ``mlp.experts.*.gate_proj`` instead of hub ``language_model.model.*`` / ``block_sparse_moe.experts.*.w{1,2,3}``). transformers' own save-side ``revert_weight_conversion`` is disabled by ModelOpt because it raises ``RuntimeError`` on 0-d scalar scale tensors, so a new quant-aware reverse conversion (``modelopt/torch/export/quant_aware_conversion.py``) derives rename/split rules from the model's conversion mapping via transformers' ``reverse_transform()`` and carries each weight's companion scale tensors (``weight_scale``, ``weight_scale_2``, ``input_scale``, ``weight_scale_inv``, ``bias``) through the renames and un-fusions, so quantized exports round-trip to the hub names. Any mapping op that cannot be reversed quant-aware yet (e.g. still-stacked fused experts) falls back to the previous in-memory names instead of aborting the export. diff --git a/examples/torch_onnx/README.md b/examples/torch_onnx/README.md index 8e1e3f1ac25..124f7625114 100644 --- a/examples/torch_onnx/README.md +++ b/examples/torch_onnx/README.md @@ -54,8 +54,7 @@ The `torch_quant_to_onnx.py` script quantizes [timm](https://github.com/huggingf - Loads a pretrained timm torch model (default: ViT-Base). - Quantizes the torch model to FP8, MXFP8, INT8, NVFP4, or INT4_AWQ using ModelOpt. - For models with Conv2d layers (e.g., SwinTransformer), automatically overrides Conv2d quantization to FP8 (for MXFP8/NVFP4 modes) or INT8 (for INT4_AWQ mode) for TensorRT compatibility. -- Uses the [timm ResNet PTQ recipes](../../modelopt_recipes/timm/resnet/ptq/) to - quantize shortcut inputs before residual addition. +- Uses ResNet FP8, INT8, and AutoQuantize recipes to quantize shortcut inputs immediately before residual adds. - Exports the quantized model to ONNX. - Postprocesses the ONNX model to be compatible with TensorRT. - Saves the final ONNX model. @@ -67,10 +66,13 @@ The `torch_quant_to_onnx.py` script quantizes [timm](https://github.com/huggingf ```bash python torch_quant_to_onnx.py \ --timm_model_name= \ - --quantize_mode= \ + --quantize_mode= \ --onnx_save_path= ``` +ResNet AutoQuantize searches FP8 and INT8 at eight effective bits and keeps its +residual connections in FP8. + ### Conv2d Quantization Override TensorRT only supports FP8 and INT8 for convolution operations. When quantizing models with Conv2d layers (like SwinTransformer), the script automatically applies the following overrides: @@ -275,7 +277,7 @@ The `auto` mode enables mixed precision quantization by searching for the optima | Parameter | Default | Description | | :--- | :---: | :--- | -| `--effective_bits` | 4.8 (8.0 for ResNet) | Target average bits per weight across the model. Lower values = more compression but potentially lower accuracy. The ResNet default remains feasible when Conv2d candidates use FP8 or INT8. | +| `--effective_bits` | 4.8 | Target average bits per weight across the model. Lower values = more compression but potentially lower accuracy. The search algorithm finds the optimal per-layer format assignment that meets this constraint while minimizing accuracy loss. For example, 4.8 means an average of 4.8 bits per weight (mix of FP4 and FP8 layers). | | `--num_score_steps` | 128 | Number of forward/backward passes used to compute per-layer sensitivity scores via gradient-based analysis. Higher values provide more accurate sensitivity estimates but increase search time. Recommended range: 64-256. | | `--calibration_data_size` | 512 | Number of calibration samples used for both sensitivity scoring and calibration. For auto mode, labels are required for loss computation. | diff --git a/examples/torch_onnx/torch_quant_to_onnx.py b/examples/torch_onnx/torch_quant_to_onnx.py index a83bffc5ae3..db3aa092444 100644 --- a/examples/torch_onnx/torch_quant_to_onnx.py +++ b/examples/torch_onnx/torch_quant_to_onnx.py @@ -36,7 +36,7 @@ import modelopt.torch.quantization as mtq from modelopt.recipe import load_recipe from modelopt.torch.quantization.nn import TensorQuantizer -from modelopt.torch.quantization.plugins.timm import is_resnet_quantization_supported +from modelopt.torch.quantization.plugins.custom import CUSTOM_POST_CONVERSION_PLUGINS """ Quantize a timm vision model and export to ONNX for TensorRT deployment. @@ -64,14 +64,11 @@ "nvfp4": mtq.NVFP4_DEFAULT_CFG, "int4_awq": mtq.INT4_AWQ_CFG, } -_RESNET_RECIPE_DIR = "timm/resnet/ptq" -_RESNET_QUANTIZE_MODES = {"fp8", "int8", "mxfp8", "nvfp4"} -_RESNET_AUTO_RECIPE_NAMES = { - "FP8_DEFAULT_CFG": "fp8", - "INT8_DEFAULT_CFG": "int8", - "MXFP8_DEFAULT_CFG": "mxfp8", - "NVFP4_DEFAULT_CFG": "nvfp4", - "NVFP4_AWQ_LITE_CFG": "nvfp4_awq_lite", + +_RESNET_RECIPES = { + "fp8": "timm/resnet/ptq/fp8", + "int8": "timm/resnet/ptq/int8", + "auto": "timm/resnet/auto_quantize/fp8_int8_at_8p0bits", } _FP8_CONV_OVERRIDE: list = [ @@ -126,7 +123,7 @@ _NEEDS_INT8_CONV_OVERRIDE: set[str] = {"INT4_AWQ_CFG"} -def get_quant_config(quantize_mode, model=None): +def get_quant_config(quantize_mode, recipe=None): """Get quantization config, overriding Conv2d for TRT compatibility. TensorRT only supports FP8 and INT8 for Conv layers. @@ -136,8 +133,8 @@ def get_quant_config(quantize_mode, model=None): - For MXFP8, NVFP4: override Conv2d to FP8 - For INT4_AWQ: override Conv2d to INT8 """ - if is_resnet_quantization_supported(model) and quantize_mode in _RESNET_QUANTIZE_MODES: - return load_recipe(f"{_RESNET_RECIPE_DIR}/{quantize_mode}").quantize.model_dump() + if recipe is not None: + return recipe.quantize.model_dump() config: dict = copy.deepcopy(QUANT_CONFIG_DICT[quantize_mode]) if quantize_mode == "fp8": @@ -157,17 +154,25 @@ def get_quant_config(quantize_mode, model=None): return config -def get_auto_quant_config(format_name, model): - recipe_name = _RESNET_AUTO_RECIPE_NAMES.get(format_name) - if is_resnet_quantization_supported(model) and recipe_name is not None: - return load_recipe(f"{_RESNET_RECIPE_DIR}/{recipe_name}").quantize.model_dump() +def _get_resnet_recipe(model, quantize_mode): + if not isinstance(model, timm.models.resnet.ResNet): + return None + recipe_path = _RESNET_RECIPES.get(quantize_mode) + return load_recipe(recipe_path) if recipe_path is not None else None - config = copy.deepcopy(getattr(mtq, format_name)) - if format_name in _NEEDS_FP8_CONV_OVERRIDE: - config["quant_cfg"].extend(_FP8_CONV_OVERRIDE) - elif format_name in _NEEDS_INT8_CONV_OVERRIDE: - config["quant_cfg"].extend(_INT8_CONV_OVERRIDE) - return config + +def _add_resnet_residual_quantizers(model): + block_types = (timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck) + for block in (module for module in model.modules() if isinstance(module, block_types)): + if block.downsample is None: + block.downsample = torch.nn.Sequential() + elif not isinstance(block.downsample, torch.nn.Sequential): + block.downsample = torch.nn.Sequential(block.downsample) + if "residual_quantizer" in block.downsample._modules: + continue + residual_quantizer = TensorQuantizer() + residual_quantizer.disable() + block.downsample.add_module("residual_quantizer", residual_quantizer) def filter_func(name): @@ -222,6 +227,30 @@ def hook(m, inp, out, _n=name): mtq.disable_quantizer(model, lambda n: n.startswith(prefixes)) +def _disable_low_channel_conv_input_quantizers(model): + """Disable ``input_quantizer`` on Conv2d modules whose ``in_channels <= 3``. + + The first Conv2d of an image backbone (e.g. ResNet50's ``conv1``) consumes raw + RGB input, so ``in_channels == 3``. On Blackwell (compute capability 12.0) TRT + fails to find an FP8/MXFP8/NVFP4 tactic for this first-layer Q→Conv fusion: + + Error Code 10: Could not find any implementation for node + /conv1/input_quantizer/TRT_FP8QuantizeLinear ... [ElementWise] + + Ada (8.9) happens to have a tactic, which is why local runs pass. Disabling the + input quantizer on the raw-RGB conv is also standard quantization practice — + first/last layers are typically left in higher precision. Weight quantization + still applies. Swin/ViT's ``patch_embed.proj`` is already excluded via + ``filter_func``'s ``patch_embed`` pattern, so this helper is effectively the + ResNet-shaped analogue. + """ + for _, mod in model.named_modules(): + if isinstance(mod, torch.nn.Conv2d) and mod.in_channels <= 3: + q = getattr(mod, "input_quantizer", None) + if q is not None and q.is_enabled: + q.disable() + + def load_calibration_data(model, data_size, batch_size, device, with_labels=False): """Load and prepare calibration data. @@ -266,14 +295,16 @@ def _disable_dead_quantizers(model): calibrates to ``amax == 0``. Disable such dead quantizers — they have nothing meaningful to quantize and would otherwise break ONNX export. """ - for quantizer in model.modules(): - if not isinstance(quantizer, TensorQuantizer) or not quantizer.is_enabled: - continue - amax = quantizer.amax - if amax is None or not torch.is_tensor(amax): - continue - if torch.any(torch.isnan(amax)) or torch.all(amax <= 0): - quantizer.disable() + for _, mod in model.named_modules(): + for attr in ("input_quantizer", "output_quantizer", "weight_quantizer"): + q = getattr(mod, attr, None) + if q is None or not q.is_enabled: + continue + amax = q.amax + if amax is None or not torch.is_tensor(amax): + continue + if torch.any(torch.isnan(amax)) or torch.all(amax <= 0): + q.disable() def _calibrate_uncalibrated_quantizers(model, data_loader): @@ -283,16 +314,15 @@ def _calibrate_uncalibrated_quantizers(model, data_loader): be calibrated because the MXFP8/NVFP4 quantization pipeline skips standard calibration. This function explicitly calibrates those uncalibrated quantizers. """ - uncalibrated = [ - quantizer - for quantizer in model.modules() - if isinstance(quantizer, TensorQuantizer) - and quantizer.is_enabled - and not quantizer.block_sizes - and quantizer.amax is None - ] - for quantizer in uncalibrated: - quantizer.enable_calib() + uncalibrated = [] + for _, module in model.named_modules(): + for attr_name in ("input_quantizer", "weight_quantizer"): + if not hasattr(module, attr_name): + continue + quantizer = getattr(module, attr_name) + if quantizer.is_enabled and not quantizer.block_sizes and quantizer.amax is None: + quantizer.enable_calib() + uncalibrated.append(quantizer) if not uncalibrated: return @@ -319,8 +349,9 @@ def forward_loop(model): else: quantized_model = mtq.quantize(model, config) - if not is_resnet_quantization_supported(quantized_model): - mtq.disable_quantizer(quantized_model, filter_func) + # Disable filtered quantizers BEFORE calibrating override quantizers so we don't + # waste time calibrating quantizers that are about to be turned off. + mtq.disable_quantizer(quantized_model, filter_func) # Calibrate any FP8 override quantizers that weren't calibrated by mtq.quantize(). if data_loader is not None: @@ -362,6 +393,7 @@ def auto_quantize_model( effective_bits=None, num_calib_steps=512, num_score_steps=128, + recipe=None, ): """Auto-quantize the model using optimal per-layer quantization search. @@ -377,21 +409,48 @@ def auto_quantize_model( Tuple of (quantized_model, search_state_dict) """ _disable_inplace_relu(model) - if effective_bits is None: - effective_bits = 8.0 if is_resnet_quantization_supported(model) else 4.8 - constraints = {"effective_bits": effective_bits} - - # Convert string format names to config objects, incorporating Conv2d TRT overrides. - # TRT DynamicQuantize requires 2D/3D input, but Conv2d operates on 4D tensors. - # By including the overrides in the format configs, the auto_quantize search - # correctly accounts for Conv2d being FP8/INT8 in the effective_bits budget. - format_configs = [ - get_auto_quant_config(fmt, model) if isinstance(fmt, str) else fmt - for fmt in quantization_formats - ] - - print(f"Starting auto-quantization search with {len(format_configs)} formats...") - print(f"Effective bits constraint: {effective_bits}") + if recipe is None: + constraints = {"effective_bits": 4.8 if effective_bits is None else effective_bits} + format_configs = [] + for fmt in quantization_formats: + if isinstance(fmt, str): + config = copy.deepcopy(getattr(mtq, fmt)) + if fmt in _NEEDS_FP8_CONV_OVERRIDE: + config["quant_cfg"].extend(_FP8_CONV_OVERRIDE) + elif fmt in _NEEDS_INT8_CONV_OVERRIDE: + config["quant_cfg"].extend(_INT8_CONV_OVERRIDE) + format_configs.append(config) + else: + format_configs.append(fmt) + fixed_quantization_config = None + module_search_spaces = None + disabled_layers = None + method = "gradient" + else: + auto_config = recipe.auto_quantize + constraints = auto_config.constraints.model_dump(exclude_none=True) + if effective_bits is not None: + constraints["effective_bits"] = effective_bits + format_configs = [] + fixed_quantization_config = recipe.quantize.model_dump() + module_search_spaces = [ + { + "module_name_patterns": search_space.module_name_patterns, + "quantization_formats": [ + candidate.model_dump() for candidate in search_space.candidate_formats + ], + "allow_no_quant": search_space.allow_no_quant, + } + for search_space in auto_config.module_search_spaces + ] + disabled_layers = auto_config.disabled_layers + method = auto_config.auto_quantize_method + + format_count = len(format_configs) or sum( + len(search_space["quantization_formats"]) for search_space in module_search_spaces or [] + ) + print(f"Starting auto-quantization search with {format_count} formats...") + print(f"Effective bits constraint: {constraints['effective_bits']}") print(f"Calibration steps: {num_calib_steps}, Scoring steps: {num_score_steps}") quantized_model, search_state = mtq.auto_quantize( @@ -404,10 +463,14 @@ def auto_quantize_model( num_calib_steps=num_calib_steps, num_score_steps=num_score_steps, verbose=True, + fixed_quantization_config=fixed_quantization_config, + module_search_spaces=module_search_spaces, + disabled_layers=disabled_layers, + method=method, ) - if not is_resnet_quantization_supported(quantized_model): - mtq.disable_quantizer(quantized_model, filter_func) + # Disable quantization for specified layers + mtq.disable_quantizer(quantized_model, filter_func) _disable_dead_quantizers(quantized_model) @@ -478,7 +541,6 @@ def main(): nargs="+", choices=[ "NVFP4_AWQ_LITE_CFG", - "NVFP4_DEFAULT_CFG", "FP8_DEFAULT_CFG", "MXFP8_DEFAULT_CFG", "INT8_DEFAULT_CFG", @@ -492,8 +554,8 @@ def main(): type=float, default=None, help=( - "Target effective bits for auto quantization. Defaults to 8.0 for ResNet " - "and 4.8 for other models." + "Target effective bits for auto quantization. Defaults to 4.8 without a recipe " + "and overrides the recipe when provided." ), ) parser.add_argument( @@ -545,6 +607,10 @@ def main(): ) print(f"Base Model - Top-1 Accuracy: {top1:.2f}%, Top-5 Accuracy: {top5:.2f}%") + resnet_recipe = _get_resnet_recipe(model, args.quantize_mode) + if resnet_recipe is not None: + CUSTOM_POST_CONVERSION_PLUGINS.add(_add_resnet_residual_quantizers) + # Quantize model based on mode if args.quantize_mode == "auto": # Auto quantization requires labels for loss computation @@ -563,13 +629,14 @@ def main(): args.effective_bits, args.calibration_data_size, args.num_score_steps, + recipe=resnet_recipe, ) else: # Standard quantization - load calibration data # Note: MXFP8 is dynamic and does not need calibration itself, but when # Conv2d layers are overridden to FP8 (for TRT compatibility), those FP8 # quantizers require calibration data. - config = get_quant_config(args.quantize_mode, model) + config = get_quant_config(args.quantize_mode, resnet_recipe) data_loader = load_calibration_data( model, @@ -586,11 +653,24 @@ def main(): # Auto mode also needs this when an MXFP8/NVFP4 candidate format is in the search set. uses_dynamic_quantize = args.quantize_mode in ("mxfp8", "nvfp4") or ( args.quantize_mode == "auto" + and resnet_recipe is None and any(fmt in _NEEDS_FP8_CONV_OVERRIDE for fmt in args.auto_quantization_formats) ) if uses_dynamic_quantize: _disable_high_rank_input_quantizers(quantized_model, input_shape, device) + # FP8-family modes emit TRT_FP8QuantizeLinear on the first-layer conv; Blackwell has + # no tactic for that 3-channel Q→Conv fusion. Skip for pure INT8 (unaffected). + uses_fp8_conv_input = args.quantize_mode in ("fp8", "mxfp8", "nvfp4") or ( + args.quantize_mode == "auto" + and ( + resnet_recipe is not None + or any(fmt != "INT8_DEFAULT_CFG" for fmt in args.auto_quantization_formats) + ) + ) + if uses_fp8_conv_input: + _disable_low_channel_conv_input_quantizers(quantized_model) + # Print quantization summary print("\nQuantization Summary:") mtq.print_quant_summary(quantized_model) diff --git a/modelopt/onnx/export/fp8_exporter.py b/modelopt/onnx/export/fp8_exporter.py index a80ad3dbb78..427a7791f3b 100644 --- a/modelopt/onnx/export/fp8_exporter.py +++ b/modelopt/onnx/export/fp8_exporter.py @@ -172,9 +172,6 @@ def _quantize_conv_weights_to_fp8(graph: gs.Graph) -> int: 2. Quantize weights to FP8E4M3FN 3. Insert a DequantizeLinear(fp8_weights, scale) before the Conv weight input - An RGB Conv that directly consumes an unquantized graph input is treated as a - filtered input stem and left entirely in high precision. - Args: graph: The onnx-graphsurgeon graph to modify in-place. @@ -182,7 +179,6 @@ def _quantize_conv_weights_to_fp8(graph: gs.Graph) -> int: Number of Conv weight DQ nodes inserted. """ count = 0 - graph_inputs = {tensor.name for tensor in graph.inputs} for node in list(graph.nodes): if node.op != "Conv": @@ -193,8 +189,7 @@ def _quantize_conv_weights_to_fp8(graph: gs.Graph) -> int: weight_input = node.inputs[1] if not isinstance(weight_input, gs.Constant): continue - if node.inputs[0].name in graph_inputs and weight_input.values.shape[1] == 3: - continue + # Skip if weight already has a DQ producer if any(out.op == "DequantizeLinear" for out in weight_input.outputs): continue diff --git a/modelopt/onnx/utils.py b/modelopt/onnx/utils.py index 0619c3ec3a9..bc37c4a9333 100644 --- a/modelopt/onnx/utils.py +++ b/modelopt/onnx/utils.py @@ -28,7 +28,6 @@ import onnx_graphsurgeon as gs from onnx.helper import get_attribute_value from onnx_graphsurgeon import Constant, Node, Variable -from onnxconverter_common.float16 import convert_np_to_float16 from modelopt.onnx.logging_config import logger @@ -1468,17 +1467,13 @@ def fold_q_fp16_to_fp32_casts(onnx_model: onnx.ModelProto) -> onnx.ModelProto: for inp in node.input: consumer_map.setdefault(inp, []).append(node) initializers = {init.name: init for init in onnx_model.graph.initializer} - tensor_types = _build_tensor_type_map(onnx_model) to_remove = [] for node in onnx_model.graph.node: if node.op_type != "Cast": continue cast_to = next((a.i for a in node.attribute if a.name == "to"), None) - if ( - cast_to != onnx.TensorProto.FLOAT - or tensor_types.get(node.input[0]) != onnx.TensorProto.FLOAT16 - ): + if cast_to != onnx.TensorProto.FLOAT: continue consumers = consumer_map.get(node.output[0], []) if not consumers or not all(c.op_type in _Q_OPS for c in consumers): @@ -1497,53 +1492,6 @@ def fold_q_fp16_to_fp32_casts(onnx_model: onnx.ModelProto) -> onnx.ModelProto: return onnx_model -def _convert_q_data_initializers_to_fp16(onnx_model: onnx.ModelProto) -> onnx.ModelProto: - """Convert FP32 initializer data inputs after Q/DQ scales have been normalized to FP16.""" - if get_opset_version(onnx_model) < BASE_MIN_OPSET: - return onnx_model - - consumers: dict[str, list[tuple[onnx.NodeProto, int]]] = defaultdict(list) - for node in onnx_model.graph.node: - for index, input_name in enumerate(node.input): - consumers[input_name].append((node, index)) - - initializers = {initializer.name: initializer for initializer in onnx_model.graph.initializer} - tensor_types = _build_tensor_type_map(onnx_model) - - for name, initializer in list(initializers.items()): - if initializer.data_type != onnx.TensorProto.FLOAT: - continue - - initializer_consumers = consumers.get(name, []) - q_consumers = [ - node for node, index in initializer_consumers if index == 0 and node.op_type in _Q_OPS - ] - if not q_consumers: - continue - - for q_node in q_consumers: - scale_type = tensor_types.get(q_node.input[1]) if len(q_node.input) >= 2 else None - if scale_type != onnx.TensorProto.FLOAT16: - raise ValueError("Q scales must be FP16 before converting Q data initializers") - - fp16_initializer = onnx.numpy_helper.from_array( - convert_np_to_float16(onnx.numpy_helper.to_array(initializer)), initializer.name - ) - if len(q_consumers) == len(initializer_consumers): - initializer.CopyFrom(fp16_initializer) - continue - - fp16_initializer.name = f"{initializer.name}_fp16_q" - while fp16_initializer.name in initializers: - fp16_initializer.name += "_" - onnx_model.graph.initializer.append(fp16_initializer) - initializers[fp16_initializer.name] = fp16_initializer - for node in q_consumers: - node.input[0] = fp16_initializer.name - - return onnx_model - - def _is_foldable_constant_cast_pattern(model: onnx.ModelProto, node: onnx.NodeProto) -> bool: """Check if a Constant -> Cast pattern can be folded.""" assert node.op_type == "Cast" diff --git a/modelopt/torch/_deploy/utils/torch_onnx.py b/modelopt/torch/_deploy/utils/torch_onnx.py index 6ef53bf33dd..01fb754bbae 100644 --- a/modelopt/torch/_deploy/utils/torch_onnx.py +++ b/modelopt/torch/_deploy/utils/torch_onnx.py @@ -45,7 +45,6 @@ ) from modelopt.onnx.quantization.qdq_utils import qdq_to_dq, replace_zero_scale_with_smallest_nonzero from modelopt.onnx.utils import ( - _convert_q_data_initializers_to_fp16, change_casts_to_fp16, check_model_uses_external_data, fold_dq_fp32_to_fp16_casts, @@ -666,11 +665,9 @@ def get_onnx_bytes_and_metadata( onnx_opt_graph = remove_redundant_casts(onnx_opt_graph) # Remove Cast nodes around Q/DQ for optimal TRT fusion - if is_fp8_quantized(model) or (is_int8_quantized(model) and weights_dtype == "fp16"): + if is_fp8_quantized(model): onnx_opt_graph = fold_q_fp16_to_fp32_casts(onnx_opt_graph) onnx_opt_graph = fold_dq_fp32_to_fp16_casts(onnx_opt_graph) - if is_int8_quantized(model) and weights_dtype == "fp16": - onnx_opt_graph = _convert_q_data_initializers_to_fp16(onnx_opt_graph) # TensorRT expects all scales to be postive onnx_opt_graph = replace_zero_scale_with_smallest_nonzero(onnx_opt_graph) diff --git a/modelopt/torch/opt/dynamic.py b/modelopt/torch/opt/dynamic.py index 4acb3bade88..7988f9f970a 100644 --- a/modelopt/torch/opt/dynamic.py +++ b/modelopt/torch/opt/dynamic.py @@ -1015,13 +1015,6 @@ def get_key(self, nn_cls: type[nn.Module] | str) -> str: assert nn_cls_ is not None return self._key_registry[nn_cls_] - def get_registered_class(self, key: str) -> type[nn.Module]: - """Retrieve the registered nn.Module class for a string key.""" - for nn_cls, registered_key in self._key_registry.items(): - if registered_key == key: - return nn_cls - raise KeyError(f"{key} is not registered for a dynamic module!") - def get_rule_class(self, nn_cls: type[nn.Module] | str) -> type[ModeloptBaseRule]: """Retrieve the rule config class that is registered for a given nn module class.""" dm_cls = self.get(nn_cls) diff --git a/modelopt/torch/quantization/algorithms.py b/modelopt/torch/quantization/algorithms.py index 9dd27866ae3..7beeef6ad7f 100644 --- a/modelopt/torch/quantization/algorithms.py +++ b/modelopt/torch/quantization/algorithms.py @@ -52,13 +52,7 @@ ) from .config import QuantizeConfig, QuantizerAttributeConfig, QuantizerCfgEntry from .conversion import set_quantizer_by_cfg -from .nn import ( - QuantLinearConvBase, - QuantModule, - QuantModuleRegistry, - SequentialQuantizer, - TensorQuantizer, -) +from .nn import QuantLinearConvBase, QuantModule, SequentialQuantizer, TensorQuantizer from .utils import is_quantized_linear @@ -107,17 +101,8 @@ def _get_quantizer_attrs(module: nn.Module) -> tuple[str, ...]: For fused MoE experts, this returns the four plural quantizer attrs (two shared input quantizers + two ``ModuleList`` of per-expert weight quantizers). - Modules can override the canonical trio with ``_auto_quantize_quantizer_attrs``. + For standard Linear-derived QuantModules, returns the canonical trio. """ - attr_names = getattr(module, "_auto_quantize_quantizer_attrs", None) - if attr_names is not None: - missing = [attr_name for attr_name in attr_names if not hasattr(module, attr_name)] - if missing: - raise AttributeError( - f"{type(module).__name__} declares missing AutoQuantize quantizer attributes: " - f"{missing}." - ) - return tuple(attr_names) if _is_hf_quant_fused_experts_module(module): try: from .plugins.huggingface import _get_fused_experts_quantizer_attr_names @@ -127,13 +112,6 @@ def _get_quantizer_attrs(module: nn.Module) -> tuple[str, ...]: return _STD_QUANTIZER_ATTRS -def _get_quant_module_parent_class(module: nn.Module) -> str | None: - try: - return QuantModuleRegistry.get_key_from_dm(module) - except KeyError: - return None - - def _make_fresh_quantizer_for_attr(module: nn.Module, attr_name: str) -> nn.Module: """Return a fresh, default quantizer object suitable to overwrite ``module.``. @@ -183,64 +161,45 @@ def _fixed_module_format_signature(module: nn.Module) -> tuple: ) -def _module_weight_compression( - quantizer_attrs: dict[str, nn.Module], - effective_bits_override: float | None = None, +def _fixed_module_weight_compression( + module: nn.Module, effective_bits_override: float | None = None ) -> float: - def tensor_compression(quantizer): - if not quantizer.is_enabled: - return 1.0 + weight_quantizers = [] + for attr_name in _get_quantizer_attrs(module): + if "weight_quantizer" not in attr_name: + continue + weight_quantizers.extend(_iter_tensor_quantizers(getattr(module, attr_name))) + + if not weight_quantizers or all(not quantizer.is_enabled for quantizer in weight_quantizers): + return 1.0 + if any(not quantizer.is_enabled for quantizer in weight_quantizers): + raise ValueError( + "The fixed quantize baseline enables only some weight quantizers within one " + "quantizable module. Move that module into an explicit AutoQuantize " + "module_search_spaces entry." + ) + if effective_bits_override is not None: + return effective_bits_override / 16 + + compressions = [] + for quantizer in weight_quantizers: effective_bits = getattr(quantizer, "_effective_bits", None) num_bits = quantizer.num_bits if effective_bits is not None: - return effective_bits / 16 - if isinstance(num_bits, tuple): - return (sum(num_bits) + 1) / 16 - if isinstance(num_bits, int): - return num_bits / 16 - raise ValueError(f"Cannot infer AutoQuantize cost from num_bits={num_bits!r}.") - - def weight_quantizer_compression(quantizer): - if isinstance(quantizer, TensorQuantizer): - return tensor_compression(quantizer) - if isinstance(quantizer, SequentialQuantizer): - stage_compressions = [weight_quantizer_compression(stage) for stage in quantizer] - return min(stage_compressions, default=1.0) - if isinstance(quantizer, nn.ModuleList): - parallel_compressions = [weight_quantizer_compression(child) for child in quantizer] - if any( - abs(value - parallel_compressions[0]) > 1e-12 for value in parallel_compressions[1:] - ): - raise ValueError( - "A quantization recipe assigns different weight formats within one " - "quantizable module. Use one weight format for the entire module." - ) - return parallel_compressions[0] if parallel_compressions else 1.0 - raise TypeError(f"Unsupported weight quantizer type {type(quantizer)}.") - - compressions = [ - weight_quantizer_compression(quantizer) - for attr_name, quantizer in quantizer_attrs.items() - if "weight_quantizer" in attr_name - ] - if not compressions or all(value == 1.0 for value in compressions): - return 1.0 + compressions.append(effective_bits / 16) + elif isinstance(num_bits, tuple): + compressions.append((sum(num_bits) + 1) / 16) + elif isinstance(num_bits, int): + compressions.append(num_bits / 16) + else: + raise ValueError(f"Cannot infer AutoQuantize cost from num_bits={num_bits!r}.") if any(abs(value - compressions[0]) > 1e-12 for value in compressions[1:]): raise ValueError( - "A quantization recipe assigns different weight formats within one quantizable " - "module. Use one weight format for the entire module." + "The fixed quantize baseline assigns different weight formats within one quantizable " + "module. Move that module into an explicit AutoQuantize module_search_spaces entry." ) - return effective_bits_override / 16 if effective_bits_override is not None else compressions[0] - - -def _fixed_module_weight_compression( - module: nn.Module, effective_bits_override: float | None = None -) -> float: - return _module_weight_compression( - {attr_name: getattr(module, attr_name) for attr_name in _get_quantizer_attrs(module)}, - effective_bits_override, - ) + return compressions[0] def estimate_quant_compression(quant_cfg: QuantizeConfig) -> float: @@ -449,10 +408,6 @@ def __init__( name: tuple(_get_replay_quantizer_attr(attr) for attr in _get_quantizer_attrs(module)) for module, name in zip(quant_modules or [], self.quant_module_names) } - self.quant_module_parent_classes = { - name: _get_quant_module_parent_class(module) - for module, name in zip(quant_modules or [], self.quant_module_names) - } assert cost_weight >= 0.0, "cost_weight must be non-negative." self.cost_weight = cost_weight self.allow_no_quant = allow_no_quant @@ -603,23 +558,14 @@ def get_cost(self, recipe: QuantRecipe, cost_weight: float | None = None) -> flo """ cost_weight = self.cost_weight if cost_weight is None else cost_weight cost = 0 - quantizer_choices = self._all_quantizer_choices.get(recipe) for quant_module in self.quant_modules: weight_size = ( _AutoQuantizeBaseSearcher._get_total_weight_size([quant_module]) * cost_weight ) - compression = ( - _module_weight_compression( - quantizer_choices[quant_module], - recipe.config.effective_bits, - ) - if quantizer_choices is not None - else recipe.compression - ) parallel_state = getattr(quant_module, "parallel_state", None) if parallel_state is None: - cost += weight_size * compression + cost += weight_size * recipe.compression continue weight_size = DistributedProcessGroup.get_dist_syncd_obj( @@ -637,7 +583,7 @@ def get_cost(self, recipe: QuantRecipe, cost_weight: float | None = None) -> flo [parallel_state.data_parallel_group], lambda a: a[0], ) - cost += weight_size * compression + cost += weight_size * recipe.compression return cost @@ -768,8 +714,6 @@ def load_search_checkpoint(self) -> bool: @staticmethod def _is_auto_quantize_module(module): - if getattr(module, "_auto_quantize_disabled", False): - return False if (is_quantized_linear(module) or isinstance(module, QuantLinearConvBase)) and isinstance( module, QuantModule ): @@ -1130,7 +1074,6 @@ def initialize_candidate_stats(self): self.candidate_stats[name]["costs"] = costs self.candidate_stats[name]["module_names"] = hparam.quant_module_names self.candidate_stats[name]["quantizer_attrs"] = hparam.quant_module_replay_attrs - self.candidate_stats[name]["parent_classes"] = hparam.quant_module_parent_classes self.candidate_stats[name]["cost_weight"] = hparam.cost_weight self.candidate_stats[name]["allow_no_quant"] = hparam.allow_no_quant self.candidate_stats[name]["is_fixed"] = hparam.is_fixed @@ -2069,16 +2012,11 @@ def _cfg_to_dict(v): for pattern in _as_list(search_state.get("disabled_layers")) ) per_module_entries: list[dict] = [] - per_module_attrs = { + _per_module_attrs = ( *_STD_QUANTIZER_ATTRS, *_FUSED_EXPERTS_REPLAY_QUANTIZER_ATTRS, *_NON_GATED_FUSED_EXPERTS_REPLAY_QUANTIZER_ATTRS, - } - for candidate_stat in search_state["candidate_stats"].values(): - quantizer_attrs = candidate_stat.get("quantizer_attrs") - if isinstance(quantizer_attrs, dict): - for attrs in quantizer_attrs.values(): - per_module_attrs.update(attrs) + ) # Track global (non per-module) recipe entries. Last recipe wins for each pattern. global_entries: dict[str, dict] = {} @@ -2089,16 +2027,10 @@ def _cfg_to_dict(v): if recipe == QuantRecipe(quant_cfg=None): continue module_names = candidate_stat["module_names"] - parent_classes = candidate_stat.get("parent_classes") for module_name in module_names: - parent_class = ( - parent_classes.get(module_name) if isinstance(parent_classes, dict) else None - ) for quantizer_attr in _get_replay_quantizer_attrs(candidate_stat, module_name): matched_cfg, matched_enable = _match_quantizer_cfg( - recipe.config.quant_cfg, - quantizer_attr, - parent_class, + recipe.config.quant_cfg, quantizer_attr ) if matched_enable is not None: entry: dict[str, Any] = { @@ -2112,7 +2044,10 @@ def _cfg_to_dict(v): # Collect non-per-module entries (e.g. *[kv]_bmm_quantizer) from winning recipes. for recipe_entry in recipe.config.quant_cfg: pattern = recipe_entry["quantizer_name"] - if pattern == "*" or any(fnmatch.fnmatch(attr, pattern) for attr in per_module_attrs): + if pattern == "*" or any( + fnmatch.fnmatch(attr, pattern) or pattern.endswith(attr) + for attr in _per_module_attrs + ): continue cfg = recipe_entry.get("cfg") enable = recipe_entry.get("enable", True) @@ -2180,29 +2115,22 @@ def _resolve_best_recipe(search_state, constraints, verbose=False): return best_recipe -def _match_quantizer_cfg(quant_cfg, quantizer_attr, module_parent_class=None): +def _match_quantizer_cfg(quant_cfg, quantizer_attr): # Last-match-wins to mirror set_quantizer_by_cfg behavior. - # AutoQuantize applies each candidate to an isolated module, so only patterns that match - # its bare quantizer attribute participate in the per-module candidate. + # Patterns may be path-scoped (e.g. "*mlp*weight_quantizer") while quantizer_attr + # is a bare name like "weight_quantizer". We match if the bare name matches directly + # OR if the pattern ends with the bare quantizer_attr (path-scoped match). matched = None matched_enable = None for entry in quant_cfg: parent_class = entry.get("parent_class") if hasattr(entry, "get") else entry.parent_class if parent_class is not None: - if module_parent_class is None: - continue - if parent_class != module_parent_class: - try: - module_cls = QuantModuleRegistry.get_registered_class(module_parent_class) - parent_cls = QuantModuleRegistry.get_registered_class(parent_class) - except KeyError: - continue - if not issubclass(module_cls, parent_cls): - continue + continue pattern = entry["quantizer_name"] cfg = entry.get("cfg") enable = entry.get("enable", True) - if fnmatch.fnmatch(quantizer_attr, pattern): + # Direct match: the bare quantizer_attr matches the whole pattern (e.g. "*weight_quantizer") + if fnmatch.fnmatch(quantizer_attr, pattern) or pattern.endswith(quantizer_attr): matched = cfg matched_enable = enable diff --git a/modelopt/torch/quantization/conversion.py b/modelopt/torch/quantization/conversion.py index 2ece1f7d461..00187d291c0 100644 --- a/modelopt/torch/quantization/conversion.py +++ b/modelopt/torch/quantization/conversion.py @@ -25,7 +25,7 @@ import torch.nn as nn from modelopt.torch.opt.conversion import ApplyModeError, ModelLikeModule, ModeloptStateManager -from modelopt.torch.opt.dynamic import DynamicModule, _DMRegistryCls +from modelopt.torch.opt.dynamic import _DMRegistryCls from modelopt.torch.opt.mode import ConvertReturnType, MetadataDict from modelopt.torch.utils import get_unwrapped_name @@ -364,21 +364,9 @@ def _match_quantizer( # Get the parent module of this quantizer. When name has no dots (root-level quantizer), # ".".join([]) == "" and get_submodule("") returns the model itself (PyTorch convention). - if parent_class is None: - return True - - parent_module = full_model.get_submodule(".".join(name.split(".")[:-1])) - if isinstance(parent_module, parent_class): - return True - if not isinstance(parent_module, DynamicModule): - return False - - try: - parent_class_key = QuantModuleRegistry.get_key_from_dm(parent_class) - registered_parent_class = QuantModuleRegistry.get_registered_class(parent_class_key) - except KeyError: - return False - return issubclass(parent_module.original_cls, registered_parent_class) + return parent_class is None or isinstance( + full_model.get_submodule(".".join(name.split(".")[:-1])), parent_class + ) def set_quantizer_attributes_full( diff --git a/modelopt/torch/quantization/plugins/__init__.py b/modelopt/torch/quantization/plugins/__init__.py index fe622ef5a3b..22b4bc2e3cc 100644 --- a/modelopt/torch/quantization/plugins/__init__.py +++ b/modelopt/torch/quantization/plugins/__init__.py @@ -58,9 +58,6 @@ with import_plugin("torch_geometric"): from .pytorch_geometric import * -with import_plugin("timm"): - from .timm import * - with import_plugin("transformer_engine"): from .transformer_engine import * diff --git a/modelopt/torch/quantization/plugins/timm.py b/modelopt/torch/quantization/plugins/timm.py deleted file mode 100644 index f661833c2db..00000000000 --- a/modelopt/torch/quantization/plugins/timm.py +++ /dev/null @@ -1,287 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Quantization support for timm modules.""" - -import torch.nn as nn -from timm.models.resnet import BasicBlock, Bottleneck, ResNet - -from ..algorithms import AutoQuantizeGradientSearcher, AutoQuantizeKLDivSearcher -from ..nn import QuantModule, QuantModuleRegistry, TensorQuantizer -from ..nn.modules.quant_conv import _QuantConv2d -from .custom import CUSTOM_MODEL_PLUGINS, CUSTOM_POST_CONVERSION_PLUGINS - - -# Forward overrides give marker subclasses distinct registry entries without changing computation. -class _ResNetInputConv2d(nn.Conv2d): - def forward(self, input): - return super().forward(input) - - -class _ResNetOutputConv2d(nn.Conv2d): - def forward(self, input): - return super().forward(input) - - -class _ResNetProjectionOutputConv2d(_ResNetOutputConv2d): - def forward(self, input): - return super().forward(input) - - -class _ResNetFinalOutputConv2d(_ResNetOutputConv2d): - def forward(self, input): - return super().forward(input) - - -class _ResNetFinalProjectionOutputConv2d(_ResNetProjectionOutputConv2d): - def forward(self, input): - return super().forward(input) - - -class _ResNetStemConv2d(nn.Conv2d): - def forward(self, input): - return super().forward(input) - - -class _ResNetBasicBlock(BasicBlock): - def forward(self, input): - return super().forward(input) - - -class _ResNetBottleneck(Bottleneck): - def forward(self, input): - return super().forward(input) - - -def is_resnet_quantization_supported(model): - """Return whether the model uses a supported timm ResNet block implementation.""" - if not isinstance(model, ResNet): - return False - blocks = [module for module in model.modules() if isinstance(module, (BasicBlock, Bottleneck))] - supported_types = (BasicBlock, Bottleneck, _ResNetBasicBlock, _ResNetBottleneck) - return bool(blocks) and all( - type(block) in supported_types or getattr(block, "original_cls", None) in supported_types - for block in blocks - ) - - -@QuantModuleRegistry.register( - { - _ResNetBasicBlock: "timm.ResNetBasicBlock", - _ResNetBottleneck: "timm.ResNetBottleneck", - } -) -class _QuantResNetBlock(QuantModule): - def _setup(self): - pass - - def forward(self, input): - output_conv = self.conv3 if isinstance(self, Bottleneck) else self.conv2 - return super().forward(output_conv.block_input_activation_quantizer(input)) - - -def _register_disabled_quantizer(module, name): - quantizer = TensorQuantizer() - quantizer.disable() - module._register_temp_attribute(name, quantizer) - - -@QuantModuleRegistry.register({_ResNetOutputConv2d: "timm.ResNetOutputConv2d"}) -class _QuantResNetOutputConv2d(_QuantConv2d): - _auto_quantize_quantizer_attrs = ( - "input_quantizer", - "weight_quantizer", - "output_quantizer", - "block_input_activation_quantizer", - ) - - def _setup(self): - super()._setup() - _register_disabled_quantizer(self, "block_input_activation_quantizer") - - -@QuantModuleRegistry.register({_ResNetProjectionOutputConv2d: "timm.ResNetProjectionOutputConv2d"}) -class _QuantResNetProjectionOutputConv2d(_QuantResNetOutputConv2d): - _auto_quantize_quantizer_attrs = ( - *_QuantResNetOutputConv2d._auto_quantize_quantizer_attrs, - "residual_quantizer", - ) - - def _setup(self): - super()._setup() - _register_disabled_quantizer(self, "residual_quantizer") - - -@QuantModuleRegistry.register({_ResNetFinalOutputConv2d: "timm.ResNetFinalOutputConv2d"}) -class _QuantResNetFinalOutputConv2d(_QuantResNetOutputConv2d): - _auto_quantize_quantizer_attrs = ( - *_QuantResNetOutputConv2d._auto_quantize_quantizer_attrs, - "model_output_activation_quantizer", - ) - - def _setup(self): - super()._setup() - _register_disabled_quantizer(self, "model_output_activation_quantizer") - - -@QuantModuleRegistry.register( - {_ResNetFinalProjectionOutputConv2d: "timm.ResNetFinalProjectionOutputConv2d"} -) -class _QuantResNetFinalProjectionOutputConv2d(_QuantResNetProjectionOutputConv2d): - _auto_quantize_quantizer_attrs = ( - *_QuantResNetProjectionOutputConv2d._auto_quantize_quantizer_attrs, - "model_output_activation_quantizer", - ) - - def _setup(self): - super()._setup() - _register_disabled_quantizer(self, "model_output_activation_quantizer") - - -QuantModuleRegistry.register({_ResNetInputConv2d: "timm.ResNetInputConv2d"})(_QuantConv2d) - - -@QuantModuleRegistry.register({_ResNetStemConv2d: "timm.ResNetStemConv2d"}) -class _QuantResNetStemConv2d(_QuantConv2d): - _auto_quantize_disabled = True - - -def _mark_resnet_convs(model): - for resnet in (module for module in model.modules() if isinstance(module, ResNet)): - if not is_resnet_quantization_supported(resnet): - continue - stem_conv = next( - (module for module in resnet.conv1.modules() if type(module) is nn.Conv2d), None - ) - if stem_conv is not None: - stem_conv.__class__ = _ResNetStemConv2d - blocks = [module for module in resnet.modules() if type(module) in (BasicBlock, Bottleneck)] - for index, block in enumerate(blocks): - is_bottleneck = isinstance(block, Bottleneck) - if type(block.conv1) is nn.Conv2d: - block.conv1.__class__ = _ResNetInputConv2d - if block.downsample is not None: - downsample_ops = list(block.downsample.children()) or [block.downsample] - for module in downsample_ops: - if type(module) is nn.Identity: - continue - if type(module) is nn.Conv2d: - module.__class__ = _ResNetInputConv2d - break - output_conv = block.conv3 if is_bottleneck else block.conv2 - if type(output_conv) is nn.Conv2d: - is_projection = block.downsample is not None - is_final = index == len(blocks) - 1 - if is_final: - output_conv.__class__ = ( - _ResNetFinalProjectionOutputConv2d - if is_projection - else _ResNetFinalOutputConv2d - ) - else: - output_conv.__class__ = ( - _ResNetProjectionOutputConv2d if is_projection else _ResNetOutputConv2d - ) - block.__class__ = _ResNetBottleneck if is_bottleneck else _ResNetBasicBlock - - -CUSTOM_MODEL_PLUGINS.add(_mark_resnet_convs) - - -def _register_resnet_quantizer_hooks(model): - for resnet in (module for module in model.modules() if isinstance(module, ResNet)): - blocks = [ - module for module in resnet.modules() if isinstance(module, (BasicBlock, Bottleneck)) - ] - for block in blocks: - if block.downsample is None: - continue - output_conv = block.conv3 if isinstance(block, Bottleneck) else block.conv2 - if not hasattr(output_conv, "residual_quantizer"): - continue - handle = block.downsample.register_forward_hook( - lambda _module, _inputs, output, conv=output_conv: conv.residual_quantizer(output) - ) - output_conv._register_temp_attribute( - "_residual_quantizer_hook", - handle, - del_hook=lambda module, name: getattr(module, name).remove(), - ) - - if not blocks: - continue - final_output_conv = ( - blocks[-1].conv3 if isinstance(blocks[-1], Bottleneck) else blocks[-1].conv2 - ) - if not hasattr(final_output_conv, "model_output_activation_quantizer"): - continue - handle = resnet.global_pool.register_forward_pre_hook( - lambda _module, inputs, conv=final_output_conv: ( - conv.model_output_activation_quantizer(inputs[0]), - *inputs[1:], - ) - ) - final_output_conv._register_temp_attribute( - "_model_output_quantizer_hook", - handle, - del_hook=lambda module, name: getattr(module, name).remove(), - ) - - -CUSTOM_POST_CONVERSION_PLUGINS.add(_register_resnet_quantizer_hooks) - - -def _resnet_block_context(model, name): - parts = name.split(".") - for index in range(len(parts) - 1, -1, -1): - block_name = ".".join(parts[:index]) - block = model.get_submodule(block_name) - if not isinstance(block, (BasicBlock, Bottleneck)): - continue - relative_name = ".".join(parts[index:]) - if relative_name not in ("conv1", "conv2", "conv3") and not relative_name.startswith( - "downsample." - ): - return None - output_conv = block.conv3 if isinstance(block, Bottleneck) else block.conv2 - if not hasattr(output_conv, "block_input_activation_quantizer"): - return None - return block_name, block, output_conv - return None - - -def _resnet_block_group(model, name): - context = _resnet_block_context(model, name) - return context[0] if context is not None else None - - -def _resnet_block_score(model, name): - context = _resnet_block_context(model, name) - if context is None: - return None - block_name, _, output_conv = context - if not hasattr(output_conv, "model_output_activation_quantizer"): - return block_name - parts = block_name.split(".") - for index in range(len(parts), -1, -1): - resnet_name = ".".join(parts[:index]) - if isinstance(model.get_submodule(resnet_name), ResNet): - return ".".join(filter(None, (resnet_name, "global_pool"))) - return block_name - - -AutoQuantizeGradientSearcher.quant_grouping_rules.append(_resnet_block_group) -AutoQuantizeGradientSearcher.score_module_rules.append(_resnet_block_score) # type: ignore[arg-type] -AutoQuantizeKLDivSearcher.score_module_rules.append(_resnet_block_score) diff --git a/modelopt_recipes/README.md b/modelopt_recipes/README.md index 9affd30d3d4..b366e4cc670 100644 --- a/modelopt_recipes/README.md +++ b/modelopt_recipes/README.md @@ -43,7 +43,6 @@ huggingface/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast`. |-----------|-----------------| | `general/` | **Model-agnostic** recipes — a good starting point for any model. PTQ combos, speculative-decoding training, and distillation. | | `huggingface//` | **Model-specific** recipes keyed by a HF `model_type`, optionally nested by released checkpoint. Use these first if your model has an entry. | -| `timm//` | **timm architecture-specific** recipes, including deployment-aware vision PTQ choices. | | `models//` | **Instance-specific** recipes that mirror a particular published checkpoint's quantization config. | | `configs/` | Shared building blocks (`numerics/`, `ptq/units/`, `ptq/presets/`) that recipes compose from via `$import`. Not run directly. | @@ -76,12 +75,6 @@ exclusions are still inherited from `configs/`. Browse folder has a `README.md` describing the exact delta. See [`ptq.md`](ptq.md) for how the model-specific recipes compare to the general ones and why they deviate. -## `timm/` — architecture-specific recipes - -Recipes under `timm//` capture quantization choices required by -vision architectures and their deployment backends. See -[`timm/resnet/ptq/`](timm/resnet/ptq/) for ResNet recipes. - ## `models/` — checkpoint-specific recipes These mirror a single **published checkpoint's** quantization config exactly — @@ -97,7 +90,6 @@ a per-component mixed-precision scheme tuned to match a specific release. Browse - **Tuned for a HF architecture** → `huggingface///`, with a `README.md` documenting the delta from the generic preset. Verify the exact `model_type` against the checkpoint's `config.json` before placing it. -- **Tuned for a timm architecture** → `timm///`. - **Mirrors a specific released checkpoint** → `models//`. - Share reused bodies via a `# modelopt-schema:`-tagged snippet and `$import` it; keep recipe wrappers thin. diff --git a/modelopt_recipes/timm/resnet/auto_quantize/fp8_int8_at_8p0bits.yaml b/modelopt_recipes/timm/resnet/auto_quantize/fp8_int8_at_8p0bits.yaml new file mode 100644 index 00000000000..233e8f88fe8 --- /dev/null +++ b/modelopt_recipes/timm/resnet/auto_quantize/fp8_int8_at_8p0bits.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# modelopt-schema: modelopt.recipe.config.ModelOptAutoQuantizeRecipe +imports: + base_disable_all: configs/ptq/units/base_disable_all + fp8: configs/numerics/fp8 + fp8_model: configs/ptq/presets/model/fp8 + int8_model: configs/ptq/presets/model/int8 + +metadata: + recipe_type: auto_quantize + description: FP8 and INT8 ResNet AutoQuantize with FP8 residual connections. + +quantize: + algorithm: max + quant_cfg: + - $import: base_disable_all + - quantizer_name: '*residual_quantizer' + cfg: + $import: fp8 + +auto_quantize: + constraints: + effective_bits: 8.0 + module_search_spaces: + - module_name_patterns: + - '*' + candidate_formats: + - $import: fp8_model + - $import: int8_model + allow_no_quant: false + auto_quantize_method: gradient + score_size: 128 diff --git a/modelopt_recipes/timm/resnet/ptq/README.md b/modelopt_recipes/timm/resnet/ptq/README.md deleted file mode 100644 index 500eb88113c..00000000000 --- a/modelopt_recipes/timm/resnet/ptq/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# timm ResNet PTQ recipes - -These recipes support timm ResNet models built from the standard `BasicBlock` -or `Bottleneck` and keep their quantizer placement and numeric choices out of -the torch ONNX example: - -- The three-channel stem convolution remains unquantized. -- Every block input is quantized once and shared by the main and identity - shortcut paths. -- Projection shortcuts are quantized immediately before the residual add. -- INT8 quantizes the final block output before global pooling. -- MXFP8 and NVFP4 recipes use FP8 for convolution and residual inputs, matching - TensorRT convolution support. - -| Recipe | Numerics | -|--------|----------| -| `fp8.yaml` | FP8 convolution and residual inputs; classifier unquantized. | -| `int8.yaml` | INT8 convolution and residual inputs; classifier unquantized. | -| `mxfp8.yaml` | MXFP8 with FP8 convolution and residual inputs. | -| `nvfp4.yaml` | NVFP4 with FP8 convolution and residual inputs. | -| `nvfp4_awq_lite.yaml` | AWQ-lite NVFP4 AutoQuantize candidate with FP8 convolution and residual inputs. | - -`static_fp8.quant_cfg.yaml` and `static_int8.quant_cfg.yaml` are shared recipe -snippets, not standalone recipes. diff --git a/modelopt_recipes/timm/resnet/ptq/fp8.yaml b/modelopt_recipes/timm/resnet/ptq/fp8.yaml index 38609c27bff..9de05bd98a9 100644 --- a/modelopt_recipes/timm/resnet/ptq/fp8.yaml +++ b/modelopt_recipes/timm/resnet/ptq/fp8.yaml @@ -1,22 +1,23 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers - fp8: configs/ptq/units/w8a8_fp8_fp8 - resnet: timm/resnet/ptq/static_fp8.quant_cfg + fp8: configs/numerics/fp8 + w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 metadata: recipe_type: ptq - description: FP8 ResNet PTQ with quantized residual inputs. + description: FP8 ResNet PTQ with quantized residual connections. quantize: algorithm: max quant_cfg: - $import: base_disable_all - - $import: fp8 + - $import: w8a8_fp8_fp8 - $import: default_disabled_quantizers - - $import: resnet - - quantizer_name: 'fc.*' - enable: false + - quantizer_name: '*residual_quantizer' + cfg: + $import: fp8 diff --git a/modelopt_recipes/timm/resnet/ptq/int8.yaml b/modelopt_recipes/timm/resnet/ptq/int8.yaml index cb313b58292..68c10f055d6 100644 --- a/modelopt_recipes/timm/resnet/ptq/int8.yaml +++ b/modelopt_recipes/timm/resnet/ptq/int8.yaml @@ -1,16 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers int8: configs/numerics/int8 int8_per_channel: configs/numerics/int8_per_channel - resnet: timm/resnet/ptq/static_int8.quant_cfg metadata: recipe_type: ptq - description: INT8 ResNet PTQ with quantized residual inputs. + description: INT8 ResNet PTQ with quantized residual connections. quantize: algorithm: max @@ -23,6 +23,6 @@ quantize: cfg: $import: int8 - $import: default_disabled_quantizers - - $import: resnet - - quantizer_name: 'fc.*' - enable: false + - quantizer_name: '*residual_quantizer' + cfg: + $import: int8 diff --git a/modelopt_recipes/timm/resnet/ptq/mxfp8.yaml b/modelopt_recipes/timm/resnet/ptq/mxfp8.yaml deleted file mode 100644 index 875d49b5ae4..00000000000 --- a/modelopt_recipes/timm/resnet/ptq/mxfp8.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -imports: - base_disable_all: configs/ptq/units/base_disable_all - default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers - mxfp8: configs/numerics/mxfp8 - resnet: timm/resnet/ptq/static_fp8.quant_cfg - -metadata: - recipe_type: ptq - description: MXFP8 ResNet PTQ with FP8 convolution and residual inputs. - -quantize: - algorithm: max - quant_cfg: - - $import: base_disable_all - - quantizer_name: '*weight_quantizer' - cfg: - $import: mxfp8 - - quantizer_name: '*input_quantizer' - cfg: - $import: mxfp8 - - $import: default_disabled_quantizers - - $import: resnet diff --git a/modelopt_recipes/timm/resnet/ptq/nvfp4.yaml b/modelopt_recipes/timm/resnet/ptq/nvfp4.yaml deleted file mode 100644 index 9f01ca857ef..00000000000 --- a/modelopt_recipes/timm/resnet/ptq/nvfp4.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -imports: - base_disable_all: configs/ptq/units/base_disable_all - default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers - nvfp4: configs/ptq/units/w4a4_nvfp4_nvfp4 - resnet: timm/resnet/ptq/static_fp8.quant_cfg - -metadata: - recipe_type: ptq - description: NVFP4 ResNet PTQ with FP8 convolution and residual inputs. - -quantize: - algorithm: max - quant_cfg: - - $import: base_disable_all - - $import: nvfp4 - - $import: default_disabled_quantizers - - $import: resnet diff --git a/modelopt_recipes/timm/resnet/ptq/nvfp4_awq_lite.yaml b/modelopt_recipes/timm/resnet/ptq/nvfp4_awq_lite.yaml deleted file mode 100644 index 33a9bbd8011..00000000000 --- a/modelopt_recipes/timm/resnet/ptq/nvfp4_awq_lite.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -imports: - base_disable_all: configs/ptq/units/base_disable_all - default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers - nvfp4: configs/ptq/units/w4a4_nvfp4_nvfp4 - resnet: timm/resnet/ptq/static_fp8.quant_cfg - -metadata: - recipe_type: ptq - description: NVFP4 AWQ-lite ResNet PTQ with FP8 convolution and residual inputs. - -quantize: - algorithm: awq_lite - quant_cfg: - - $import: base_disable_all - - $import: nvfp4 - - $import: default_disabled_quantizers - - $import: resnet diff --git a/modelopt_recipes/timm/resnet/ptq/static_fp8.quant_cfg.yaml b/modelopt_recipes/timm/resnet/ptq/static_fp8.quant_cfg.yaml deleted file mode 100644 index 0902d15406a..00000000000 --- a/modelopt_recipes/timm/resnet/ptq/static_fp8.quant_cfg.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# modelopt-schema: modelopt.torch.quantization.config.QuantizerCfgListConfig -imports: - fp8: configs/numerics/fp8 ---- - - parent_class: nn.Conv2d - quantizer_name: '*weight_quantizer' - cfg: - $import: fp8 - - parent_class: nn.Conv2d - quantizer_name: '*input_quantizer' - cfg: - $import: fp8 - - parent_class: timm.ResNetInputConv2d - quantizer_name: '*input_quantizer' - enable: false - - quantizer_name: '*block_input_activation_quantizer' - cfg: - $import: fp8 - - quantizer_name: '*residual_quantizer' - cfg: - $import: fp8 - - parent_class: timm.ResNetStemConv2d - quantizer_name: '*' - enable: false - - parent_class: nn.MaxPool2d - quantizer_name: '*input_quantizer' - enable: false - - parent_class: nn.AvgPool2d - quantizer_name: '*input_quantizer' - enable: false - - parent_class: nn.AdaptiveAvgPool2d - quantizer_name: '*input_quantizer' - enable: false diff --git a/modelopt_recipes/timm/resnet/ptq/static_int8.quant_cfg.yaml b/modelopt_recipes/timm/resnet/ptq/static_int8.quant_cfg.yaml deleted file mode 100644 index 6150d96cef2..00000000000 --- a/modelopt_recipes/timm/resnet/ptq/static_int8.quant_cfg.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# modelopt-schema: modelopt.torch.quantization.config.QuantizerCfgListConfig -imports: - int8: configs/numerics/int8 - int8_per_channel: configs/numerics/int8_per_channel ---- - - parent_class: nn.Conv2d - quantizer_name: '*weight_quantizer' - cfg: - $import: int8_per_channel - - parent_class: nn.Conv2d - quantizer_name: '*input_quantizer' - cfg: - $import: int8 - - parent_class: timm.ResNetInputConv2d - quantizer_name: '*input_quantizer' - enable: false - - quantizer_name: '*block_input_activation_quantizer' - cfg: - $import: int8 - - quantizer_name: '*residual_quantizer' - cfg: - $import: int8 - - quantizer_name: '*model_output_activation_quantizer' - cfg: - $import: int8 - - parent_class: timm.ResNetStemConv2d - quantizer_name: '*' - enable: false - - parent_class: nn.MaxPool2d - quantizer_name: '*input_quantizer' - enable: false - - parent_class: nn.AvgPool2d - quantizer_name: '*input_quantizer' - enable: false - - parent_class: nn.AdaptiveAvgPool2d - quantizer_name: '*input_quantizer' - enable: false diff --git a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py index 15785bc4fb4..d9da18dee8c 100644 --- a/tests/examples/torch_onnx/test_torch_quant_to_onnx.py +++ b/tests/examples/torch_onnx/test_torch_quant_to_onnx.py @@ -23,6 +23,7 @@ # TODO: Add int4_awq once the INT4 exporter supports non-MatMul/Gemm consumer patterns # (e.g., DQ -> Reshape -> Slice in small ViT / SwinTransformer ONNX graphs). _QUANT_MODES = ["fp8", "int8", "mxfp8", "nvfp4", "auto"] +_RESNET_QUANT_MODES = {"fp8", "int8", "auto"} _MODELS = { "vit_tiny": ("vit_tiny_patch16_224", '{"depth": 1}'), @@ -32,9 +33,8 @@ } -def _assert_residual_adds_are_quantized(onnx_save_path, quantize_mode): +def _assert_residual_inputs_are_quantized(onnx_save_path): model = onnx.load(onnx_save_path) - initializers = {initializer.name: initializer for initializer in model.graph.initializer} consumers = defaultdict(list) producers = {} for node in model.graph.node: @@ -57,71 +57,17 @@ def _assert_residual_adds_are_quantized(onnx_save_path, quantize_mode): ] assert any( node.op_type.endswith("DequantizeLinear") + and "/downsample/residual_quantizer/" in node.name and producers[node.input[0]].op_type.endswith("QuantizeLinear") for node in input_producers ) - first_conv = next(node for node in model.graph.node if node.op_type == "Conv") - assert all( - producers.get(input_name) is None - or not producers[input_name].op_type.endswith("DequantizeLinear") - for input_name in first_conv.input - ) - - if quantize_mode not in ("int8", "fp8"): - return - - activation_quantizers = [ - node - for node in model.graph.node - if node.op_type.endswith("QuantizeLinear") and node.input[0] not in initializers - ] - assert len(activation_quantizers) == (53 if quantize_mode == "int8" else 52) - assert len({node.input[0] for node in activation_quantizers}) == len(activation_quantizers) - assert all( - producers.get(node.input[0]) is None or producers[node.input[0]].op_type != "Cast" - for node in activation_quantizers - ) - - dq_fanouts = [ - sorted(consumer.op_type for consumer in consumers[node.output[0]]) - for node in model.graph.node - if node.op_type.endswith("DequantizeLinear") - ] - assert dq_fanouts.count(["Add", "Conv"]) == 12 - assert dq_fanouts.count(["Conv", "Conv"]) == 4 - assert dq_fanouts.count(["Add"]) == 4 - - gemm = next(node for node in model.graph.node if node.op_type == "Gemm") - assert all( - producers.get(input_name) is None - or not producers[input_name].op_type.endswith("DequantizeLinear") - for input_name in gemm.input - ) - - global_pool = next(node for node in model.graph.node if node.op_type == "GlobalAveragePool") - pool_input_producer = producers[global_pool.input[0]] - if quantize_mode == "int8": - assert pool_input_producer.op_type.endswith("DequantizeLinear") - weight_quantizers = [ - node - for node in model.graph.node - if node.op_type.endswith("QuantizeLinear") and node.input[0] in initializers - ] - assert len(weight_quantizers) == 52 - assert all( - initializers[node.input[0]].data_type == onnx.TensorProto.FLOAT16 - for node in weight_quantizers - ) - else: - assert pool_input_producer.op_type == "Relu" - @pytest.mark.parametrize("quantize_mode", _QUANT_MODES) @pytest.mark.parametrize("model_key", list(_MODELS)) -def test_torch_onnx(tmp_path, model_key, quantize_mode): +def test_torch_onnx(model_key, quantize_mode): timm_model_name, model_kwargs = _MODELS[model_key] - onnx_save_path = tmp_path / f"{model_key}.{quantize_mode}.onnx" + onnx_save_path = f"{model_key}.{quantize_mode}.onnx" cmd_parts = extend_cmd_parts( ["python", "torch_quant_to_onnx.py"], @@ -135,5 +81,5 @@ def test_torch_onnx(tmp_path, model_key, quantize_mode): cmd_parts.extend(["--no_pretrained", "--trt_build"]) run_example_command(cmd_parts, "torch_onnx") - if model_key == "resnet50": - _assert_residual_adds_are_quantized(onnx_save_path, quantize_mode) + if model_key == "resnet50" and quantize_mode in _RESNET_QUANT_MODES: + _assert_residual_inputs_are_quantized(onnx_save_path) diff --git a/tests/unit/onnx/quantization/test_fp8_mha_exporter.py b/tests/unit/onnx/quantization/test_fp8_mha_exporter.py index 7ed6011265b..1f7251a9ad9 100644 --- a/tests/unit/onnx/quantization/test_fp8_mha_exporter.py +++ b/tests/unit/onnx/quantization/test_fp8_mha_exporter.py @@ -40,28 +40,6 @@ def _graph(nodes, inputs, outputs): return gs.Graph(nodes=nodes, inputs=inputs, outputs=outputs, opset=19) -@pytest.mark.parametrize( - ("direct_input", "input_channels", "expected_count"), - [(True, 3, 0), (True, 17, 1), (False, 3, 1)], -) -def test_quantize_conv_weights_to_fp8_skips_unquantized_rgb_graph_input( - direct_input, input_channels, expected_count -): - x = gs.Variable("x", dtype=np.float16, shape=[1, input_channels, 32, 32]) - conv_input = x - nodes = [] - if not direct_input: - conv_input = gs.Variable("conv_input", dtype=np.float16, shape=x.shape) - nodes.append(gs.Node(op="Identity", inputs=[x], outputs=[conv_input])) - - weight = gs.Constant("weight", np.ones((64, input_channels, 3, 3), dtype=np.float16)) - y = gs.Variable("y", dtype=np.float16) - nodes.append(gs.Node(op="Conv", inputs=[conv_input, weight], outputs=[y])) - graph = _graph(nodes, [x], [y]) - - assert FP8QuantExporter._quantize_conv_weights_to_fp8(graph) == expected_count - - def test_move_mul_before_qdq_rewrites_dq_mul_matmul_pattern(): """``DQ → Mul(const) → MatMul`` collapses to ``Mul → Q → DQ → MatMul``.""" x, k, y, mul_out = _var("x"), _var("k"), _var("y"), _var("mul_out") diff --git a/tests/unit/onnx/test_fold_casts.py b/tests/unit/onnx/test_fold_casts.py index 717ee5889b8..59a434d1206 100644 --- a/tests/unit/onnx/test_fold_casts.py +++ b/tests/unit/onnx/test_fold_casts.py @@ -16,15 +16,10 @@ """Tests for the FP16 Q/DQ scale cast-folding helpers in ``modelopt.onnx.utils``.""" import numpy as np -import onnx import pytest from onnx import TensorProto, helper, numpy_helper -from modelopt.onnx.utils import ( - _convert_q_data_initializers_to_fp16, - fold_dq_fp32_to_fp16_casts, - fold_q_fp16_to_fp32_casts, -) +from modelopt.onnx.utils import fold_dq_fp32_to_fp16_casts, fold_q_fp16_to_fp32_casts def _dq_cast_model(opset): @@ -51,7 +46,7 @@ def _dq_cast_model(opset): ) -def _cast_q_model(opset, input_dtype=TensorProto.FLOAT16): +def _cast_q_model(opset): """``Cast(FP16→FP32) → Q → DQ → MatMul`` with FP32 scale.""" nodes = [ helper.make_node("Cast", ["x"], ["c_out"], "cast", to=TensorProto.FLOAT), @@ -68,7 +63,7 @@ def _cast_q_model(opset, input_dtype=TensorProto.FLOAT16): helper.make_graph( nodes, "g", - [helper.make_tensor_value_info("x", input_dtype, [None, 4])], + [helper.make_tensor_value_info("x", TensorProto.FLOAT16, [None, 4])], [helper.make_tensor_value_info("y", TensorProto.FLOAT, [None, 4])], initializer=inits, ), @@ -76,37 +71,6 @@ def _cast_q_model(opset, input_dtype=TensorProto.FLOAT16): ) -def _initializer_q_model(opset, shared=False): - scale_dtype = np.float16 if opset >= 19 else np.float32 - nodes = [ - helper.make_node("QuantizeLinear", ["w", "scale", "zp"], ["q_out"], "q"), - helper.make_node("DequantizeLinear", ["q_out", "scale", "zp"], ["dq_out"], "dq"), - helper.make_node("MatMul", ["x", "dq_out"], ["y"], "matmul"), - ] - outputs = [helper.make_tensor_value_info("y", TensorProto.FLOAT16, [None, 4])] - if shared: - nodes.append(helper.make_node("Identity", ["w"], ["w_out"], "identity")) - outputs.append(helper.make_tensor_value_info("w_out", TensorProto.FLOAT, [4, 4])) - inits = [ - numpy_helper.from_array(np.ones((4, 4), dtype=np.float32), "w"), - numpy_helper.from_array(np.array(0.1, dtype=scale_dtype), "scale"), - numpy_helper.from_array(np.array(0, dtype=np.int8), "zp"), - ] - input_dtype = TensorProto.FLOAT16 if opset >= 19 else TensorProto.FLOAT - output_dtype = TensorProto.FLOAT16 if opset >= 19 else TensorProto.FLOAT - outputs[0].type.tensor_type.elem_type = output_dtype - return helper.make_model( - helper.make_graph( - nodes, - "g", - [helper.make_tensor_value_info("x", input_dtype, [None, 4])], - outputs, - initializer=inits, - ), - opset_imports=[helper.make_opsetid("", opset)], - ) - - @pytest.mark.parametrize( ("fold_fn", "build_model", "scale_name"), [ @@ -133,61 +97,3 @@ def test_fold_is_noop_below_min_opset(fold_fn, build_model, scale_name): assert "Cast" in {n.op_type for n in folded.graph.node} scale = next(i for i in folded.graph.initializer if i.name == scale_name) assert scale.data_type == TensorProto.FLOAT - - -def test_fold_q_preserves_cast_with_non_fp16_source(): - model = _cast_q_model(opset=19, input_dtype=TensorProto.FLOAT) - folded = fold_q_fp16_to_fp32_casts(model) - assert "Cast" in {node.op_type for node in folded.graph.node} - scale = next( - initializer for initializer in folded.graph.initializer if initializer.name == "scale" - ) - assert scale.data_type == TensorProto.FLOAT - onnx.checker.check_model(folded) - - -@pytest.mark.parametrize("shared", [False, True]) -def test_convert_q_data_initializers_to_fp16(shared): - converted = _convert_q_data_initializers_to_fp16(_initializer_q_model(opset=19, shared=shared)) - initializers = {initializer.name: initializer for initializer in converted.graph.initializer} - q_node = next(node for node in converted.graph.node if node.op_type == "QuantizeLinear") - - assert initializers[q_node.input[0]].data_type == TensorProto.FLOAT16 - assert initializers[q_node.input[1]].data_type == TensorProto.FLOAT16 - if shared: - identity = next(node for node in converted.graph.node if node.op_type == "Identity") - assert identity.input[0] == "w" - assert initializers["w"].data_type == TensorProto.FLOAT - else: - assert q_node.input[0] == "w" - onnx.checker.check_model(converted) - - -def test_convert_q_data_initializers_is_noop_below_min_opset(): - model = _initializer_q_model(opset=18) - converted = _convert_q_data_initializers_to_fp16(model) - weight = next( - initializer for initializer in converted.graph.initializer if initializer.name == "w" - ) - assert weight.data_type == TensorProto.FLOAT - - -def test_convert_q_data_initializers_requires_fp16_scale(): - model = _initializer_q_model(opset=19) - scale = next( - initializer for initializer in model.graph.initializer if initializer.name == "scale" - ) - scale.CopyFrom(numpy_helper.from_array(np.array(0.1, dtype=np.float32), "scale")) - with pytest.raises(ValueError, match="Q scales must be FP16"): - _convert_q_data_initializers_to_fp16(model) - - -def test_convert_q_data_initializers_rejects_fp32_graph_input_scale(): - model = _initializer_q_model(opset=19) - scale = next( - initializer for initializer in model.graph.initializer if initializer.name == "scale" - ) - model.graph.initializer.remove(scale) - model.graph.input.append(helper.make_tensor_value_info("scale", TensorProto.FLOAT, [])) - with pytest.raises(ValueError, match="Q scales must be FP16"): - _convert_q_data_initializers_to_fp16(model) diff --git a/tests/unit/torch/nas/test_registry.py b/tests/unit/torch/nas/test_registry.py index b072bb30a00..a425abcf29c 100644 --- a/tests/unit/torch/nas/test_registry.py +++ b/tests/unit/torch/nas/test_registry.py @@ -67,9 +67,6 @@ def _test_new_cls(cls, name): assert DMRegistry2.get_key_from_dm(DMRegistry2[nn.Linear]) == "nn.Linear" assert DMRegistry2.get_key_from_dm(DMRegistry2[Linear]) == "nn.Linear" assert DMRegistry2.get_key_from_dm(DMRegistry2[Linear2]) == "nn.Linear" - assert DMRegistry2.get_registered_class("nn.Linear") is nn.Linear - with pytest.raises(KeyError): - DMRegistry2.get_registered_class("missing") # unregister nn.Linear (only works with truly registered class) with pytest.raises(KeyError): diff --git a/tests/unit/torch/quantization/plugins/test_timm.py b/tests/unit/torch/quantization/plugins/test_timm.py deleted file mode 100644 index 9a03fcc2bc1..00000000000 --- a/tests/unit/torch/quantization/plugins/test_timm.py +++ /dev/null @@ -1,469 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -from copy import deepcopy - -import pytest -import torch - -timm = pytest.importorskip("timm") - -import modelopt.torch.opt as mto -import modelopt.torch.quantization as mtq -from modelopt.recipe import load_recipe -from modelopt.torch.quantization.algorithms import ( - AutoQuantizeGradientSearcher, - QuantRecipe, - QuantRecipeHparam, -) -from modelopt.torch.quantization.plugins.timm import is_resnet_quantization_supported - - -def _get_resnet(block=timm.models.resnet.BasicBlock): - return timm.models.resnet.ResNet( - block=block, - layers=[2, 1, 1, 1], - num_classes=8, - stem_width=8, - channels=(8, 16, 32, 64), - ).eval() - - -def _get_output_conv(block): - return block.conv3 if isinstance(block, timm.models.resnet.Bottleneck) else block.conv2 - - -class _UnsupportedBottleneck(timm.models.resnet.Bottleneck): - def forward(self, input): - return super().forward(input) - - -def test_resnet_recipe_support_is_limited_to_standard_blocks(): - model = timm.models.resnet.ResNet( - block=_UnsupportedBottleneck, - layers=[1, 1, 1, 1], - num_classes=8, - stem_width=8, - channels=(8, 16, 32, 64), - ).eval() - - assert not is_resnet_quantization_supported(model) - model = mtq.quantize(model, {**deepcopy(mtq.INT8_DEFAULT_CFG), "algorithm": None}) - assert not hasattr(_get_output_conv(model.layer1[0]), "block_input_activation_quantizer") - - -@pytest.mark.parametrize( - ("recipe_name", "conv_num_bits", "conv_weight_axis", "fc_num_bits", "fc_block_size"), - [ - ("fp8", (4, 3), None, (4, 3), None), - ("int8", 8, 0, 8, None), - ("mxfp8", (4, 3), None, (4, 3), 32), - ("nvfp4", (4, 3), None, (2, 1), 16), - ("nvfp4_awq_lite", (4, 3), None, (2, 1), 16), - ], -) -@pytest.mark.parametrize( - "block_type", [timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck] -) -def test_resnet_recipe_quantizer_choices( - recipe_name, conv_num_bits, conv_weight_axis, fc_num_bits, fc_block_size, block_type -): - recipe = load_recipe(f"timm/resnet/ptq/{recipe_name}") - config = recipe.quantize.model_dump() - config["algorithm"] = None - model = mtq.quantize(_get_resnet(block_type), config) - - assert is_resnet_quantization_supported(model) - assert not model.conv1.input_quantizer.is_enabled - assert not model.conv1.weight_quantizer.is_enabled - assert not model.maxpool.input_quantizer.is_enabled - - for block in (model.layer1[1], model.layer2[0]): - output_conv = _get_output_conv(block) - assert output_conv.block_input_activation_quantizer.is_enabled - assert output_conv.block_input_activation_quantizer.num_bits == conv_num_bits - assert output_conv.block_input_activation_quantizer.axis is None - assert output_conv.block_input_activation_quantizer.block_sizes is None - assert output_conv.input_quantizer.num_bits == conv_num_bits - assert output_conv.weight_quantizer.num_bits == conv_num_bits - assert output_conv.weight_quantizer.axis == conv_weight_axis - assert not output_conv.output_quantizer.is_enabled - assert not block.conv1.input_quantizer.is_enabled - - projection_block = model.layer2[0] - assert not projection_block.downsample[0].input_quantizer.is_enabled - assert _get_output_conv(projection_block).residual_quantizer.is_enabled - assert _get_output_conv(projection_block).residual_quantizer.num_bits == conv_num_bits - assert not hasattr(_get_output_conv(model.layer1[1]), "residual_quantizer") - - pool_input_quantizers = [ - module - for name, module in model.named_modules() - if name.startswith("global_pool.") and name.endswith("input_quantizer") - ] - assert not any(quantizer.is_enabled for quantizer in pool_input_quantizers) - - final_output_conv = _get_output_conv(model.layer4[-1]) - assert final_output_conv.model_output_activation_quantizer.is_enabled == (recipe_name == "int8") - if recipe_name == "int8": - assert final_output_conv.model_output_activation_quantizer.num_bits == 8 - - assert model.fc.weight_quantizer.num_bits == fc_num_bits - if fc_block_size is None: - assert model.fc.weight_quantizer.block_sizes is None - else: - assert model.fc.weight_quantizer.block_sizes[-1] == fc_block_size - if recipe_name in ("fp8", "int8"): - assert not model.fc.weight_quantizer.is_enabled - assert not model.fc.input_quantizer.is_enabled - else: - assert model.fc.weight_quantizer.is_enabled - assert model.fc.input_quantizer.is_enabled - - -@pytest.mark.parametrize( - "config_name", - ["FP8_DEFAULT_CFG", "INT8_DEFAULT_CFG", "MXFP8_DEFAULT_CFG", "NVFP4_DEFAULT_CFG"], -) -def test_stock_configs_do_not_enable_resnet_residual_quantizers(config_name): - config = deepcopy(getattr(mtq, config_name)) - config["algorithm"] = None - model = mtq.quantize(_get_resnet(), config) - - for block in (model.layer1[1], model.layer2[0]): - output_conv = _get_output_conv(block) - assert not output_conv.block_input_activation_quantizer.is_enabled - if block.downsample is not None: - assert not output_conv.residual_quantizer.is_enabled - assert block.conv1.input_quantizer.is_enabled - assert model.layer2[0].downsample[0].input_quantizer.is_enabled - assert not _get_output_conv(model.layer4[-1]).model_output_activation_quantizer.is_enabled - - -def test_parent_conv_rule_matches_resnet_conv_subclasses(): - model = mtq.quantize( - _get_resnet(), - { - "quant_cfg": [ - {"quantizer_name": "*", "enable": False}, - { - "parent_class": "nn.Conv2d", - "quantizer_name": "*weight_quantizer", - "cfg": {"num_bits": 7}, - }, - ], - "algorithm": None, - }, - ) - - for conv in ( - model.conv1, - model.layer1[0].conv1, - _get_output_conv(model.layer1[0]), - model.layer2[0].downsample[0], - _get_output_conv(model.layer2[0]), - ): - assert conv.weight_quantizer.is_enabled - assert conv.weight_quantizer.num_bits == 7 - - -def test_resnet_recipe_disables_only_first_deep_stem_convolution(): - model = timm.models.resnet.ResNet( - block=timm.models.resnet.Bottleneck, - layers=[1, 1, 1, 1], - num_classes=8, - stem_width=8, - stem_type="deep", - channels=(8, 16, 32, 64), - ).eval() - recipe = load_recipe("timm/resnet/ptq/int8") - config = recipe.quantize.model_dump() - config["algorithm"] = None - model = mtq.quantize(model, config) - - stem_convs = [module for module in model.conv1.modules() if hasattr(module, "weight_quantizer")] - assert len(stem_convs) == 3 - assert not stem_convs[0].input_quantizer.is_enabled - assert not stem_convs[0].weight_quantizer.is_enabled - assert all(conv.input_quantizer.is_enabled for conv in stem_convs[1:]) - assert all(conv.weight_quantizer.is_enabled for conv in stem_convs[1:]) - - -def test_resnet_recipe_quantizes_replacement_stem_pool_convolution(): - model = timm.models.resnet.ResNet( - block=timm.models.resnet.Bottleneck, - layers=[1, 1, 1, 1], - num_classes=8, - stem_width=8, - stem_type="deep", - replace_stem_pool=True, - channels=(8, 16, 32, 64), - ).eval() - recipe = load_recipe("timm/resnet/ptq/int8") - config = recipe.quantize.model_dump() - config["algorithm"] = None - model = mtq.quantize(model, config) - - assert model.maxpool[0].input_quantizer.is_enabled - assert model.maxpool[0].weight_quantizer.is_enabled - - -def test_resnet_recipe_handles_avg_down_and_antialias_pools(): - model = timm.models.resnet.ResNet( - block=timm.models.resnet.Bottleneck, - layers=[1, 1, 1, 1], - num_classes=8, - stem_width=8, - avg_down=True, - aa_layer=torch.nn.AvgPool2d, - channels=(8, 16, 32, 64), - ).eval() - recipe = load_recipe("timm/resnet/ptq/mxfp8") - config = recipe.quantize.model_dump() - config["algorithm"] = None - model = mtq.quantize(model, config) - - block = model.layer2[0] - assert not block.aa.input_quantizer.is_enabled - assert not block.downsample[0].input_quantizer.is_enabled - assert block.downsample[1].input_quantizer.is_enabled - assert block.downsample[1].input_quantizer.num_bits == (4, 3) - - -@pytest.mark.parametrize( - "block_type", [timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck] -) -def test_resnet_recipe_calibrates_residual_quantizers_in_one_pass(block_type): - model = _get_resnet(block_type) - recipe = load_recipe("timm/resnet/ptq/int8") - calibration_calls = 0 - - def forward_loop(quantized_model): - nonlocal calibration_calls - calibration_calls += 1 - quantized_model(torch.randn(2, 3, 32, 32)) - - model = mtq.quantize(model, recipe.quantize.model_dump(), forward_loop=forward_loop) - - assert calibration_calls == 1 - for block in (model.layer1[1], model.layer2[0]): - quantizers = [_get_output_conv(block).block_input_activation_quantizer] - if block.downsample is not None: - quantizers.append(_get_output_conv(block).residual_quantizer) - for quantizer in quantizers: - assert quantizer.amax is not None - assert torch.isfinite(quantizer.amax).all() - assert torch.all(quantizer.amax > 0) - - model_output_quantizer = _get_output_conv(model.layer4[-1]).model_output_activation_quantizer - assert model_output_quantizer.amax is not None - assert torch.isfinite(model_output_quantizer.amax).all() - assert torch.all(model_output_quantizer.amax > 0) - - -def test_auto_quantize_uses_resnet_conv_format_for_cost_and_replay(): - recipe = load_recipe("timm/resnet/ptq/nvfp4") - config = recipe.quantize.model_dump() - config["algorithm"] = None - model = mtq.quantize(_get_resnet(), config) - block = model.layer2[0] - output_conv = _get_output_conv(block) - quant_recipe = QuantRecipe(recipe.quantize.model_dump(), name="resnet_nvfp4") - hparam = QuantRecipeHparam( - [quant_recipe], - quant_modules=[output_conv], - score_modules=[block], - quant_module_names=["layer2.0.conv2"], - ) - - assert hparam.quant_module_replay_attrs["layer2.0.conv2"] == ( - "input_quantizer", - "weight_quantizer", - "output_quantizer", - "block_input_activation_quantizer", - "residual_quantizer", - ) - assert hparam.quant_module_parent_classes["layer2.0.conv2"] == ( - "timm.ResNetProjectionOutputConv2d" - ) - quantizer_choice = hparam._all_quantizer_choices[quant_recipe][output_conv] - for quantizer_name in ( - "input_quantizer", - "weight_quantizer", - "block_input_activation_quantizer", - "residual_quantizer", - ): - assert quantizer_choice[quantizer_name].num_bits == (4, 3) - assert hparam.get_cost(quant_recipe) == pytest.approx(output_conv.weight.numel() * 0.5) - - hparam_name = "layer2.0.conv2.quant_recipe" - search_state = { - "best": {"recipe": {hparam_name: quant_recipe}}, - "candidate_stats": { - hparam_name: { - "module_names": hparam.quant_module_names, - "quantizer_attrs": hparam.quant_module_replay_attrs, - "parent_classes": hparam.quant_module_parent_classes, - } - }, - } - with pytest.warns(UserWarning, match="algorithm='max'"): - replay_config = mtq.get_auto_quantize_config(search_state) - entries = {entry["quantizer_name"]: entry for entry in replay_config["quant_cfg"]} - for quantizer_name in hparam.quant_module_replay_attrs["layer2.0.conv2"]: - entry = entries[f"layer2.0.conv2.{quantizer_name}"] - if quantizer_name == "output_quantizer": - assert not entry["enable"] - else: - assert entry["enable"] - assert entry["cfg"]["num_bits"] == (4, 3) - - assert AutoQuantizeGradientSearcher._is_auto_quantize_module(output_conv) - assert not AutoQuantizeGradientSearcher._is_auto_quantize_module(model.conv1) - - -def test_auto_quantize_replay_keeps_resnet_block_input_convs_disabled(): - recipe = load_recipe("timm/resnet/ptq/int8") - config = recipe.quantize.model_dump() - config["algorithm"] = None - model = mtq.quantize(_get_resnet(), config) - conv = model.layer2[0].conv1 - quant_recipe = QuantRecipe(recipe.quantize.model_dump(), name="resnet_int8") - hparam = QuantRecipeHparam( - [quant_recipe], - quant_modules=[conv], - quant_module_names=["layer2.0.conv1"], - ) - hparam_name = "layer2.0.conv1.quant_recipe" - search_state = { - "best": {"recipe": {hparam_name: quant_recipe}}, - "candidate_stats": { - hparam_name: { - "module_names": hparam.quant_module_names, - "quantizer_attrs": hparam.quant_module_replay_attrs, - "parent_classes": hparam.quant_module_parent_classes, - } - }, - } - - with pytest.warns(UserWarning, match="algorithm='max'"): - replay_config = mtq.get_auto_quantize_config(search_state) - entries = {entry["quantizer_name"]: entry for entry in replay_config["quant_cfg"]} - - assert not entries["layer2.0.conv1.input_quantizer"]["enable"] - - -@pytest.mark.parametrize( - "block_type", [timm.models.resnet.BasicBlock, timm.models.resnet.Bottleneck] -) -def test_auto_quantize_calibrates_and_scores_resnet_residual_quantizers(block_type): - recipes = [load_recipe(f"timm/resnet/ptq/{name}") for name in ("int8", "fp8")] - data = [{"image": torch.randn(1, 3, 32, 32), "label": torch.tensor([1])}] - - model, search_state = mtq.auto_quantize( - _get_resnet(block_type), - constraints={"effective_bits": 8.0}, - quantization_formats=[recipe.quantize.model_dump() for recipe in recipes], - data_loader=data, - forward_step=lambda model, batch: model(batch["image"]), - loss_func=lambda output, batch: torch.nn.functional.cross_entropy(output, batch["label"]), - num_calib_steps=1, - num_score_steps=1, - ) - - block = model.layer2[0] - output_conv = _get_output_conv(block) - for quantizer in ( - output_conv.block_input_activation_quantizer, - output_conv.residual_quantizer, - ): - assert quantizer.is_enabled - assert quantizer.amax is not None - assert torch.isfinite(quantizer.amax).all() - assert torch.all(quantizer.amax > 0) - - hparam = output_conv.get_hparam("quant_recipe") - assert hparam.score_modules == [block] - block_convs = [block.conv1, block.conv2, block.downsample[0]] - if isinstance(block, timm.models.resnet.Bottleneck): - block_convs.append(block.conv3) - assert all(conv.get_hparam("quant_recipe") is hparam for conv in block_convs) - assert all( - conv.weight_quantizer.num_bits == output_conv.block_input_activation_quantizer.num_bits - for conv in block_convs - ) - output_conv_name = ( - "layer2.0.conv3" if isinstance(block, timm.models.resnet.Bottleneck) else "layer2.0.conv2" - ) - candidate_stat = next( - stat - for stat in search_state["candidate_stats"].values() - if output_conv_name in stat["module_names"] - ) - assert candidate_stat["quantizer_attrs"][output_conv_name][-2:] == ( - "block_input_activation_quantizer", - "residual_quantizer", - ) - assert candidate_stat["parent_classes"][output_conv_name] == ( - "timm.ResNetProjectionOutputConv2d" - ) - - final_block = model.layer4[-1] - final_output_conv = _get_output_conv(final_block) - final_hparam = final_output_conv.get_hparam("quant_recipe") - assert final_hparam.score_modules == [model.global_pool] - assert final_output_conv.model_output_activation_quantizer.is_enabled == ( - final_output_conv.block_input_activation_quantizer.num_bits == 8 - ) - for quantizer_choices in final_hparam._all_quantizer_choices.values(): - quantizer = quantizer_choices[final_output_conv]["model_output_activation_quantizer"] - if quantizer.is_enabled: - assert quantizer.amax is not None - assert torch.isfinite(quantizer.amax).all() - assert torch.all(quantizer.amax > 0) - - -def test_resnet_quantizer_hooks_survive_save_restore(): - inputs = torch.randn(1, 3, 32, 32) - recipe = load_recipe("timm/resnet/ptq/int8") - model = mtq.quantize( - _get_resnet(), - recipe.quantize.model_dump(), - forward_loop=lambda quantized_model: quantized_model(inputs), - ) - expected = model(inputs) - - buffer = io.BytesIO() - mto.save(model, buffer) - buffer.seek(0) - restored = mto.restore(_get_resnet(), buffer).eval() - - block = restored.layer2[0] - output_conv = _get_output_conv(block) - counts = {"block_input": 0, "projection": 0} - - def count_call(name): - def hook(_module, _inputs, _output): - counts[name] += 1 - - return hook - - output_conv.block_input_activation_quantizer.register_forward_hook(count_call("block_input")) - output_conv.residual_quantizer.register_forward_hook(count_call("projection")) - - assert torch.equal(restored(inputs), expected) - assert counts == {"block_input": 1, "projection": 1} - assert len(block.downsample._forward_hooks) == 1 diff --git a/tests/unit/torch/quantization/test_autoquant.py b/tests/unit/torch/quantization/test_autoquant.py index 24e3daa4329..e83f7fa0a70 100644 --- a/tests/unit/torch/quantization/test_autoquant.py +++ b/tests/unit/torch/quantization/test_autoquant.py @@ -36,17 +36,10 @@ QuantRecipe, QuantRecipeHparam, _AutoQuantizeBaseSearcher, - _get_quantizer_attrs, _module_search_space_signature, - _module_weight_compression, estimate_quant_compression, ) -from modelopt.torch.quantization.config import ( - QuantizerAttributeConfig, - _base_disable_all, - _default_disabled_quantizer_cfg, -) -from modelopt.torch.quantization.nn import TensorQuantizer +from modelopt.torch.quantization.config import _base_disable_all, _default_disabled_quantizer_cfg from modelopt.torch.utils import safe_load, safe_save from modelopt.torch.utils.distributed import DistributedProcessGroup @@ -211,40 +204,6 @@ def test_quant_recipe_hparam_zero_cost_weight(): assert hparam.get_cost(QuantRecipe(mtq.INT8_DEFAULT_CFG)) == pytest.approx(0.0) -def test_custom_auto_quantize_attrs_are_explicit(): - module = torch.nn.Module() - module.custom_quantizer = TensorQuantizer() - module._auto_quantize_quantizer_attrs = () - assert _get_quantizer_attrs(module) == () - - module._auto_quantize_quantizer_attrs = ("custom_quantizer",) - assert _get_quantizer_attrs(module) == ("custom_quantizer",) - - module._auto_quantize_quantizer_attrs = ("missing_quantizer",) - with pytest.raises(AttributeError, match="missing_quantizer"): - _get_quantizer_attrs(module) - - -def test_candidate_cost_rejects_mixed_weight_formats(): - quantizers = { - "first_weight_quantizer": TensorQuantizer(QuantizerAttributeConfig(num_bits=8)), - "second_weight_quantizer": TensorQuantizer(QuantizerAttributeConfig(num_bits=4)), - } - with pytest.raises(ValueError, match="different weight formats"): - _module_weight_compression(quantizers) - - -def test_candidate_cost_supports_sequential_weight_quantization(): - recipe = QuantRecipe(mtq.W4A8_AWQ_BETA_CFG) - model = mtq.quantize( - torch.nn.Linear(4, 16), - {"quant_cfg": [{"quantizer_name": "*", "enable": False}], "algorithm": None}, - ) - hparam = QuantRecipeHparam([recipe], quant_modules=[model]) - - assert hparam.get_cost(recipe) == pytest.approx(model.weight.numel() * 0.25) - - def test_quant_recipe_hparam_cost_weight_and_effective_bits_compose(): """cost_weight (active_moe) and effective_bits stack multiplicatively in get_cost.""" model_test = mtq.quantize(torch.nn.Linear(4, 16), mtq.NVFP4_DEFAULT_CFG) diff --git a/tests/unit/torch/quantization/test_config_validation.py b/tests/unit/torch/quantization/test_config_validation.py index c8f71c9adba..4b969d3259c 100644 --- a/tests/unit/torch/quantization/test_config_validation.py +++ b/tests/unit/torch/quantization/test_config_validation.py @@ -463,13 +463,13 @@ def test_star_matches_any_bare_name(self): assert matched is None # enable-only entry has cfg=None assert enable is False - def test_path_scoped_pattern_does_not_match_bare_name(self): + def test_path_scoped_pattern_matches_matching_suffix(self): + """'*mlp*weight_quantizer' matches bare 'weight_quantizer' (suffix match).""" quant_cfg = normalize_quant_cfg_list( [{"quantizer_name": "*mlp*weight_quantizer", "cfg": {"num_bits": 4}}] ) matched, enable = _match_quantizer_cfg(quant_cfg, "weight_quantizer") - assert matched is None - assert enable is None + assert matched.model_dump(exclude_unset=True) == {"num_bits": 4} def test_path_scoped_pattern_does_not_match_different_suffix(self): """'*mlp*weight_quantizer' does NOT match bare 'input_quantizer'.""" @@ -507,25 +507,6 @@ def test_parent_class_scoped_entries_are_ignored_for_bare_autoquant_lookup(self) assert matched.model_dump(exclude_unset=True) == {"num_bits": 8} assert enable is True - def test_parent_class_scoped_entry_matches_persisted_autoquant_parent(self): - quant_cfg = normalize_quant_cfg_list( - [ - {"quantizer_name": "*weight_quantizer", "cfg": {"num_bits": 4}}, - { - "parent_class": "nn.Conv2d", - "quantizer_name": "*weight_quantizer", - "cfg": {"num_bits": 8}, - }, - ] - ) - - matched, enable = _match_quantizer_cfg( - quant_cfg, "weight_quantizer", module_parent_class="nn.Conv2d" - ) - - assert matched.model_dump(exclude_unset=True) == {"num_bits": 8} - assert enable is True - def test_no_match_returns_none(self): """No matching entry returns (None, None).""" quant_cfg = normalize_quant_cfg_list( diff --git a/tests/unit/torch/quantization/test_quantize_cpu.py b/tests/unit/torch/quantization/test_quantize_cpu.py index 433f4af150a..3e4925e7b63 100644 --- a/tests/unit/torch/quantization/test_quantize_cpu.py +++ b/tests/unit/torch/quantization/test_quantize_cpu.py @@ -84,35 +84,6 @@ } -class _LinearSubclass(torch.nn.Linear): - pass - - -def test_parent_class_config_matches_registered_subclass(): - model = mtq.quantize( - _LinearSubclass(4, 4), - { - "quant_cfg": [ - {"quantizer_name": "*", "enable": False}, - { - "parent_class": "nn.Linear", - "quantizer_name": "*weight_quantizer", - "cfg": {"num_bits": 7}, - }, - { - "parent_class": "nn.Conv2d", - "quantizer_name": "*weight_quantizer", - "cfg": {"num_bits": 3}, - }, - ], - "algorithm": None, - }, - ) - - assert model.weight_quantizer.is_enabled - assert model.weight_quantizer.num_bits == 7 - - class NewMaxCalibrator(MaxCalibrator): def compute_amax(self): return 2 * self._calib_amax