Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions backends/nxp/tests/generic_tests/test_aot_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,3 +334,77 @@ def test_aot_example__mlperf_tiny_kws__profiling():
with _cleanup_generated_files(pte_file, etrecord_file):
result = _run_compile(cmd)
_assert_profiling(result, pte_file, etrecord_file)


def test_aot_example__mlperf_tiny_ad():
"""Test that the MLPerf Tiny Anomaly detection (DeepAutoEncoder) can be lowered to Neutron backend via
`aot_neutron_compile.py` and all ops are delegated."""

# Number of random samples to generate, must be divisible by number of classes
num_random_samples = 60

# Run the compilation script as a module (like run_aot_example.sh does).
cmd = [
sys.executable,
"-m",
"examples.nxp.aot_neutron_compile",
"--model_name",
"mlperf_tiny_anomaly_detection",
"--delegate",
"--quantize",
"--target",
"imxrt700",
"--use_random_dataset",
"--num_random_samples",
str(num_random_samples),
]

# Output file will be created in executorch_root
pte_file = Path(
os.path.join(EXECUTORCH_ROOT, "mlperf_tiny_anomaly_detection_nxp_delegate.pte")
)

with _cleanup_generated_files(pte_file):
result = _run_compile(cmd)
_assert_delegation(result, pte_file)


def test_aot_example__mlperf_tiny_ad__profiling():
"""Test that the MLPerf Tiny Anomaly detection (DeepAutoEncoder) can be lowered to Neutron backend via
`aot_neutron_compile.py` and profiling works as intended."""

# Number of random samples to generate, must be divisible by number of classes
num_random_samples = 60

# Run the compilation script as a module (like run_aot_example.sh does)
cmd = [
sys.executable,
"-m",
"examples.nxp.aot_neutron_compile",
"--model_name",
"mlperf_tiny_anomaly_detection",
"--delegate",
"--quantize",
"--target",
"imxrt700",
"--remove-quant-io-ops",
"--use_profiling", # Generate profilable model and create ETRecord
"--use_random_dataset",
"--num_random_samples",
str(num_random_samples),
]

pte_file = Path(
os.path.join(
EXECUTORCH_ROOT, "mlperf_tiny_anomaly_detection_nxp_delegate_profile.pte"
)
)
etrecord_file = Path(
os.path.join(
EXECUTORCH_ROOT, "etrecord", "mlperf_tiny_anomaly_detection_etrecord.bin"
)
)

with _cleanup_generated_files(pte_file, etrecord_file):
result = _run_compile(cmd)
_assert_profiling(result, pte_file, etrecord_file)
127 changes: 127 additions & 0 deletions backends/nxp/tests/models/test_mlperf_tiny_anomaly_detection.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
# Copyright 2026 NXP
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

import os
from functools import partial

import numpy as np

# noinspection PyUnusedImports
import pytest
import torch

from executorch.backends.nxp.tests.dataset_creator import (
FromCalibrationDataDatasetCreator,
)
from executorch.backends.nxp.tests.executorch_pipeline import ModelInputSpec
from executorch.backends.nxp.tests.graph_verifier import BaseGraphVerifier
from executorch.backends.nxp.tests.model_output_comparator import (
ClassificationAccuracyOutputComparator,
NumericalStatsOutputComparator,
)
from executorch.backends.nxp.tests.nsys_testing import (
get_test_name,
lower_run_compare,
lower_run_compare_ptq_qat,
OUTPUTS_DIR,
)
from executorch.backends.nxp.tests.use_qat import * # noqa F403
from executorch.examples.nxp.models.mlperf_tiny.anomaly_detection.mlperf_tiny_anomaly_detection import (
MLPerfTinyAnomalyDetection,
)

BOUNDS_MSE = {
"PTQ": 1.4e-08,
"QAT": 5.205e-06,
}


@pytest.fixture(autouse=True)
def reseed_model_per_test_run():
torch.manual_seed(23)
np.random.seed(23)


def test_mlperf_tiny_anomaly_detection_mse_cpu_vs_npu(
mocker,
request,
use_qat,
):
num_samples = 60

anomaly_detection = MLPerfTinyAnomalyDetection(
num_samples=num_samples, use_random_dataset=True
)
model = anomaly_detection.get_eager_model()
dataset = anomaly_detection.dataset
labels = anomaly_detection.labels

dataset_creator = FromCalibrationDataDatasetCreator(
dataset, num_examples=num_samples, idx_to_label=labels
)

input_spec = ModelInputSpec(anomaly_detection.input_shape)
quant_type_key = "QAT" if use_qat else "PTQ"

mse = BOUNDS_MSE[quant_type_key]
comparator = NumericalStatsOutputComparator(max_mse_error=mse)
Comment thread
roman-janik-nxp marked this conversation as resolved.
model_verifier = BaseGraphVerifier(1, [])
train_fn = anomaly_detection.train_model_fn if use_qat else None

lower_run_compare(
model,
[input_spec],
model_verifier,
request,
dataset_creator=dataset_creator,
output_comparator=comparator,
mocker=mocker,
use_qat=use_qat,
train_fn=train_fn,
)


def test_mlperf_tiny_anomaly_detection_ptq_qat_equivalence(request):
num_samples = 60

anomaly_detection = MLPerfTinyAnomalyDetection(
num_samples=num_samples, use_random_dataset=True
)

model = anomaly_detection.get_eager_model()
dataset = anomaly_detection.dataset
labels = anomaly_detection.labels

dataset_creator = FromCalibrationDataDatasetCreator(
dataset, num_examples=num_samples, idx_to_label=labels
)

test_name = get_test_name(request)
input_parent_path = os.path.join(
OUTPUTS_DIR,
test_name,
"dataset/calibration/",
)

comparator = ClassificationAccuracyOutputComparator(
class_dict=labels,
postprocess_fn=partial(
anomaly_detection.get_class_from_reconstruction_error,
input_parent_path=input_parent_path,
),
)

input_spec = ModelInputSpec(anomaly_detection.input_shape)
model_verifier = BaseGraphVerifier(1, [])

lower_run_compare_ptq_qat(
model,
[input_spec],
model_verifier,
request,
train_fn=anomaly_detection.train_model_fn,
dataset_creator=dataset_creator,
output_comparator=comparator,
)
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
from functools import partial

import numpy as np

# noinspection PyUnusedImports
import pytest
import torch
from executorch.backends.nxp.tests.dataset_creator import (
Expand Down
17 changes: 15 additions & 2 deletions examples/nxp/aot_neutron_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@
train_cifarnet_model,
verify_cifarnet_model,
)
from executorch.examples.nxp.models.mlperf_tiny.anomaly_detection.mlperf_tiny_anomaly_detection import (
MLPerfTinyAnomalyDetection,
)
from executorch.examples.nxp.models.mlperf_tiny.image_classification.mlperf_tiny_image_classification import (
MLPerfTinyImageClassification,
)
Expand All @@ -67,6 +70,7 @@
MODELS = {
"cifar10": CifarNet,
"mobilenetv2": MobilenetV2,
"mlperf_tiny_anomaly_detection": MLPerfTinyAnomalyDetection,
"mlperf_tiny_image_classification": MLPerfTinyImageClassification,
"mlperf_tiny_keyword_spotting": MLPerfTinyKeywordSpotting,
}
Expand Down Expand Up @@ -126,7 +130,11 @@ def _get_model_info_from_name(
)
model_cls_inst = model_cls()

elif model_cls in (MLPerfTinyImageClassification, MLPerfTinyKeywordSpotting):
elif model_cls in (
MLPerfTinyImageClassification,
MLPerfTinyKeywordSpotting,
MLPerfTinyAnomalyDetection,
):
model_cls_inst = model_cls(
dataset_path=dataset_path,
use_random_dataset=use_random_dataset,
Expand Down Expand Up @@ -337,7 +345,12 @@ def _get_arg_parser():
if args.use_qat:
if not isinstance(
model_cls_inst,
(CifarNet, MLPerfTinyImageClassification, MLPerfTinyKeywordSpotting),
(
CifarNet,
MLPerfTinyImageClassification,
MLPerfTinyKeywordSpotting,
MLPerfTinyAnomalyDetection,
),
):
raise ValueError(
f"QAT training is not supported for model '{args.model_name}'"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Copyright 2026 NXP
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
Loading
Loading