frontend.py Source File

frontend.py Source File#

Mobilint SDK qb Compiler: frontend.py Source File
Mobilint SDK qb Compiler v1.1
MCS002-EN
 
import os
import pathlib
import warnings
from pathlib import Path
from typing import Any, List, Optional, Union
 
import torch
 
from qbcompiler.compiler.compiler import Compiler
from qbcompiler.compiler.compiler import _quantization_task
from qbcompiler.configs import (
    CompileConfig,
    CalibrationConfig,
    BitConfig,
    ResourceManagementConfig,
    OptqConfig,
    ModConfig,
    LlmConfig,
    EquivalentTransformationConfig,
    SearchWeightScaleConfig,
    SaveSampleConfig,
    Uint8InputConfig,
    PreprocessingConfig,
)
from qbcompiler.compiler.utils import _to_list
from qbcompiler.version import __version__
 
_UNSET = object()
"""Sentinel to distinguish 'not passed' from explicit defaults."""
 
 
def _resolve_deprecated_kwargs(
    *,
    calibration_config,
    optq_config,
    mod_config,
    equivalent_transformation_config,
    search_weight_scale_config,
    uint8_input_config,
    preprocessing_config,
    save_sample_config,
    **kwargs,
):
    """Extract and resolve deprecated parameters from kwargs into modern equivalents.
 
    Mutates kwargs by popping consumed deprecated keys.
    Returns a tuple of resolved sub-config values.
    """
    quantization_config = kwargs.pop("quantization_config", None)
    if quantization_config is not None and calibration_config is None:
        warnings.warn(
            "quantization_config is deprecated. Use calibration_config instead.",
            DeprecationWarning,
            stacklevel=3,
        )
        calibration_config = quantization_config
 
    advanced_quantization_config = kwargs.pop("advanced_quantization_config", None)
    if advanced_quantization_config is not None:
        warnings.warn(
            "advanced_quantization_config is deprecated. Use individual configs "
            "(optq_config, mod_config, equivalent_transformation_config, search_weight_scale_config) instead.",
            DeprecationWarning,
            stacklevel=3,
        )
        if hasattr(advanced_quantization_config, "optq") and optq_config is None:
            optq_config = advanced_quantization_config.optq
        if hasattr(advanced_quantization_config, "mod") and mod_config is None:
            mod_config = advanced_quantization_config.mod
        if (
            hasattr(advanced_quantization_config, "equivalent_transformation")
            and equivalent_transformation_config is None
        ):
            equivalent_transformation_config = (
                advanced_quantization_config.equivalent_transformation
            )
        if (
            hasattr(advanced_quantization_config, "search_weight_scale")
            and search_weight_scale_config is None
        ):
            search_weight_scale_config = (
                advanced_quantization_config.search_weight_scale
            )
 
    input_process_config = kwargs.pop("input_process_config", None)
    if input_process_config is not None:
        warnings.warn(
            "input_process_config is deprecated. Use uint8_input_config and preprocessing_config instead.",
            DeprecationWarning,
            stacklevel=3,
        )
        if hasattr(input_process_config, "uint8_input") and uint8_input_config is None:
            uint8_input_config = input_process_config.uint8_input
        if (
            hasattr(input_process_config, "preprocessing")
            and preprocessing_config is None
        ):
            preprocessing_config = input_process_config.preprocessing
 
    save_sample = kwargs.pop("save_sample", None)
    sample_dtype = kwargs.pop("sample_dtype", None)
    if (
        save_sample is not None or sample_dtype is not None
    ) and save_sample_config is None:
        warnings.warn(
            "save_sample and sample_dtype are deprecated. Use save_sample_config instead.",
            DeprecationWarning,
            stacklevel=3,
        )
        save_sample_kwargs = {}
        if save_sample is not None:
            save_sample_kwargs["apply"] = save_sample
        if sample_dtype is not None:
            save_sample_kwargs["dtype"] = sample_dtype
        save_sample_config = SaveSampleConfig(**save_sample_kwargs)
 
    return (
        calibration_config,
        optq_config,
        mod_config,
        equivalent_transformation_config,
        search_weight_scale_config,
        uint8_input_config,
        preprocessing_config,
        save_sample_config,
    )
 
 
 
 
class Model_Dict(Compiler):
    """
    @brief Wrapper around the Mobilint compiler to support compilation and inference workflows.
 
    @details Capabilities include:
    - Compilation of models into MXQ artifacts runnable on Mobilint NPUs.
    - Inference using the full-precision high-level compiled model on CPU or GPU.
    - Inference with the quantized model on CPU or GPU.
    """
 
    def __init__(
        self,
        model,
        backend="onnx",
        device="cpu",
        feed_dict=None,
        dynamic_axes=None,
        in_dformats=None,
        yolo_decode_include=False,
        use_custom_mask=False,
        exclude_first_subgraph=False,
        **kwargs,
    ):
        """
        @brief Initialize the Mobilint compiler wrapper.
 
        @param model string or model instance. Model path or in-memory model to compile.
        @param backend string. Framework identifier for the model (for example "onnx"). Defaults to "onnx".
        @param device string. Target device for inference ("cpu" or "gpu"). Defaults to "cpu".
        @param feed_dict dict. Example inputs for shape resolution.
        @param dynamic_axes dict. Marks model axes as dynamic.
        @param in_dformats dict. Describes input data formats.
        @param yolo_decode_include bool. Runs YOLO decode on NPU when @c True.
        @param use_custom_mask bool. For non-LLM models that include a mask, use a custom mask provided as a separate input instead of the causal mask.
        @param exclude_first_subgraph bool. Applies only when CPU offloading is enabled: exclude the first subgraph from the final graph if it is unsupported.
        @param kwargs dict. Additional compiler arguments.
        """
 
        super().__init__(
            model=model,
            backend=backend,
            device=device,
            feed_dict=feed_dict,
            in_dformats=in_dformats,
            dynamic_axes=dynamic_axes,
            yolo_decode_include=yolo_decode_include,
            use_custom_mask=use_custom_mask,
            exclude_first_subgraph=exclude_first_subgraph,
            **kwargs,
        )
 
    def compile(
        self,
        save_subgraph_type: int = 0,
        output_subgraph_path="",
        compile_config: CompileConfig = None,
        **kwargs,
    ):
        """
        @brief Compile the wrapped model into an MXQ artifact.
 
        @param save_subgraph_type int. Controls optional MBLT exports: 0 disables; 1 saves graph structure; 2 saves
        structure and weights; 3 splits structure into multiple subgraphs; 4 splits structure and weights.
        @param output_subgraph_path string. Destination for exported .mblt when @p save_subgraph_type is 1–4.
        @param compile_config CompileConfig. Compile configuration object.
        @param kwargs dict. Additional arguments forwarded to the compiler.
 
        """
        save_sample = kwargs.pop("save_sample", None)
        sample_dtype = kwargs.pop("sample_dtype", None)
        if save_sample is not None or sample_dtype is not None:
            warnings.warn(
                "save_sample and sample_dtype are deprecated. Use compile_config with saveSample instead.",
                DeprecationWarning,
                stacklevel=2,
            )
            if compile_config is None:
                compile_config = CompileConfig()
            if not compile_config.save_sample.apply:
                save_sample_kwargs = {}
                if save_sample is not None:
                    save_sample_kwargs["apply"] = save_sample
                if sample_dtype is not None:
                    save_sample_kwargs["dtype"] = sample_dtype
                compile_config = compile_config.with_save_sample(**save_sample_kwargs)
 
        super().compile(
            save_subgraph_type=save_subgraph_type,
            output_subgraph_path=output_subgraph_path,
            compile_config=compile_config,
            **kwargs,
        )
 
 
def mxq_compile(
    model,
    calib_data_path: Union[str, List[str]] = _UNSET,
    save_subgraph_type: int = 0,
    output_subgraph_path="",
    save_path: Union[str, List[str]] = _UNSET,
    backend="onnx",
    # -- ModelDict params --
    feed_dict=None,
    dynamic_axes=None,
    in_dformats=None,
    yolo_decode_include=False,
    use_custom_mask=False,
    exclude_first_subgraph=False,
    # -- CompileConfig-overridable (_UNSET = defer to config) --
    device=_UNSET,
    inference_scheme=_UNSET,
    use_random_calib=_UNSET,
    cpu_offload=_UNSET,
    optimize_option=_UNSET,
    buffer_mode=_UNSET,
    input_shape_dict=_UNSET,
    force_npu_input_reposition=_UNSET,
    force_npu_output_reposition=_UNSET,
    image_channels=_UNSET,
    split_blocks=_UNSET,
    split_parts=_UNSET,
    # -- Config objects --
    config_preset: Optional[str] = None,
    compile_config: Optional[CompileConfig] = None,
    resource_management_config: Optional[ResourceManagementConfig] = None,
    calibration_config: Optional[CalibrationConfig] = None,
    bit_config: Optional[BitConfig] = None,
    llm_config: Optional[LlmConfig] = None,
    optq_config: Optional[OptqConfig] = None,
    mod_config: Optional[ModConfig] = None,
    equivalent_transformation_config: Optional[EquivalentTransformationConfig] = None,
    search_weight_scale_config: Optional[SearchWeightScaleConfig] = None,
    save_sample_config: Optional[SaveSampleConfig] = None,
    uint8_input_config: Optional[Uint8InputConfig] = None,
    preprocessing_config: Optional[PreprocessingConfig] = None,
    **kwargs,
):
    """
    @brief Compile a model into a Mobilint eXeCUtable (MXQ) package for execution on Mobilint NPUs.
 
    @details When no explicit value is provided for a parameter marked with @c _UNSET, the default
    from CompileConfig is used. This allows a compile_config file or object to supply the value
    without being overridden by function-level defaults.
 
    Configuration is resolved in priority order (highest to lowest):
    1. Explicitly passed function arguments
    2. Individual sub-config objects (calibration_config, llm_config, etc.)
    3. kwargs partial overrides (quantization_method, weight_dtype, etc.)
    4. compile_config (CompileConfig object or JSON/YAML file) or config_preset
    5. CompileConfig field defaults
 
    @param model string or model instance. Model path. When using @c backend="onnx", this should be the path to an ONNX
    model file. When @c backend is "torchscript", provide a TorchScript module; when "torch", provide a standard
    PyTorch model. For @c backend="tf", pass the directory that contains the TensorFlow SavedModel graph and assets. For
    @c backend="tflite", pass the TF Lite model path.
    @param calib_data_path string or list of strings. Path(s) to the calibration dataset. Accepts either a text/json file
    that lists NumPy files or a directory that contains the pre-processed NumPy files.
    @param save_subgraph_type int. Controls optional MBLT subgraph exports: 0 disables exports; 1 saves only the graph
    structure; 2 saves graph structure plus weights; 3 saves the graph structure split into multiple subgraphs; 4 saves
    both structure and weights split into multiple subgraphs. Defaults to 0.
    @param output_subgraph_path string. Destination path for the exported .mblt file when @p save_subgraph_type is 1–4.
    The resulting file can be used for visualization. Defaults to "".
    @param save_path string or list of strings. Output MXQ filename(s). When omitted, defaults to "{model_name}.mxq"
    derived from the model path basename.
    @param backend string. Framework used to generate the Mobilint IR. Must be one of "onnx", "tf", "tflite",
    "torchscript", or "torch". Defaults to "onnx".
    @param device string. Compilation and inference device: "cpu" or "gpu". When omitted, uses CompileConfig default.
    @param feed_dict dict. Example input tensors for shape inference and inference validation.
    @param dynamic_axes dict. Marks model axes as dynamic. Keys are input names and values map dimension indices to
    aliases (for example {"input": {2: "seq_len"}}).
    @param in_dformats dict. Describes input tensor data formats.
    @param inference_scheme string. NPU inference scheme. One of "single", "multi", "global", "global4", or
    "global8". When omitted, uses CompileConfig default.
    @param yolo_decode_include bool. Determines whether YOLO decode runs on NPU. Defaults to @c False.
    @param use_random_calib bool. Generates random calibration data to validate model compilability.
    When omitted, uses CompileConfig default.
    @param cpu_offload bool. Enables CPU offloading during NPU inference. When omitted, uses CompileConfig default.
    @param optimize_option int. Compiler optimization strategy selector. When omitted, uses CompileConfig default.
    @param buffer_mode int. Buffer serialization mode: 0 uses a naive buffer, 1 uses an mmap-backed buffer.
    When omitted, uses CompileConfig default.
    @param input_shape_dict dict. Multi-shape compilation specification for STT/TTS models only (e.g., Conformer-CTC,
    MeloTTS). The dynamic axis is defined with respect to each input tensor's shape. During compilation, the same axis
    is varied across all model inputs using the provided values. Currently, only a single entry key ("multi_shape0") is
    supported. Example: {"multi_shape0": {"axis": 2, "values": [100, 200, 300]}}
    @param force_npu_input_reposition bool. Force input reposition operations to run on NPU instead of CPU.
    When omitted, uses CompileConfig default.
    @param force_npu_output_reposition bool. Force output reposition operations to run on NPU instead of CPU.
    When omitted, uses CompileConfig default.
    @param image_channels int. Number of image channels (0 for auto-detect).
    When omitted, uses CompileConfig default.
    @param split_blocks list of int. Multi-MXQ split points by transformer block index.
    Only supported for LLM models. When omitted, uses CompileConfig default.
    @param split_parts int. Evenly split transformer blocks into N MXQ parts.
    Only supported for LLM models. When omitted, uses CompileConfig default.
    @param use_custom_mask bool. For non-LLM models that include a mask, use a custom mask provided as a separate input
    instead of the causal mask. For LLMs that incorporate RoPE, setting LLMConfig.dynamic_rope to @c True not only
    enables the use of dynamic RoPE provided via an additional input, but also activates the custom mask. Defaults to
    @c False.
    @param exclude_first_subgraph bool. Applies only when CPU offloading is enabled: exclude the first subgraph from
    the final graph if it is unsupported. Defaults to @c False.
    @param config_preset string. Name of a built-in configuration preset. When provided, loads the preset
    via CompileConfig.from_preset(). Defaults to None (no preset). Available presets:
    - "classification": Image classification models (ResNet, EfficientNet, ViT, etc.)
    - "detection": Object detection models (YOLO, SSD, DETR, etc.)
    - "classification_torchvision": Torchvision classification models with standard preprocessing
    - "yolo_640": YOLO detection models with 640x640 letterbox preprocessing
    - "yolo_1280": YOLO detection models with 1280x1280 letterbox preprocessing
    - "llm": Large Language Models (LLaMA, Qwen, Gemma, etc.)
    - "llm_fast": LLM with faster compilation (less accuracy optimization)
    - "vision_transformer": Vision Transformer models (ViT, DeiT, Swin, etc.)
    - "multimodal": Multimodal models (CLIP, BLIP, LLaVA, etc.)
    @param compile_config CompileConfig or string. CompileConfig object or path to JSON/YAML configuration file
    containing all compilation settings (resourceManagement, calibration, bit, optq, mod, llm, etc.).
    @param resource_management_config ResourceManagementConfig. Resource management configuration object.
    @param calibration_config CalibrationConfig. Calibration configuration object.
    @param bit_config BitConfig. Bit configuration object for quantization precision settings.
    @param llm_config LlmConfig. LLM configuration object.
    @param optq_config OptqConfig. OPTQ (Optimal Brain Quantization) configuration object.
    @param mod_config ModConfig. MOD (Metric-based Optimization and Distillation) configuration object.
    @param equivalent_transformation_config EquivalentTransformationConfig. Configuration for equivalent
    transformations like SmoothQuant.
    @param search_weight_scale_config SearchWeightScaleConfig. Configuration for weight scale search.
    @param save_sample_config SaveSampleConfig. Configuration for sample data generation and saving.
    @param uint8_input_config Uint8InputConfig. Configuration for uint8 input handling.
    @param preprocessing_config PreprocessingConfig. Preprocessing pipeline configuration.
    @param kwargs dict. Additional compiler arguments. Supports partial config overrides such as
    @c quantization_method, @c quantization_mode, @c percentile, @c weight_dtype, @c ram_usage,
    @c max_sequence_length, etc. Also accepts deprecated parameters (@c quantization_config,
    @c advanced_quantization_config, @c input_process_config, @c save_sample, @c sample_dtype)
    which will emit DeprecationWarning.
    @return None.
 
    @par Using configuration
    There are three ways to configure quantization settings:
 
    1. Load all settings from a JSON/YAML config file:
    @code{.py}
    from qbcompiler import mxq_compile
 
    mxq_compile(
        model="path/to/model.onnx",
        calib_data_path="path/to/calib",
        compile_config="path/to/config.json",  # or config.yaml
        device="gpu",
    )
    @endcode
 
    2. Pass individual sub-config objects:
    @code{.py}
    from qbcompiler import mxq_compile
    from qbcompiler.configs import (
        ResourceManagementConfig,
        CalibrationConfig,
        BitConfig,
        OptqConfig,
        ModConfig,
        LlmConfig,
    )
 
    resource_mgmt = ResourceManagementConfig(weight_dtype="float32")
    calib_cfg = CalibrationConfig(method=1, mode=1)
    bit_cfg = BitConfig(...)
    optq_cfg = OptqConfig(apply=True)
    mod_cfg = ModConfig(apply=False)
    llm_cfg = LlmConfig(apply=True)
 
    mxq_compile(
        model="path/to/model.onnx",
        calib_data_path="path/to/calib",
        resource_management_config=resource_mgmt,
        calibration_config=calib_cfg,
        bit_config=bit_cfg,
        optq_config=optq_cfg,
        mod_config=mod_cfg,
        llm_config=llm_cfg,
        device="gpu",
    )
    @endcode
 
    3. Automatically applied partial configuration overrides:
    @code{.py}
    from qbcompiler import mxq_compile
 
    mxq_compile(
        model="path/to/model.onnx",
        calib_data_path="path/to/calib",
        quantization_method=1, # per channel quantization
        quantization_mode=1, # max percentile quantization
        percentile=0.999, # percentile value for max percentile quantization
        quantization_output=0, # per layer quantization for the output layer
        device="gpu",
    )
    @endcode
    Please refer to the mxq_compile function and the quantization configuration section for the meaning of the quantization-related numeric values.
 
 
    @par Compiling models with custom inputs (ONNX/Torch/TensorFlow)
    Provide NumPy inputs whenever the model omits shape information so that qbcompiler can infer unknown dimensions and data
    formats.
 
    @code{.py}
    from qbcompiler import mxq_compile
    import numpy as np
 
    example_input = {
        "input_node_name_1": np.random.randn(1, 3, 224, 224).astype(np.float32),
        "input_node_name_2": np.random.randn(1, 8, 224, 224).astype(np.float32),
        "input_node_name_3": np.random.randn(1, 4, 56, 56).astype(np.float32),
    }
 
    in_dformats = {
        "input_node_name_1": "NCHW",
        "input_node_name_2": "NCHW",
        "input_node_name_3": "NCHW",
    }
 
    onnx_model_path = "path/to/your/model.onnx"
    mxq_compile(
        model=onnx_model_path,
        feed_dict=example_input,
        in_dformats=in_dformats,
        backend="onnx",
        compile_config="path/to/config.json",
    )
    @endcode
    """
    # --- Handle deprecated kwargs ---
    (
        calibration_config,
        optq_config,
        mod_config,
        equivalent_transformation_config,
        search_weight_scale_config,
        uint8_input_config,
        preprocessing_config,
        save_sample_config,
    ) = _resolve_deprecated_kwargs(
        calibration_config=calibration_config,
        optq_config=optq_config,
        mod_config=mod_config,
        equivalent_transformation_config=equivalent_transformation_config,
        search_weight_scale_config=search_weight_scale_config,
        uint8_input_config=uint8_input_config,
        preprocessing_config=preprocessing_config,
        save_sample_config=save_sample_config,
        **kwargs,
    )
 
    # --- Step 1: Base CompileConfig ---
    if config_preset is not None:
        compile_cfg = CompileConfig.from_preset(config_preset)
    elif compile_config is None:
        compile_cfg = CompileConfig()
    elif isinstance(compile_config, CompileConfig):
        compile_cfg = compile_config
    elif isinstance(compile_config, str):
        compile_cfg = CompileConfig.from_file(compile_config)
    else:
        raise ValueError(
            "compile_config must be a CompileConfig object or a path to JSON/YAML configuration file"
        )
 
    # --- Step 2: Override with individual sub-configs ---
    _sub_config_pairs = [
        ("resource_management", resource_management_config, ResourceManagementConfig),
        ("calibration", calibration_config, CalibrationConfig),
        ("bit", bit_config, BitConfig),
        ("llm", llm_config, LlmConfig),
        ("optq", optq_config, OptqConfig),
        ("mod", mod_config, ModConfig),
        (
            "equivalent_transformation",
            equivalent_transformation_config,
            EquivalentTransformationConfig,
        ),
        ("search_weight_scale", search_weight_scale_config, SearchWeightScaleConfig),
        ("save_sample", save_sample_config, SaveSampleConfig),
        ("uint8_input", uint8_input_config, Uint8InputConfig),
        ("preprocessing", preprocessing_config, PreprocessingConfig),
    ]
    sub_overrides = {}
    for field_name, config_obj, expected_cls in _sub_config_pairs:
        if config_obj is None:
            continue
        if not isinstance(config_obj, expected_cls):
            raise TypeError(
                f"Invalid type for '{field_name}': expected {expected_cls.__name__} or None, "
                f"but got {type(config_obj).__name__}."
            )
        sub_overrides[field_name] = config_obj
    if sub_overrides:
        compile_cfg = compile_cfg.model_copy(update=sub_overrides)
 
    # --- Step 3: Override with kwargs (partial config overrides) ---
    calib_kwargs = {}
    if "quantization_method" in kwargs:
        calib_kwargs["method"] = kwargs.pop("quantization_method")
    if "quantization_output" in kwargs:
        calib_kwargs["output"] = kwargs.pop("quantization_output")
    if "quantization_mode" in kwargs:
        calib_kwargs["mode"] = kwargs.pop("quantization_mode")
    if "percentile" in kwargs:
        if "percentile_config" not in calib_kwargs:
            calib_kwargs["percentile_config"] = {}
        calib_kwargs["percentile_config"]["percentile"] = kwargs.pop("percentile")
    if "act_scale_min" in kwargs:
        calib_kwargs["act_scale_min"] = kwargs.pop("act_scale_min")
    if "weight_scale_min" in kwargs:
        calib_kwargs["weight_scale_min"] = kwargs.pop("weight_scale_min")
    if calib_kwargs:
        updated_calib = compile_cfg.calibration.model_copy(update=calib_kwargs)
        compile_cfg = compile_cfg.model_copy(update={"calibration": updated_calib})
 
    llm_kwargs = {}
    if "llm_config_apply" in kwargs:
        llm_kwargs["apply"] = kwargs.pop("llm_config_apply")
    if "max_sequence_length" in kwargs:
        if "attributes" not in llm_kwargs:
            llm_kwargs["attributes"] = {}
        llm_kwargs["attributes"]["max_sequence_length"] = kwargs.pop(
            "max_sequence_length"
        )
    if "max_cache_length" in kwargs:
        if "attributes" not in llm_kwargs:
            llm_kwargs["attributes"] = {}
        llm_kwargs["attributes"]["max_cache_length"] = kwargs.pop("max_cache_length")
    if llm_kwargs:
        updated_llm = compile_cfg.llm.model_copy(update=llm_kwargs)
        compile_cfg = compile_cfg.model_copy(update={"llm": updated_llm})
 
    rm_kwargs = {}
    if "weight_dtype" in kwargs:
        rm_kwargs["weight_dtype"] = kwargs.pop("weight_dtype")
    if "ram_usage" in kwargs:
        rm_kwargs["ram_usage"] = kwargs.pop("ram_usage")
    if rm_kwargs:
        updated_rm = compile_cfg.resource_management.model_copy(update=rm_kwargs)
        compile_cfg = compile_cfg.model_copy(update={"resource_management": updated_rm})
 
    # --- Step 4: Override with explicitly passed external params ---
    external_overrides = {}
    if save_path is not _UNSET:
        external_overrides["save_paths"] = _to_list(save_path)
    elif isinstance(model, str):
        external_overrides["save_paths"] = [f"{Path(model).stem}.mxq"]
    else:
        raise ValueError("Please set the save_path with .mxq extension")
    if calib_data_path is not _UNSET:
        external_overrides["calib_data_path"] = _to_list(calib_data_path)
    if use_random_calib is not _UNSET:
        external_overrides["use_random_calib"] = use_random_calib
    if inference_scheme is not _UNSET:
        external_overrides["inference_scheme"] = inference_scheme
    if cpu_offload is not _UNSET:
        external_overrides["cpu_offload"] = cpu_offload
    if optimize_option is not _UNSET:
        external_overrides["optimize_option"] = optimize_option
    if buffer_mode is not _UNSET:
        external_overrides["buffer_mode"] = buffer_mode
    if input_shape_dict is not _UNSET:
        external_overrides["input_shape_dict"] = input_shape_dict
    if device is not _UNSET:
        external_overrides["device"] = device
    if force_npu_input_reposition is not _UNSET:
        external_overrides["force_npu_input_reposition"] = force_npu_input_reposition
    if force_npu_output_reposition is not _UNSET:
        external_overrides["force_npu_output_reposition"] = force_npu_output_reposition
    if image_channels is not _UNSET:
        external_overrides["image_channels"] = image_channels
    if split_blocks is not _UNSET:
        external_overrides["split_blocks"] = split_blocks
    if split_parts is not _UNSET:
        external_overrides["split_parts"] = split_parts
    compile_cfg = compile_cfg.model_copy(update=external_overrides)
 
    # Resolve values from final config for downstream use
    resolved_device = compile_cfg.device
 
    if isinstance(model, str) and model.endswith(".mblt") and os.path.isfile(model):
        _quantization_task(model, compile_config=compile_cfg)
    else:
        model_dict = Model_Dict(
            model=model,
            backend=backend,
            device=resolved_device,
            feed_dict=feed_dict,
            in_dformats=in_dformats,
            dynamic_axes=dynamic_axes,
            yolo_decode_include=yolo_decode_include,
            use_custom_mask=use_custom_mask,
            exclude_first_subgraph=exclude_first_subgraph,
            **kwargs,
        )
        model_dict.compile(
            save_subgraph_type=save_subgraph_type,
            output_subgraph_path=output_subgraph_path,
            compile_config=compile_cfg,
        )
        torch.cuda.empty_cache()
        del model_dict
 
 
def mblt_compile(
    model: str | Any,
    mblt_save_path: str,
    backend="onnx",
    device="cpu",
    feed_dict=None,
    dynamic_axes=None,
    in_dformats=None,
    yolo_decode_include=False,
    cpu_offload=False,
    use_custom_mask=False,
    exclude_first_subgraph=False,
    **kwargs,
):
    """
    @brief Export a model to the Mobilint .mblt format without producing an MXQ package.
 
    @param model string or model instance. Source model or path to compile.
    @param mblt_save_path string. Output path for the .mblt artifact.
    @param backend string. Framework identifier ("onnx", "tf", "tflite", "torchscript", or "torch"). Defaults to
    "onnx".
    @param device string. Compilation device ("cpu" or "gpu"). Defaults to "cpu".
    @param feed_dict dict. Example inputs used for shape inference.
    @param dynamic_axes dict. Declares dynamic axes per input name.
    @param in_dformats dict. Input dataformat metadata.
    @param yolo_decode_include bool. Runs YOLO decode on NPU when True.
    @param cpu_offload bool. Enables CPU offloading for unsupported groups.
    @param use_custom_mask bool. For non-LLM models that include a mask, use a custom mask provided as a separate input instead of the causal mask.
    For LLMs that incorporate RoPE, setting LLMConfig.dynamic_rope to True not only enables the use of dynamic RoPE provided via an additional input, but also activates the custom mask.
    @param exclude_first_subgraph bool. Applies only when CPU offloading is enabled: exclude the first subgraph from the final graph if it is unsupported.
    @param kwargs dict. Additional arguments forwarded to the compiler.
    @return None.
    """
    model_dict = Model_Dict(
        model=model,
        backend=backend,
        device=device,
        feed_dict=feed_dict,
        dynamic_axes=dynamic_axes,
        in_dformats=in_dformats,
        yolo_decode_include=yolo_decode_include,
        use_custom_mask=use_custom_mask,
        exclude_first_subgraph=exclude_first_subgraph,
        **kwargs,
    )
 
    model_dict.mblt_compile(
        model,
        mblt_save_path=mblt_save_path,
        backend=backend,
        cpu_offload=cpu_offload,
        **kwargs,
    )
 
 
 
 
def run_mblt_model(
    path_to_mblt: str,
    save_all_outputs: bool = True,
    inference_output_path: Optional[str] = None,
    target_subgraph: Optional[int] = None,
):
    from qbcompiler.model_dict.common.model_dict import ValueDict
    from qbcompiler.model_dict.inference.model import ModelInference
    from qbcompiler.model_dict.serialize import load_mblt_model
 
    assert path_to_mblt.endswith(".mblt")
    if not save_all_outputs:
        raise NotImplementedError(
            f"save_all_outputs={save_all_outputs} is not supported yet."
        )
    if target_subgraph:
        raise NotImplementedError(
            f"target_subgraph={target_subgraph} is not supported yet."
        )
    import numpy
 
    if not inference_output_path:
        inference_output_path = str(pathlib.Path(path_to_mblt).with_suffix(".infer"))
 
    vd = ValueDict()
    md, wd = load_mblt_model(path_to_mblt, load_weight=True)
    for sg in md.subgraphs:
        for iidx in sg.inputs:
            inact = sg.activations[iidx]
            if inact.name in set(md.inputs):
                vd[inact.name] = numpy.random.random(tuple(map(int, inact.src_shape)))
 
    model_inference = ModelInference(
        model_dict=md,
        weight_dict=wd,
        value_dict=vd,
        save_all_outputs=save_all_outputs,
        inference_all_outputs_write_path=inference_output_path,
    )
 
    model_inference.run_inference(use_value_dict_data_for_next_input=False)