models.py Source File

Mobilint SDK qb Compiler v1.3 MCS002-KR

• src
• qbcompiler
• configs

models.py

1"""Auto-generated Pydantic models from config_schema.yaml."""

2

3from __future__ import annotations

4from typing import Any, Dict, List, Optional, Union

5from pydantic import BaseModel, Field, ConfigDict, model_validator

6import yaml

7import json

8from pathlib import Path

9

10SCHEMA_VERSION = "1.0.0"

11

12

13class Uint8InputConfig(BaseModel):

14 """

15 @brief Configuration for uint8 input handling

16

17 @details Defines whether inputs should be treated as uint8 and which specific inputs to apply this to.

18

19 @param apply bool. If true, treat specified inputs as uint8

20 @param inputs List[str]. List of input names to treat as uint8. If empty and apply is true, applies to all inputs

21 @param division_factor float. Division factor for uint8 to float conversion (e.g., 255.0 for [0,1], 127.5 for [0,2])

22 """

23

24 model_config = ConfigDict(

25 populate_by_name=True,

26 extra="forbid",

27 )

28

29 apply: bool = Field(

30 default=False, description="If true, treat specified inputs as uint8"

31 )

32 inputs: List[str] = Field(

33 default=[],

34 description="List of input names to treat as uint8. If empty and apply is true, applies to all inputs",

35 )

36 division_factor: float = Field(

37 default=255.0,

38 alias="divisionFactor",

39 description="Division factor for uint8 to float conversion (e.g., 255.0 for [0,1], 127.5 for [0,2])",

40 )

41

42 def with_updates(self, **kwargs) -> "Uint8InputConfig":

43 """Return a copy with updated fields."""

44 return self.model_copy(update=kwargs)

45

46

47class PreprocessingConfig(BaseModel):

48 """

49 @brief Configuration for input preprocessing pipeline

50

51 @details Defines preprocessing operations to be applied to model inputs,

52 including operations like resize, normalize, color conversion, etc.

53

54 @param apply bool. If true, apply preprocessing pipeline

55 @param auto_convert_format bool. If true, automatically convert input format

56 @param pipeline List[Dict[str, Any]]. List of preprocessing operations to apply globally

57 @param input_configs Dict[str, Any]. Per-input preprocessing configurations. Keys are input names

58 """

59

60 model_config = ConfigDict(

61 populate_by_name=True,

62 extra="forbid",

63 )

64

65 apply: bool = Field(

66 default=False, description="If true, apply preprocessing pipeline"

67 )

68 auto_convert_format: bool = Field(

69 default=False,

70 alias="autoConvertFormat",

71 description="If true, automatically convert input format",

72 )

73 pipeline: Any = Field(

74 default=[], description="List of preprocessing operations to apply globally"

75 )

76 input_configs: Any = Field(

77 default={},

78 alias="inputConfigs",

79 description="Per-input preprocessing configurations. Keys are input names",

80 )

81

82 def with_updates(self, **kwargs) -> "PreprocessingConfig":

83 """Return a copy with updated fields."""

84 return self.model_copy(update=kwargs)

85

86

87class ResourceManagementConfig(BaseModel):

88 """

89 @brief Configuration for resource management during model compilation

90

91 @details Controls GPU and memory management settings during the compilation process.

92

93 @param weight_dtype str. Weight data type for calibration (e.g., 'float32', 'float16')

94 @param use_gpu_only_for_calibration bool. If True, use GPU only during the calibration phase

95 @param weight_memory WeightMemory. Weight memory management configuration

96 """

97

98 model_config = ConfigDict(

99 populate_by_name=True,

100 extra="forbid",

101 )

102

103 weight_dtype: str = Field(

104 default="float32",

105 alias="weightDtype",

106 description="Weight data type for calibration (e.g., 'float32', 'float16')",

107 )

108 use_gpu_only_for_calibration: bool = Field(

109 default=True,

110 alias="useGPUOnlyForCalibration",

111 description="If True, use GPU only during the calibration phase",

112 )

113

114 class WeightMemory(BaseModel):

115 """

116 @brief Weight memory management configuration

117

118 @param method int. Weight memory management method index:<br>

119 0: DeleteFloat - Delete float weights after quantization.<br>

120 1: SaveFloat - Save float weights to disk.<br>

121 2: MoveFloat - Move float weights to CPU.<br>

122 3: KeepFloat - Keep float weights in memory.<br>

123 4: KeepAll - Keep all weights in memory.<br>

124 """

125

126 method_list: List[str] = Field(

127 default=["DeleteFloat", "SaveFloat", "MoveFloat", "KeepFloat", "KeepAll"],

128 alias="methodList",

129 )

130 method: int = Field(default=0, alias="method")

131

132 weight_memory: WeightMemory = Field(

133 default_factory=WeightMemory, alias="weightMemory"

134 )

135

136 def with_updates(self, **kwargs) -> "ResourceManagementConfig":

137 """Return a copy with updated fields."""

138 return self.model_copy(update=kwargs)

139

140

141class CalibrationConfig(BaseModel):

142 """

143 @brief Configuration for calibration during quantization

144

145 @details Defines calibration and quantization parameterization used to derive activation/weight scales

146 and related statistics during quantized compilation.

147

148 @param method int. Calibration method index:<br>

149 0: WChALayer - Weight per-channel, Activation per-layer.<br>

150 1: WChAMulti - Weight per-channel, Activation multi-layer.<br>

151 2: WChALayerZeropoint - Weight per-channel, Activation per-layer with zeropoint.<br>

152 3: WChAMultiZeropoint - Weight per-channel, Activation multi-layer with zeropoint.<br>

153 @param output int. Output quantization type index:<br>

154 0: Layer - Per-layer quantization.<br>

155 1: Ch - Per-channel quantization.<br>

156 2: Sigmoid - Sigmoid-based quantization.<br>

157 @param mode int. Quantization mode index:<br>

158 0: Max - Maximum value calibration.<br>

159 1: MaxPercentile - Maximum percentile calibration.<br>

160 2: Histogram - Histogram-based calibration.<br>

161 @param act_scale_min float. Minimum allowed activation scale (lower bound clamp)

162 @param act16_scale_min float. Minimum 16-bit activation scale (actScaleMin / 256)

163 @param weight_scale_min float. Minimum allowed weight scale (lower bound clamp)

164 @param weight16_scale_min float. Minimum 16-bit weight scale (weightScaleMin / 256)

165 @param min_clip_ratio float. Minimum clip ratio constraint applied during calibration

166 @param max_calib_data_size int. Maximum number of calibration samples kept after loading or generation

167 @param max_sample_size_for_quant_scheme int. Maximum number of calibration samples used per quant scheme stage

168 @param max_percentile MaxPercentile. MaxPercentile mode configuration

169 @param fast_dist FastDist. Fast distribution calibration configuration

170 @param histogram Histogram. Histogram-based calibration configuration

171 @param layer_overrides LayerOverrides. Layer-specific override settings for calibration

172 @param statistics Statistics. Statistics save/load configuration with percentile selection

173 @param group_lut GroupLut. LUT grouping algorithm configuration

174 @param optimize_lut OptimizeLut. LUT optimization configuration

175 """

176

177 model_config = ConfigDict(

178 populate_by_name=True,

179 extra="forbid",

180 )

181

182 method_list: List[str] = Field(

183 default=["WChALayer", "WChAMulti", "WChALayerZeropoint", "WChAMultiZeropoint"],

184 alias="methodList",

185 )

186 method: int = Field(default=1, alias="method")

187 output_list: List[str] = Field(

188 default=["Layer", "Ch", "Sigmoid"], alias="outputList"

189 )

190 output: int = Field(default=0, alias="output")

191 mode_list: List[str] = Field(

192 default=["Max", "MaxPercentile", "Histogram"], alias="modeList"

193 )

194 mode: int = Field(default=1, alias="mode")

195

196 act_scale_min: float = Field(

197 default=0.0005,

198 alias="actScaleMin",

199 description="Minimum allowed activation scale (lower bound clamp)",

200 ge=0,

201 le=1,

202 )

203 act16_scale_min: float = Field(

204 default=1.953125e-06,

205 alias="act16ScaleMin",

206 description="Minimum 16-bit activation scale (actScaleMin / 256)",

207 )

208 weight_scale_min: float = Field(

209 default=1e-06,

210 alias="weightScaleMin",

211 description="Minimum allowed weight scale (lower bound clamp)",

212 ge=0,

213 le=1,

214 )

215 weight16_scale_min: float = Field(

216 default=3.90625e-09,

217 alias="weight16ScaleMin",

218 description="Minimum 16-bit weight scale (weightScaleMin / 256)",

219 )

220 min_clip_ratio: float = Field(

221 default=-1,

222 alias="minClipRatio",

223 description="Minimum clip ratio constraint applied during calibration",

224 ge=-1,

225 le=1,

226 )

227 max_calib_data_size: int = Field(

228 default=-1,

229 alias="maxCalibDataSize",

230 description="Maximum number of calibration samples kept after loading or generation",

231 ge=-1,

232 )

233 max_sample_size_for_quant_scheme: int = Field(

234 default=16,

235 alias="maxSampleSizeForQuantScheme",

236 description="Maximum number of calibration samples used per quant scheme stage",

237 ge=1,

238 )

239

240 class MaxPercentile(BaseModel):

241 model_config = ConfigDict(populate_by_name=True)

242

243 """

244 @brief MaxPercentile mode configuration

245

246 @param percentile float. Percentile value for maxPercentile mode

247 @param topk_ratio float. Top-k ratio used in maxPercentile mode

248 @param max_each int. Maximum number of samples processed per iteration

249 @param max_total int. Total maximum number of samples

250 @param per_ch_divisor int. Divisor for per-channel buffer capacity (bufferCap = max(maxTotal / perChDivisor, maxEach))

251 """

252 percentile: float = Field(

253 default=0.9999, description="Percentile value for maxPercentile mode"

254 )

255 topk_ratio: float = Field(

256 default=0.01,

257 alias="topKRatio",

258 description="Top-k ratio used in maxPercentile mode",

259 )

260 max_each: int = Field(

261 default=128,

262 alias="maxEach",

263 description="Maximum number of samples processed per iteration",

264 )

265 max_total: int = Field(

266 default=65536,

267 alias="maxTotal",

268 description="Total maximum number of samples",

269 )

270 per_ch_divisor: int = Field(

271 default=16,

272 alias="perChDivisor",

273 description="Divisor for per-channel buffer capacity (bufferCap = max(maxTotal / perChDivisor, maxEach))",

274 ge=1,

275 )

276

277 max_percentile: MaxPercentile = Field(

278 default_factory=MaxPercentile, alias="maxPercentile"

279 )

280

281 class FastDist(BaseModel):

282 model_config = ConfigDict(populate_by_name=True)

283

284 """

285 @brief Fast distribution calibration configuration

286

287 @param size_cali int.

288 @param kernel_size int.

289 @param stack_size int.

290 """

291 size_cali: int = Field(default=100, alias="sizeCali")

292 kernel_size: int = Field(default=9, alias="kernelSize")

293 stack_size: int = Field(default=32768, alias="stackSize")

294

295 fast_dist: FastDist = Field(default_factory=FastDist, alias="fastDist")

296

297 class Histogram(BaseModel):

298 model_config = ConfigDict(populate_by_name=True)

299

300 """

301 @brief Histogram-based calibration configuration

302

303 @param search_type int. Search type for histogram calibration:<br>

304 0: Percentile.<br>

305 1: MSE.<br>

306 2: KL.<br>

307 @param percentile float. Percentile value for histogram calibration

308 @param use_gpu bool. Use GPU for histogram computation

309 @param num_bins int. Number of bins for histogram

310 @param num_samples int. Number of samples for histogram calibration

311 @param buffer_size int. Buffer size for histogram computation (-1 for auto)

312 @param min_bin_width float. Minimum bin width for histogram

313 @param search_percentile_min float. Minimum search percentile

314 @param search_percentile_max float. Maximum search percentile

315 @param num_search int. Number of search iterations

316 """

317 search_type_list: List[str] = Field(

318 default=["Percentile", "MSE", "KL"], alias="searchTypeList"

319 )

320 search_type: int = Field(default=0, alias="searchType")

321 percentile: float = Field(

322 default=0.9999, description="Percentile value for histogram calibration"

323 )

324 use_gpu: bool = Field(

325 default=True,

326 alias="useGPU",

327 description="Use GPU for histogram computation",

328 )

329 num_bins: int = Field(

330 default=256, alias="numBins", description="Number of bins for histogram"

331 )

332 num_samples: int = Field(

333 default=128,

334 alias="numSamples",

335 description="Number of samples for histogram calibration",

336 )

337 buffer_size: int = Field(

338 default=-1,

339 alias="bufferSize",

340 description="Buffer size for histogram computation (-1 for auto)",

341 )

342 min_bin_width: float = Field(

343 default=1e-06,

344 alias="minBinWidth",

345 description="Minimum bin width for histogram",

346 )

347 search_percentile_min: float = Field(

348 default=0.9999,

349 alias="searchPercentileMin",

350 description="Minimum search percentile",

351 )

352 search_percentile_max: float = Field(

353 default=1.0,

354 alias="searchPercentileMax",

355 description="Maximum search percentile",

356 )

357 num_search: int = Field(

358 default=128, alias="numSearch", description="Number of search iterations"

359 )

360

361 histogram: Histogram = Field(default_factory=Histogram, alias="histogram")

362

363 class LayerOverrides(BaseModel):

364 model_config = ConfigDict(populate_by_name=True)

365

366 """

367 @brief Layer-specific override settings for calibration

368

369 @param act_scale_min dict. Per-layer activation scale minimum overrides. Keys are actScaleMin values (e.g. '0.0005'), values are lists of layer names to apply that override. (e.g. {'0.0005' : ['layer1', 'layer2']})

370 @param percentile dict. Per-layer maxPercentile percentile overrides. Keys are layer names, values are percentile floats. (e.g. {'/model/layer0/conv': 0.999, '/model/layer5/attn': 0.99})

371 @param method dict. Per-layer calibration method overrides. Keys are layer names, values are method ints (0=WChALayer, 1=WChAMulti, 2=WChALayerZeropoint, 3=WChAMultiZeropoint). (e.g. {'/model/layer0/conv': 1, '/model/layer5/attn': 3})

372 """

373 act_scale_min: Any = Field(

374 default={},

375 alias="actScaleMin",

376 description="Per-layer activation scale minimum overrides. Keys are actScaleMin values (e.g. '0.0005'), values are lists of layer names to apply that override. (e.g. {'0.0005' : ['layer1', 'layer2']})",

377 )

378 percentile: Any = Field(

379 default={},

380 description="Per-layer maxPercentile percentile overrides. Keys are layer names, values are percentile floats. (e.g. {'/model/layer0/conv': 0.999, '/model/layer5/attn': 0.99})",

381 )

382 method: Any = Field(

383 default={},

384 description="Per-layer calibration method overrides. Keys are layer names, values are method ints (0=WChALayer, 1=WChAMulti, 2=WChALayerZeropoint, 3=WChAMultiZeropoint). (e.g. {'/model/layer0/conv': 1, '/model/layer5/attn': 3})",

385 )

386

387 layer_overrides: LayerOverrides = Field(

388 default_factory=LayerOverrides, alias="layerOverrides"

389 )

390

391 class Statistics(BaseModel):

392 model_config = ConfigDict(populate_by_name=True)

393

394 """

395 @brief Statistics save/load configuration with percentile selection

396

397 @param apply bool. Enable statistics save/load

398 @param save_path str. Path to save statistics. If empty, not saved

399 @param load_path str. Path to load statistics. If empty, not loaded

400 @param percentiles List[float]. List of percentile candidates

401 @param percentile_index int. Index into percentiles list to select active percentile

402 """

403 apply: bool = Field(default=False, description="Enable statistics save/load")

404 save_path: str = Field(

405 default="",

406 alias="savePath",

407 description="Path to save statistics. If empty, not saved",

408 )

409 load_path: str = Field(

410 default="",

411 alias="loadPath",

412 description="Path to load statistics. If empty, not loaded",

413 )

414 percentiles: List[float] = Field(

415 default=[0.9999, 0.999, 0.99, 0.9],

416 description="List of percentile candidates",

417 )

418 percentile_index: int = Field(

419 default=0,

420 alias="percentileIndex",

421 description="Index into percentiles list to select active percentile",

422 )

423

424 statistics: Statistics = Field(default_factory=Statistics, alias="statistics")

425

426 class GroupLut(BaseModel):

427 model_config = ConfigDict(populate_by_name=True)

428

429 """

430 @brief LUT grouping algorithm configuration

431

432 @param irls_iter int. Number of IRLS iterations for optimal scale computation (1 = single WLS)

433 @param dro_eps float. DRO worst-point coefficient; larger = more conservative clipping (0 = disable)

434 @param cover_floor float. LUTAM cluster scale lower bound as fraction of coverage

435 """

436 irls_iter: int = Field(

437 default=3,

438 alias="irlsIter",

439 description="Number of IRLS iterations for optimal scale computation (1 = single WLS)",

440 ge=1,

441 )

442 dro_eps: float = Field(

443 default=0,

444 alias="droEps",

445 description="DRO worst-point coefficient; larger = more conservative clipping (0 = disable)",

446 ge=0,

447 )

448 cover_floor: float = Field(

449 default=0.85,

450 alias="coverFloor",

451 description="LUTAM cluster scale lower bound as fraction of coverage",

452 ge=0,

453 le=1,

454 )

455

456 group_lut: GroupLut = Field(default_factory=GroupLut, alias="groupLut")

457

458 class OptimizeLut(BaseModel):

459 model_config = ConfigDict(populate_by_name=True)

460

461 """

462 @brief LUT optimization configuration

463

464 @param optimization_level int. Optimizer search depth (0 = surrogate-only fast, 1 = surrogate + true-objective refinement)

465 """

466 optimization_level: int = Field(

467 default=1,

468 alias="optimizationLevel",

469 description="Optimizer search depth (0 = surrogate-only fast, 1 = surrogate + true-objective refinement)",

470 ge=0,

471 le=1,

472 )

473

474 optimize_lut: OptimizeLut = Field(default_factory=OptimizeLut, alias="optimizeLut")

475

476 def with_updates(self, **kwargs) -> "CalibrationConfig":

477 """Return a copy with updated fields."""

478 return self.model_copy(update=kwargs)

479

480

481class BitConfig(BaseModel):

482 """

483 @brief Configuration for bit precision

484

485 @details Defines bit-width parameterization for activations and weights used in

486 mixed-precision quantization (e.g., attention and FFN components).

487

488 @param transformer Transformer. Transformer-specific bit-width configuration

489 @param save_info SaveInfo. Bit allocation save/load configuration

490 @param layer_overrides LayerOverrides. Layer-specific bit-width override settings

491 """

492

493 model_config = ConfigDict(

494 populate_by_name=True,

495 extra="forbid",

496 )

497

498 class Transformer(BaseModel):

499 model_config = ConfigDict(populate_by_name=True)

500

501 """

502 @brief Transformer-specific bit-width configuration

503

504 @param activation Activation. Activation bit-widths for transformer components

505 @param weight Weight. Weight bit-widths for transformer components

506 @param mixed_precision MixedPrecision. Mixed precision configuration

507 """

508

509 class Activation(BaseModel):

510 model_config = ConfigDict(populate_by_name=True)

511

512 """

513 @brief Activation bit-widths for transformer components

514

515 @param query int. Query activation bit-width

516 @param key int. Key activation bit-width

517 @param value int. Value activation bit-width

518 @param output int. Output activation bit-width

519 @param head int. Head activation bit-width

520 @param router int. MoE router gate activation bit-width

521 @param ffn Ffn. FFN activation bit-widths (int shorthand sets all sublayers)

522 """

523 query: int = Field(default=8, description="Query activation bit-width")

524 key: int = Field(default=8, description="Key activation bit-width")

525 value: int = Field(default=8, description="Value activation bit-width")

526 output: int = Field(default=16, description="Output activation bit-width")

527 head: int = Field(default=8, description="Head activation bit-width")

528 router: int = Field(

529 default=8, description="MoE router gate activation bit-width"

530 )

531

532 class Ffn(BaseModel):

533 """

534 @brief FFN activation bit-widths (int shorthand sets all sublayers)

535

536 @param up int. FFN up-projection activation bit-width

537 @param gate int. FFN gate (SwiGLU) activation bit-width

538 @param down int. FFN down-projection activation bit-width

539 """

540

541 @model_validator(mode="before")

542 @classmethod

543 def expand_int_shorthand(cls, v):

544 if isinstance(v, bool):

545 raise ValueError("bool is not a valid bit-width")

546 if isinstance(v, int):

547 return {"up": int(v), "gate": int(v), "down": int(v)}

548 return v

549

550 up: int = Field(

551 default=16, description="FFN up-projection activation bit-width"

552 )

553 gate: int = Field(

554 default=16, description="FFN gate (SwiGLU) activation bit-width"

555 )

556 down: int = Field(

557 default=16, description="FFN down-projection activation bit-width"

558 )

559

560 ffn: Ffn = Field(default_factory=Ffn, alias="ffn")

561

562 activation: Activation = Field(default_factory=Activation, alias="activation")

563

564 class Weight(BaseModel):

565 model_config = ConfigDict(populate_by_name=True)

566

567 """

568 @brief Weight bit-widths for transformer components

569

570 @param query int. Query weight bit-width

571 @param key int. Key weight bit-width

572 @param value int. Value weight bit-width

573 @param output int. Output weight bit-width

574 @param head int. Head weight bit-width

575 @param router int. MoE router gate weight bit-width

576 @param ffn Ffn. FFN weight bit-widths (int shorthand sets all sublayers)

577 """

578 query: int = Field(default=8, description="Query weight bit-width")

579 key: int = Field(default=8, description="Key weight bit-width")

580 value: int = Field(default=8, description="Value weight bit-width")

581 output: int = Field(default=8, description="Output weight bit-width")

582 head: int = Field(default=8, description="Head weight bit-width")

583 router: int = Field(

584 default=8, description="MoE router gate weight bit-width"

585 )

586

587 class Ffn(BaseModel):

588 """

589 @brief FFN weight bit-widths (int shorthand sets all sublayers)

590

591 @param up int. FFN up-projection weight bit-width

592 @param gate int. FFN gate (SwiGLU) weight bit-width

593 @param down int. FFN down-projection weight bit-width

594 """

595

596 @model_validator(mode="before")

597 @classmethod

598 def expand_int_shorthand(cls, v):

599 if isinstance(v, bool):

600 raise ValueError("bool is not a valid bit-width")

601 if isinstance(v, int):

602 return {"up": int(v), "gate": int(v), "down": int(v)}

603 return v

604

605 up: int = Field(

606 default=8, description="FFN up-projection weight bit-width"

607 )

608 gate: int = Field(

609 default=8, description="FFN gate (SwiGLU) weight bit-width"

610 )

611 down: int = Field(

612 default=8, description="FFN down-projection weight bit-width"

613 )

614

615 ffn: Ffn = Field(default_factory=Ffn, alias="ffn")

616

617 weight: Weight = Field(default_factory=Weight, alias="weight")

618

619 class MixedPrecision(BaseModel):

620 model_config = ConfigDict(populate_by_name=True)

621

622 """

623 @brief Mixed precision configuration

624

625 @param weight Weight. Mixed precision configuration for weights

626 @param activation Activation. Per-layer activation mixed precision configuration

627 """

628

629 class Weight(BaseModel):

630 model_config = ConfigDict(populate_by_name=True)

631

632 """

633 @brief Mixed precision configuration for weights

634

635 @param apply bool. If true, apply mixed-precision according to the specified bit-widths

636 @param type_wise bool. Apply type-wise mixed precision

637 @param prune float. Pruning ratio

638 @param bit_2 float. Ratio of 2-bit quantization

639 @param bit_4 float. Ratio of 4-bit quantization

640 @param bit_8 float. Ratio of 8-bit quantization

641 @param importance_threshold_low float. Low importance threshold

642 @param importance_threshold_high float. High importance threshold

643 """

644 apply: bool = Field(

645 default=False,

646 description="If true, apply mixed-precision according to the specified bit-widths",

647 )

648 type_wise: bool = Field(

649 default=True,

650 alias="typeWise",

651 description="Apply type-wise mixed precision",

652 )

653 prune: float = Field(default=0, description="Pruning ratio")

654 bit_2: float = Field(

655 default=0, alias="bit2", description="Ratio of 2-bit quantization"

656 )

657 bit_4: float = Field(

658 default=0, alias="bit4", description="Ratio of 4-bit quantization"

659 )

660 bit_8: float = Field(

661 default=1, alias="bit8", description="Ratio of 8-bit quantization"

662 )

663 importance_threshold_low: float = Field(

664 default=-1,

665 alias="importanceThreshold_low",

666 description="Low importance threshold",

667 )

668 importance_threshold_high: float = Field(

669 default=-1,

670 alias="importanceThreshold_high",

671 description="High importance threshold",

672 )

673

674 weight: Weight = Field(default_factory=Weight, alias="weight")

675

676 class Activation(BaseModel):

677 model_config = ConfigDict(populate_by_name=True)

678

679 """

680 @brief Per-layer activation mixed precision configuration

681

682 @param apply bool. If true, apply per-layer activation mixed precision

683 @param ratio_16bit float. Ratio of layers assigned 16-bit (used when importanceThreshold < 0)

684 @param importance_threshold float. Normalized importance threshold for 16-bit assignment (negative = use ratio_16bit)

685 @param search_range int. Target layer range: -1=all layers, N=first N layers

686 """

687 apply: bool = Field(

688 default=False,

689 description="If true, apply per-layer activation mixed precision",

690 )

691 ratio_16bit: float = Field(

692 default=0.45,

693 alias="ratio16Bit",

694 description="Ratio of layers assigned 16-bit (used when importanceThreshold < 0)",

695 )

696 importance_threshold: float = Field(

697 default=-1,

698 alias="importanceThreshold",

699 description="Normalized importance threshold for 16-bit assignment (negative = use ratio_16bit)",

700 )

701 search_range: int = Field(

702 default=-1,

703 alias="searchRange",

704 description="Target layer range: -1=all layers, N=first N layers",

705 )

706

707 activation: Activation = Field(

708 default_factory=Activation, alias="activation"

709 )

710

711 mixed_precision: MixedPrecision = Field(

712 default_factory=MixedPrecision, alias="mixedPrecision"

713 )

714

715 transformer: Transformer = Field(default_factory=Transformer, alias="transformer")

716

717 class SaveInfo(BaseModel):

718 model_config = ConfigDict(populate_by_name=True)

719

720 """

721 @brief Bit allocation save/load configuration

722

723 @param save_path str. Path to save the bit allocation. If empty, not saved

724 @param load_path str. Path to load the bit allocation. If empty, not loaded

725 """

726 save_path: str = Field(

727 default="",

728 alias="savePath",

729 description="Path to save the bit allocation. If empty, not saved",

730 )

731 load_path: str = Field(

732 default="",

733 alias="loadPath",

734 description="Path to load the bit allocation. If empty, not loaded",

735 )

736

737 save_info: SaveInfo = Field(default_factory=SaveInfo, alias="saveInfo")

738

739 class LayerOverrides(BaseModel):

740 model_config = ConfigDict(populate_by_name=True)

741

742 """

743 @brief Layer-specific bit-width override settings

744

745 @param activation_16bits list[string]. Layer names to force 16-bit activations

746 @param weight_16bits list[string]. Layer names to force 16-bit weights

747 @param weight_8bits list[string]. Layer names to force 8-bit weights

748 """

749 activation_16bits: List[str] = Field(

750 default=[],

751 alias="activation16Bits",

752 description="Layer names to force 16-bit activations",

753 )

754 weight_16bits: List[str] = Field(

755 default=[],

756 alias="weight16Bits",

757 description="Layer names to force 16-bit weights",

758 )

759 weight_8bits: List[str] = Field(

760 default=[],

761 alias="weight8Bits",

762 description="Layer names to force 8-bit weights",

763 )

764

765 layer_overrides: LayerOverrides = Field(

766 default_factory=LayerOverrides, alias="layerOverrides"

767 )

768

769 def with_updates(self, **kwargs) -> "BitConfig":

770 """Return a copy with updated fields."""

771 return self.model_copy(update=kwargs)

772

773

774class HessianQuantConfig(BaseModel):

775 """

776 @brief Configuration for HessianQuant algorithm

777

778 @details Defines parameters controlling whether and how HessianQuant is applied during quantization,

779 including layer-level inclusion/exclusion lists.

780

781 @param apply bool. If true, apply HessianQuant

782 @param hessian_dtype str. Storage dtype for the accumulated HessianQuant Hessian. bf16 halves its memory footprint (host RAM when the Hessian lives on CPU, VRAM when on GPU) — critical for large models such as MoE with thousands of expert FFN Hessians. Compute stays float32 regardless: the per-batch matmul and accumulation run in float32 and the solve upcasts back to float32; only the persistent accumulator is bfloat16.:<br>

783 0: fp32 - Store the Hessian in float32.<br>

784 1: bf16 - Store the Hessian in bfloat16 (half the memory).<br>

785 @param accumulation_device str. Device the HessianQuant Hessian is accumulated on during calibration. gpu accumulates on the calibration device, which is 3-5x faster because the whole d x d accumulator is otherwise copied off the device once per layer per batch; the Hessian is parked on the host as soon as the last batch is in, so this does not raise peak VRAM for the rest of the compile. cpu accumulates on the host instead, bounding VRAM during calibration itself for models whose summed Hessian does not fit alongside the activations. auto follows resourceManagement.useGPUOnlyForCalibration, which is the knob that already says whether this compile is VRAM-bound: false picks gpu, true picks cpu. Resolved once at config load and logged; a CPU compile always accumulates on the host regardless.:<br>

786 0: auto - Follow useGPUOnlyForCalibration: false picks gpu, true picks cpu.<br>

787 1: cpu - Accumulate on the host, bounding VRAM during calibration itself.<br>

788 2: gpu - Accumulate on the calibration device, then park on the host.<br>

789 @param attributes Attributes. HessianQuant algorithm attributes

790 """

791

792 model_config = ConfigDict(

793 populate_by_name=True,

794 extra="forbid",

795 )

796

797 apply: bool = Field(default=False, description="If true, apply HessianQuant")

798 hessian_dtype: str = Field(

799 default="fp32",

800 alias="hessianDtype",

801 description="Storage dtype for the accumulated HessianQuant Hessian. bf16 halves its memory footprint (host RAM when the Hessian lives on CPU, VRAM when on GPU) — critical for large models such as MoE with thousands of expert FFN Hessians. Compute stays float32 regardless: the per-batch matmul and accumulation run in float32 and the solve upcasts back to float32; only the persistent accumulator is bfloat16.",

802 )

803 accumulation_device: str = Field(

804 default="auto",

805 alias="accumulationDevice",

806 description="Device the HessianQuant Hessian is accumulated on during calibration. gpu accumulates on the calibration device, which is 3-5x faster because the whole d x d accumulator is otherwise copied off the device once per layer per batch; the Hessian is parked on the host as soon as the last batch is in, so this does not raise peak VRAM for the rest of the compile. cpu accumulates on the host instead, bounding VRAM during calibration itself for models whose summed Hessian does not fit alongside the activations. auto follows resourceManagement.useGPUOnlyForCalibration, which is the knob that already says whether this compile is VRAM-bound: false picks gpu, true picks cpu. Resolved once at config load and logged; a CPU compile always accumulates on the host regardless.",

807 )

808

809 class Attributes(BaseModel):

810 model_config = ConfigDict(populate_by_name=True)

811

812 """

813 @brief HessianQuant algorithm attributes

814

815 @param act_order bool. If true, use activation order

816 @param block_size int. Block size used for HessianQuant

817 @param perc_damp float. Percentage dampening factor

818 @param apply_layers List[str]. Layer names to apply HessianQuant. If empty, applies to all eligible layers

819 @param exclude_layers List[str]. Layer names to exclude from HessianQuant

820 """

821 act_order: bool = Field(

822 default=True, alias="actOrder", description="If true, use activation order"

823 )

824 block_size: int = Field(

825 default=256,

826 alias="blockSize",

827 description="Block size used for HessianQuant",

828 )

829 perc_damp: float = Field(

830 default=0.01, alias="percDamp", description="Percentage dampening factor"

831 )

832 apply_layers: List[str] = Field(

833 default=[],

834 alias="applyLayers",

835 description="Layer names to apply HessianQuant. If empty, applies to all eligible layers",

836 )

837 exclude_layers: List[str] = Field(

838 default=[],

839 alias="excludeLayers",

840 description="Layer names to exclude from HessianQuant",

841 )

842

843 attributes: Attributes = Field(default_factory=Attributes, alias="attributes")

844

845 def with_updates(self, **kwargs) -> "HessianQuantConfig":

846 """Return a copy with updated fields."""

847 return self.model_copy(update=kwargs)

848

849

850class LayerBiasCorrectionConfig(BaseModel):

851 """

852 @brief Configuration for calibration-derived layer bias correction

853

854 @details Measures systematic per-channel float-to-integer activation error throughout

855 the quantized network and folds damped corrections into integer convolution

856 biases. Corrections are recomputed between iterations to account for upstream

857 changes. This is calibration-only bias correction and does not use labels or

858 Minimum Output Difference optimization.

859

860 @param apply bool. If true, correct systematic per-layer quantization bias

861 @param attributes Attributes. Layer bias correction attributes

862 """

863

864 model_config = ConfigDict(

865 populate_by_name=True,

866 extra="forbid",

867 )

868

869 apply: bool = Field(

870 default=False,

871 description="If true, correct systematic per-layer quantization bias",

872 )

873

874 class Attributes(BaseModel):

875 model_config = ConfigDict(populate_by_name=True)

876

877 """

878 @brief Layer bias correction attributes

879

880 @param num_samples int. Maximum number of calibration samples used per iteration

881 @param iterations int. Number of damped correction iterations

882 @param correction_rate float. Fraction of the measured bias error applied per iteration

883 """

884 num_samples: int = Field(

885 default=256,

886 alias="numSamples",

887 description="Maximum number of calibration samples used per iteration",

888 ge=1,

889 )

890 iterations: int = Field(

891 default=5, description="Number of damped correction iterations", ge=1

892 )

893 correction_rate: float = Field(

894 default=0.05,

895 alias="correctionRate",

896 description="Fraction of the measured bias error applied per iteration",

897 ge=1e-06,

898 le=1.0,

899 )

900

901 attributes: Attributes = Field(default_factory=Attributes, alias="attributes")

902

903 def with_updates(self, **kwargs) -> "LayerBiasCorrectionConfig":

904 """Return a copy with updated fields."""

905 return self.model_copy(update=kwargs)

906

907

908class ModConfig(BaseModel):

909 """

910 @brief Configuration for Minimum Output Difference algorithm

911

912 @details Defines parameters controlling whether and how MOD is applied during quantization,

913 including layer-level inclusion/exclusion lists.

914

915 @param apply bool. If true, apply MOD

916 @param attributes Attributes. MOD algorithm attributes

917 """

918

919 model_config = ConfigDict(

920 populate_by_name=True,

921 extra="forbid",

922 )

923

924 apply: bool = Field(default=False, description="If true, apply MOD")

925

926 class Attributes(BaseModel):

927 model_config = ConfigDict(populate_by_name=True)

928

929 """

930 @brief MOD algorithm attributes

931

932 @param epochs int. Number of training epochs

933 @param warmup_epochs int. Number of warmup epochs

934 @param lr_min_ratio float. Minimum learning rate ratio

935 @param save_dir str. Directory to save MOD results

936 @param seed int. Random seed for MOD

937 @param apply_layers List[str]. Layer names to apply MOD. If empty, applies to all eligible layers

938 @param exclude_layers List[str]. Layer names to exclude from MOD

939 @param mod_after_layer_name str. Apply MOD after this layer

940 @param anchors List. Anchor configurations for detection models. Nested list of anchor box sizes for detection models. Structure: List[List[List[int]]] where outer list is per detection head (e.g. small/medium/large), middle list is anchors per head, inner list is [width, height]. Example: [[[12,16],[19,36],[40,28]], [[36,75],[76,55],[72,146]], [[142,110],[192,243],[459,401]]]

941 @param use_xyxy bool. Use XYXY format for bounding boxes

942 @param learning_rates LearningRates. Learning rate configuration for MOD

943 @param training Training. MOD training configuration

944 @param loss Loss. MOD loss configuration

945 @param post_processing PostProcessing. Post-processing configuration for detection models

946 """

947 epochs: int = Field(default=4, description="Number of training epochs")

948 warmup_epochs: int = Field(

949 default=1, alias="warmupEpochs", description="Number of warmup epochs"

950 )

951 lr_min_ratio: float = Field(

952 default=0.0001,

953 alias="lrMinRatio",

954 description="Minimum learning rate ratio",

955 )

956 save_dir: str = Field(

957 default="", alias="saveDir", description="Directory to save MOD results"

958 )

959 seed: int = Field(default=0, description="Random seed for MOD")

960 apply_layers: List[str] = Field(

961 default=[],

962 alias="applyLayers",

963 description="Layer names to apply MOD. If empty, applies to all eligible layers",

964 )

965 exclude_layers: List[str] = Field(

966 default=[],

967 alias="excludeLayers",

968 description="Layer names to exclude from MOD",

969 )

970 mod_after_layer_name: str = Field(

971 default="",

972 alias="modAfterLayerName",

973 description="Apply MOD after this layer",

974 )

975 anchors: Any = Field(

976 default=[],

977 description="Anchor configurations for detection models. Nested list of anchor box sizes for detection models. Structure: List[List[List[int]]] where outer list is per detection head (e.g. small/medium/large), middle list is anchors per head, inner list is [width, height]. Example: [[[12,16],[19,36],[40,28]], [[36,75],[76,55],[72,146]], [[142,110],[192,243],[459,401]]]",

978 )

979 use_xyxy: bool = Field(

980 default=False,

981 alias="useXYXY",

982 description="Use XYXY format for bounding boxes",

983 )

984

985 class LearningRates(BaseModel):

986 model_config = ConfigDict(populate_by_name=True)

987

988 """

989 @brief Learning rate configuration for MOD

990

991 @param act_scale float. Learning rate for activation scale

992 @param zeropoint float. Learning rate for zeropoint

993 @param weight_scale float. Learning rate for weight scale

994 @param weight float. Learning rate for weight

995 @param bias float. Learning rate for bias

996 """

997 act_scale: float = Field(

998 default=0.0,

999 alias="actScale",

1000 description="Learning rate for activation scale",

1001 )

1002 zeropoint: float = Field(

1003 default=0.0, description="Learning rate for zeropoint"

1004 )

1005 weight_scale: float = Field(

1006 default=0.0,

1007 alias="weightScale",

1008 description="Learning rate for weight scale",

1009 )

1010 weight: float = Field(default=4e-06, description="Learning rate for weight")

1011 bias: float = Field(default=4e-06, description="Learning rate for bias")

1012

1013 learning_rates: LearningRates = Field(

1014 default_factory=LearningRates, alias="learningRates"

1015 )

1016

1017 class Training(BaseModel):

1018 model_config = ConfigDict(populate_by_name=True)

1019

1020 """

1021 @brief MOD training configuration

1022

1023 @param batch_size int. Batch size for MOD training

1024 @param q_drop float. Quantization drop probability

1025 @param quantize_weight bool. Whether to quantize weights

1026 @param weight_scale_init str. Weight scale initialization method

1027 @param downresol_mode str. Downresolution mode

1028 @param scheduler_type str. LR scheduler type

1029 """

1030 batch_size: int = Field(

1031 default=1, alias="batchSize", description="Batch size for MOD training"

1032 )

1033 q_drop: float = Field(

1034 default=0.0, alias="qDrop", description="Quantization drop probability"

1035 )

1036 quantize_weight: bool = Field(

1037 default=True,

1038 alias="quantizeWeight",

1039 description="Whether to quantize weights",

1040 )

1041 weight_scale_init: str = Field(

1042 default="MinMax",

1043 alias="weightScaleInit",

1044 description="Weight scale initialization method",

1045 )

1046 downresol_mode: str = Field(

1047 default="STE", alias="downresolMode", description="Downresolution mode"

1048 )

1049 scheduler_type: str = Field(

1050 default="Cosine", alias="schedulerType", description="LR scheduler type"

1051 )

1052

1053 training: Training = Field(default_factory=Training, alias="training")

1054

1055 class Loss(BaseModel):

1056 model_config = ConfigDict(populate_by_name=True)

1057

1058 """

1059 @brief MOD loss configuration

1060

1061 @param type str. Loss type (MSE, KL, etc.)

1062 @param use_outputs bool. Use model outputs for loss computation

1063 @param kl_temperature float. KL divergence temperature

1064 @param recon_prob float. Reconstruction probability

1065 @param recon_coeff float. Reconstruction coefficient

1066 @param lambda_0 float. Loss weight lambda_0

1067 @param lambda_1 float. Loss weight lambda_1

1068 @param lambda_2 float. Loss weight lambda_2

1069 @param lambda_3 float. Loss weight lambda_3

1070 @param custom_loss_jit_path str. Path to custom JIT-compiled loss function. Refer to /workspace/quantizer/pyutils/mel.pt

1071 """

1072 type: str = Field(default="MSE", description="Loss type (MSE, KL, etc.)")

1073 use_outputs: bool = Field(

1074 default=False,

1075 alias="useOutputs",

1076 description="Use model outputs for loss computation",

1077 )

1078 kl_temperature: float = Field(

1079 default=1.0,

1080 alias="KLTemperature",

1081 description="KL divergence temperature",

1082 )

1083 recon_prob: float = Field(

1084 default=1.0, alias="reconProb", description="Reconstruction probability"

1085 )

1086 recon_coeff: float = Field(

1087 default=1.0,

1088 alias="reconCoeff",

1089 description="Reconstruction coefficient",

1090 )

1091 lambda_0: float = Field(

1092 default=1.0, alias="lambda0", description="Loss weight lambda_0"

1093 )

1094 lambda_1: float = Field(

1095 default=1.0, alias="lambda1", description="Loss weight lambda_1"

1096 )

1097 lambda_2: float = Field(

1098 default=1.0, alias="lambda2", description="Loss weight lambda_2"

1099 )

1100 lambda_3: float = Field(

1101 default=1.0, alias="lambda3", description="Loss weight lambda_3"

1102 )

1103 custom_loss_jit_path: str = Field(

1104 default="",

1105 alias="customLossJITPath",

1106 description="Path to custom JIT-compiled loss function. Refer to /workspace/quantizer/pyutils/mel.pt",

1107 )

1108

1109 loss: Loss = Field(default_factory=Loss, alias="loss")

1110

1111 class PostProcessing(BaseModel):

1112 model_config = ConfigDict(populate_by_name=True)

1113

1114 """

1115 @brief Post-processing configuration for detection models

1116

1117 @param post str. Post-processing type

1118 @param box_conf_thres float. Box confidence threshold

1119 @param box_iou_thres float. Box IoU threshold

1120 """

1121 post: str = Field(default="", description="Post-processing type")

1122 box_conf_thres: float = Field(

1123 default=0, alias="boxConfThres", description="Box confidence threshold"

1124 )

1125 box_iou_thres: float = Field(

1126 default=0, alias="boxIoUThres", description="Box IoU threshold"

1127 )

1128

1129 post_processing: PostProcessing = Field(

1130 default_factory=PostProcessing, alias="postProcessing"

1131 )

1132

1133 attributes: Attributes = Field(default_factory=Attributes, alias="attributes")

1134

1135 def with_updates(self, **kwargs) -> "ModConfig":

1136 """Return a copy with updated fields."""

1137 return self.model_copy(update=kwargs)

1138

1139

1140class LlmConfig(BaseModel):

1141 """

1142 @brief Configuration for Large Language Model (LLM) compilation

1143

1144 @details Defines LLM-specific settings including sequence lengths, cache configurations,

1145 and runtime parameters for efficient LLM inference.

1146

1147 @param apply bool. If True, apply LLM-specific configurations

1148 @param npu_parallel_degree int. Number of NPU partitions for FFN tensor parallelism (1 = disabled). Applied before OptimizeFFN if both active.

1149 @param attributes Attributes. LLM attributes configuration

1150 """

1151

1152 model_config = ConfigDict(

1153 populate_by_name=True,

1154 extra="forbid",

1155 )

1156

1157 apply: bool = Field(

1158 default=False, description="If True, apply LLM-specific configurations"

1159 )

1160 npu_parallel_degree: int = Field(

1161 default=1,

1162 alias="npuParallelDegree",

1163 description="Number of NPU partitions for FFN tensor parallelism (1 = disabled). Applied before OptimizeFFN if both active.",

1164 )

1165

1166 class Attributes(BaseModel):

1167 model_config = ConfigDict(populate_by_name=True)

1168

1169 """

1170 @brief LLM attributes configuration

1171

1172 @param max_data_length int. Maximum data length

1173 @param max_sequence_length int. Maximum sequence length

1174 @param max_cache_length int. Maximum cache length

1175 @param max_core_data_length int. Maximum core data length

1176 @param calibration Calibration. LLM calibration settings

1177 @param runtime Runtime. LLM runtime settings

1178 @param debug Debug. LLM debug settings

1179 """

1180 max_data_length: int = Field(

1181 default=4096, alias="maxDataLength", description="Maximum data length"

1182 )

1183 max_sequence_length: int = Field(

1184 default=4096,

1185 alias="maxSequenceLength",

1186 description="Maximum sequence length",

1187 )

1188 max_cache_length: int = Field(

1189 default=4096, alias="maxCacheLength", description="Maximum cache length"

1190 )

1191 max_core_data_length: int = Field(

1192 default=128,

1193 alias="maxCoreDataLength",

1194 description="Maximum core data length",

1195 )

1196

1197 class Calibration(BaseModel):

1198 model_config = ConfigDict(populate_by_name=True)

1199

1200 """

1201 @brief LLM calibration settings

1202

1203 @param random_seq_length int. Random sequence length used for calibration

1204 @param use_full_seq_length bool. If True, use the full sequence length for calibration

1205 """

1206 random_seq_length: int = Field(

1207 default=80,

1208 alias="randomSeqLength",

1209 description="Random sequence length used for calibration",

1210 )

1211 use_full_seq_length: bool = Field(

1212 default=False,

1213 alias="useFullSeqLength",

1214 description="If True, use the full sequence length for calibration",

1215 )

1216

1217 calibration: Calibration = Field(

1218 default_factory=Calibration, alias="calibration"

1219 )

1220

1221 class Runtime(BaseModel):

1222 model_config = ConfigDict(populate_by_name=True)

1223

1224 """

1225 @brief LLM runtime settings

1226

1227 @param use_global_core bool. If True, use a global core

1228 @param batch_size int. Batch size

1229 @param npu_core_ids List[int]. List of NPU core IDs

1230 @param dynamic_rope bool. If True, enable dynamic RoPE (rotary position embedding)

1231 @param dynamic_mask bool. If True, enable dynamic mask (attention mask as runtime input)

1232 """

1233 use_global_core: bool = Field(

1234 default=False,

1235 alias="useGlobalCore",

1236 description="If True, use a global core",

1237 )

1238 batch_size: int = Field(

1239 default=1, alias="batchSize", description="Batch size"

1240 )

1241 npu_core_ids: List[int] = Field(

1242 default=[0], alias="npuCoreIds", description="List of NPU core IDs"

1243 )

1244 dynamic_rope: bool = Field(

1245 default=False,

1246 alias="dynamicRope",

1247 description="If True, enable dynamic RoPE (rotary position embedding)",

1248 )

1249 dynamic_mask: bool = Field(

1250 default=False,

1251 alias="dynamicMask",

1252 description="If True, enable dynamic mask (attention mask as runtime input)",

1253 )

1254

1255 runtime: Runtime = Field(default_factory=Runtime, alias="runtime")

1256

1257 class Debug(BaseModel):

1258 model_config = ConfigDict(populate_by_name=True)

1259

1260 """

1261 @brief LLM debug settings

1262

1263 @param apply bool. Enable LLM debug mode

1264 @param batch_debug_bundle_size int. Batch debug bundle size

1265 """

1266 apply: bool = Field(default=False, description="Enable LLM debug mode")

1267 batch_debug_bundle_size: int = Field(

1268 default=0,

1269 alias="batchDebugBundleSize",

1270 description="Batch debug bundle size",

1271 )

1272

1273 debug: Debug = Field(default_factory=Debug, alias="debug")

1274

1275 attributes: Attributes = Field(default_factory=Attributes, alias="attributes")

1276

1277 def with_updates(self, **kwargs) -> "LlmConfig":

1278 """Return a copy with updated fields."""

1279 return self.model_copy(update=kwargs)

1280

1281

1282class MoeConfig(BaseModel):

1283 """

1284 @brief Sparse MoE expert-selection configuration (calibration only)

1285

1286 @details Controls which experts are calibrated inside SparseMoe modules. selectionMode

1287 is a calibration-only knob: it picks which experts collect statistics (and, for

1288 TopK, on which tokens). It does NOT change inference routing — the forward path

1289 always routes the router's top-K experts regardless of this setting.

1290 scoreThreshold is only used when selectionMode is Threshold.

1291

1292 @param selection_mode int. Expert selection mode index (calibration only; inference is always top-K):<br>

1293 0: TopK - Calibrate only the router's top-K experts per token (matches inference routing).<br>

1294 1: All - Calibrate every expert on the full sequence.<br>

1295 2: Threshold - Calibrate all experts whose routing score exceeds scoreThreshold.<br>

1296 @param score_threshold float. Routing score threshold used when selectionMode is Threshold (calibration only)

1297 """

1298

1299 model_config = ConfigDict(

1300 populate_by_name=True,

1301 extra="forbid",

1302 )

1303

1304 selection_mode_list: List[str] = Field(

1305 default=["TopK", "All", "Threshold"], alias="selectionModeList"

1306 )

1307 selection_mode: int = Field(default=0, alias="selectionMode")

1308

1309 score_threshold: float = Field(

1310 default=0.0,

1311 alias="scoreThreshold",

1312 description="Routing score threshold used when selectionMode is Threshold (calibration only)",

1313 ge=0,

1314 )

1315

1316 def with_updates(self, **kwargs) -> "MoeConfig":

1317 """Return a copy with updated fields."""

1318 return self.model_copy(update=kwargs)

1319

1320

1321class EquivalentTransformationConfig(BaseModel):

1322 """

1323 @brief Configuration for equivalent transformation techniques

1324

1325 @details Defines parameters for various equivalent transformation methods including

1326 NormConv, QK smoothing, and rotation matrices for improved quantization.

1327

1328 @param seed int. Random seed for transformation

1329 @param apply_hadamard_rotation_matrix bool. Apply Hadamard rotation matrix

1330 @param norm_conv NormConv. NormConv equivalent transformation

1331 @param qk Qk. QK smoothing transformation

1332 @param ud Ud. UD transformation

1333 @param vo Vo. VO transformation

1334 @param feed_forward_multi_lut FeedForwardMultiLut. Feed-forward multi-LUT transformation

1335 @param spin_r1 SpinR1. SpinR1 rotation transformation

1336 @param head_out_ch_rotation HeadOutChRotation. Head output channel rotation transformation

1337 @param in_rotation InRotation. Input rotation transformation

1338 @param spin_r2 SpinR2. SpinR2 rotation transformation

1339 @param qk_rotation QkRotation. QK rotation transformation

1340 @param flatten_quant FlattenQuant. Flatten quantization transformation

1341 @param optimize_ffn OptimizeFfn. FFN optimization

1342 """

1343

1344 model_config = ConfigDict(

1345 populate_by_name=True,

1346 extra="forbid",

1347 )

1348

1349 seed: int = Field(default=0, description="Random seed for transformation")

1350 apply_hadamard_rotation_matrix: bool = Field(

1351 default=True,

1352 alias="applyHadamardRotationMatrix",

1353 description="Apply Hadamard rotation matrix",

1354 )

1355

1356 class NormConv(BaseModel):

1357 model_config = ConfigDict(populate_by_name=True)

1358

1359 """

1360 @brief NormConv equivalent transformation

1361

1362 @param apply bool. Apply NormConv transformation

1363 @param learn bool. Learn transformation parameters

1364 @param smoothing_factor float. Smoothing factor

1365 @param min_gamma float. Minimum gamma value

1366 @param max_gamma float. Maximum gamma value

1367 """

1368 apply: bool = Field(default=False, description="Apply NormConv transformation")

1369 learn: bool = Field(

1370 default=False, description="Learn transformation parameters"

1371 )

1372 smoothing_factor: float = Field(

1373 default=0.5, alias="smoothingFactor", description="Smoothing factor"

1374 )

1375 min_gamma: float = Field(

1376 default=0.0001, alias="minGamma", description="Minimum gamma value"

1377 )

1378 max_gamma: float = Field(

1379 default=10000.0, alias="maxGamma", description="Maximum gamma value"

1380 )

1381

1382 norm_conv: NormConv = Field(default_factory=NormConv, alias="NormConv")

1383

1384 class Qk(BaseModel):

1385 model_config = ConfigDict(populate_by_name=True)

1386

1387 """

1388 @brief QK smoothing transformation

1389

1390 @param apply bool. Apply QK transformation

1391 @param smoothing_factor float. Smoothing factor

1392 @param min_gamma float. Minimum gamma value

1393 @param max_gamma float. Maximum gamma value

1394 """

1395 apply: bool = Field(default=False, description="Apply QK transformation")

1396 smoothing_factor: float = Field(

1397 default=0.5, alias="smoothingFactor", description="Smoothing factor"

1398 )

1399 min_gamma: float = Field(

1400 default=0.0001, alias="minGamma", description="Minimum gamma value"

1401 )

1402 max_gamma: float = Field(

1403 default=10000.0, alias="maxGamma", description="Maximum gamma value"

1404 )

1405

1406 qk: Qk = Field(default_factory=Qk, alias="QK")

1407

1408 class Ud(BaseModel):

1409 model_config = ConfigDict(populate_by_name=True)

1410

1411 """

1412 @brief UD transformation

1413

1414 @param apply bool. Apply UD transformation

1415 @param learn bool. Learn transformation parameters

1416 @param smoothing_factor float. Smoothing factor

1417 @param min_gamma float. Minimum gamma value

1418 @param max_gamma float. Maximum gamma value

1419 """

1420 apply: bool = Field(default=False, description="Apply UD transformation")

1421 learn: bool = Field(

1422 default=False, description="Learn transformation parameters"

1423 )

1424 smoothing_factor: float = Field(

1425 default=0.5, alias="smoothingFactor", description="Smoothing factor"

1426 )

1427 min_gamma: float = Field(

1428 default=0.0001, alias="minGamma", description="Minimum gamma value"

1429 )

1430 max_gamma: float = Field(

1431 default=10000.0, alias="maxGamma", description="Maximum gamma value"

1432 )

1433

1434 ud: Ud = Field(default_factory=Ud, alias="UD")

1435

1436 class Vo(BaseModel):

1437 model_config = ConfigDict(populate_by_name=True)

1438

1439 """

1440 @brief VO transformation

1441

1442 @param apply bool. Apply VO transformation

1443 @param smoothing_factor float. Smoothing factor

1444 @param min_gamma float. Minimum gamma value

1445 @param max_gamma float. Maximum gamma value

1446 """

1447 apply: bool = Field(default=False, description="Apply VO transformation")

1448 smoothing_factor: float = Field(

1449 default=0.5, alias="smoothingFactor", description="Smoothing factor"

1450 )

1451 min_gamma: float = Field(

1452 default=0.0001, alias="minGamma", description="Minimum gamma value"

1453 )

1454 max_gamma: float = Field(

1455 default=10000.0, alias="maxGamma", description="Maximum gamma value"

1456 )

1457

1458 vo: Vo = Field(default_factory=Vo, alias="VO")

1459

1460 class FeedForwardMultiLut(BaseModel):

1461 """

1462 @brief Feed-forward multi-LUT transformation

1463

1464 @param apply bool. Apply feed-forward multi-LUT transformation

1465 @param breakpoints List[float]. Breakpoints for multi-LUT

1466 """

1467

1468 apply: bool = Field(

1469 default=False, description="Apply feed-forward multi-LUT transformation"

1470 )

1471 breakpoints: List[float] = Field(

1472 default=[-8.0, -4.0, 0], description="Breakpoints for multi-LUT"

1473 )

1474

1475 feed_forward_multi_lut: FeedForwardMultiLut = Field(

1476 default_factory=FeedForwardMultiLut, alias="FeedForwardMultiLUT"

1477 )

1478

1479 class SpinR1(BaseModel):

1480 model_config = ConfigDict(populate_by_name=True)

1481

1482 """

1483 @brief SpinR1 rotation transformation

1484

1485 @param apply bool. Apply SpinR1 transformation

1486 @param matrix_path str. Path to rotation matrix file

1487 """

1488 apply: bool = Field(default=False, description="Apply SpinR1 transformation")

1489 matrix_path: str = Field(

1490 default="", alias="matrixPath", description="Path to rotation matrix file"

1491 )

1492

1493 spin_r1: SpinR1 = Field(default_factory=SpinR1, alias="SpinR1")

1494

1495 class HeadOutChRotation(BaseModel):

1496 model_config = ConfigDict(populate_by_name=True)

1497

1498 """

1499 @brief Head output channel rotation transformation

1500

1501 @param apply bool. Apply head output channel rotation

1502 @param matrix_path str. Path to rotation matrix file

1503 """

1504 apply: bool = Field(

1505 default=False, description="Apply head output channel rotation"

1506 )

1507 matrix_path: str = Field(

1508 default="", alias="matrixPath", description="Path to rotation matrix file"

1509 )

1510

1511 head_out_ch_rotation: HeadOutChRotation = Field(

1512 default_factory=HeadOutChRotation, alias="HeadOutChRotation"

1513 )

1514

1515 class InRotation(BaseModel):

1516 model_config = ConfigDict(populate_by_name=True)

1517

1518 """

1519 @brief Input rotation transformation

1520

1521 @param apply bool. Apply input rotation

1522 @param matrix_path str. Path to rotation matrix file

1523 @param input_names List[str]. Names of the input layers to rotate

1524 """

1525 apply: bool = Field(default=False, description="Apply input rotation")

1526 matrix_path: str = Field(

1527 default="", alias="matrixPath", description="Path to rotation matrix file"

1528 )

1529 input_names: List[str] = Field(

1530 default=[],

1531 alias="inputNames",

1532 description="Names of the input layers to rotate",

1533 )

1534

1535 in_rotation: InRotation = Field(default_factory=InRotation, alias="InRotation")

1536

1537 class SpinR2(BaseModel):

1538 model_config = ConfigDict(populate_by_name=True)

1539

1540 """

1541 @brief SpinR2 rotation transformation

1542

1543 @param apply bool. Apply SpinR2 transformation

1544 @param learn bool. Learn rotation matrix

1545 @param matrix_path str. Path to rotation matrix file

1546 """

1547 apply: bool = Field(default=False, description="Apply SpinR2 transformation")

1548 learn: bool = Field(default=False, description="Learn rotation matrix")

1549 matrix_path: str = Field(

1550 default="", alias="matrixPath", description="Path to rotation matrix file"

1551 )

1552

1553 spin_r2: SpinR2 = Field(default_factory=SpinR2, alias="SpinR2")

1554

1555 class QkRotation(BaseModel):

1556 model_config = ConfigDict(populate_by_name=True)

1557

1558 """

1559 @brief QK rotation transformation

1560

1561 @param apply bool. Apply QK rotation transformation

1562 @param matrix_path str. Path to rotation matrix file

1563 """

1564 apply: bool = Field(

1565 default=False, description="Apply QK rotation transformation"

1566 )

1567 matrix_path: str = Field(

1568 default="", alias="matrixPath", description="Path to rotation matrix file"

1569 )

1570

1571 qk_rotation: QkRotation = Field(default_factory=QkRotation, alias="QKRotation")

1572

1573 class FlattenQuant(BaseModel):

1574 model_config = ConfigDict(populate_by_name=True)

1575

1576 """

1577 @brief Flatten quantization transformation

1578

1579 @param apply bool. Apply flatten quantization

1580 @param learn bool. Learn flattening parameters

1581 @param apply_threshold float. Threshold for applying flatten quantization

1582 @param max_overhead float. Maximum overhead allowed for flattening

1583 """

1584 apply: bool = Field(default=False, description="Apply flatten quantization")

1585 learn: bool = Field(default=False, description="Learn flattening parameters")

1586 apply_threshold: float = Field(

1587 default=0.33,

1588 alias="applyThreshold",

1589 description="Threshold for applying flatten quantization",

1590 )

1591 max_overhead: float = Field(

1592 default=0.02,

1593 alias="maxOverhead",

1594 description="Maximum overhead allowed for flattening",

1595 )

1596

1597 flatten_quant: FlattenQuant = Field(

1598 default_factory=FlattenQuant, alias="FlattenQuant"

1599 )

1600

1601 class OptimizeFfn(BaseModel):

1602 model_config = ConfigDict(populate_by_name=True)

1603

1604 """

1605 @brief FFN optimization

1606

1607 @param apply bool. Apply FFN optimization

1608 @param ch_per_ffn int. Optimize FFN split (-1 for auto)

1609 """

1610 apply: bool = Field(default=False, description="Apply FFN optimization")

1611 ch_per_ffn: int = Field(

1612 default=-1, alias="chPerFFN", description="Optimize FFN split (-1 for auto)"

1613 )

1614

1615 optimize_ffn: OptimizeFfn = Field(default_factory=OptimizeFfn, alias="OptimizeFFN")

1616

1617 def with_updates(self, **kwargs) -> "EquivalentTransformationConfig":

1618 """Return a copy with updated fields."""

1619 return self.model_copy(update=kwargs)

1620

1621

1622class SearchWeightScaleConfig(BaseModel):

1623 """

1624 @brief Configuration for weight scale search

1625

1626 @details Defines which transformer components should have their weight scales

1627 searched for optimal quantization.

1628

1629 @param apply bool. If true, apply weight scale search

1630 @param transformer Transformer. Transformer components for weight scale search

1631 """

1632

1633 model_config = ConfigDict(

1634 populate_by_name=True,

1635 extra="forbid",

1636 )

1637

1638 apply: bool = Field(default=False, description="If true, apply weight scale search")

1639

1640 class Transformer(BaseModel):

1641 """

1642 @brief Transformer components for weight scale search

1643

1644 @param query bool. Search weight scale for query

1645 @param key bool. Search weight scale for key

1646 @param value bool. Search weight scale for value

1647 @param out bool. Search weight scale for output

1648 @param ffn bool. Search weight scale for FFN

1649 """

1650

1651 query: bool = Field(default=False, description="Search weight scale for query")

1652 key: bool = Field(default=False, description="Search weight scale for key")

1653 value: bool = Field(default=False, description="Search weight scale for value")

1654 out: bool = Field(default=False, description="Search weight scale for output")

1655 ffn: bool = Field(default=False, description="Search weight scale for FFN")

1656

1657 transformer: Transformer = Field(default_factory=Transformer, alias="transformer")

1658

1659 def with_updates(self, **kwargs) -> "SearchWeightScaleConfig":

1660 """Return a copy with updated fields."""

1661 return self.model_copy(update=kwargs)

1662

1663

1664class LoadScaleConfig(BaseModel):

1665 """

1666 @brief QAT activation scale loading configuration

1667

1668 @details Loads pre-trained activation scales from safetensors files and applies them

1669 to specified layers before scale/zeropoint computation.

1670 NOTE: entries is stored as raw JSON (type: list) because the generator does

1671 not support list[CustomStruct]. The schema records the field shape; parsing

1672 is done manually in applyQATLoadScaleQuantType / applyQATLoadScales.

1673

1674 @param apply bool. If true, load and apply QAT scales from safetensors files

1675 @param entries List. List of file entries. Each entry is a dict:

1676 { path: str, scales: [ { key: str, layers: [str] } ] }

1677 path: safetensors file path; key: tensor name in the file;

1678 layers: layer names whose activation scale will be overridden.

1679

1680 """

1681

1682 model_config = ConfigDict(

1683 populate_by_name=True,

1684 extra="forbid",

1685 )

1686

1687 apply: bool = Field(

1688 default=False,

1689 description="If true, load and apply QAT scales from safetensors files",

1690 )

1691 entries: Any = Field(

1692 default=[],

1693 description="List of file entries. Each entry is a dict: { path: str, scales: [ { key: str, layers: [str] } ] } path: safetensors file path; key: tensor name in the file; layers: layer names whose activation scale will be overridden.",

1694 )

1695

1696 def with_updates(self, **kwargs) -> "LoadScaleConfig":

1697 """Return a copy with updated fields."""

1698 return self.model_copy(update=kwargs)

1699

1700

1701class RuntimeOptions(BaseModel):

1702 """

1703 @brief Runtime options for compilation

1704

1705 @details Contains runtime-specific settings like version info and cache options.

1706

1707 @param version str. Compiler version string (e.g., 0.0.0)

1708 """

1709

1710 model_config = ConfigDict(

1711 populate_by_name=True,

1712 extra="forbid",

1713 )

1714

1715 version: str = Field(

1716 default="0.0.0", description="Compiler version string (e.g., 0.0.0)"

1717 )

1718

1719 def with_updates(self, **kwargs) -> "RuntimeOptions":

1720 """Return a copy with updated fields."""

1721 return self.model_copy(update=kwargs)

1722

1723

1724class SaveSampleConfig(BaseModel):

1725 """

1726 @brief Sample data generation and saving configuration

1727

1728 @param apply bool. Enable sample data saving

1729 @param mode str. Inference mode: infer (standard) or inferWithCache (LLM cache models)

1730 @param batch_size int. Number of inference batches to generate

1731 @param batch_seq_lens List. Per-batch step-wise sequence lengths for inferWithCache mode. e.g. [[80, 1], [240, 10]]

1732 @param save_folder str. Output folder for sample data

1733 @param dtype str. Data type for saved samples: float or int8

1734 """

1735

1736 model_config = ConfigDict(

1737 populate_by_name=True,

1738 extra="forbid",

1739 )

1740

1741 apply: bool = Field(default=False, description="Enable sample data saving")

1742 mode: str = Field(

1743 default="infer",

1744 description="Inference mode: infer (standard) or inferWithCache (LLM cache models)",

1745 )

1746 batch_size: int = Field(

1747 default=1,

1748 alias="batchSize",

1749 description="Number of inference batches to generate",

1750 )

1751 batch_seq_lens: Any = Field(

1752 default=[],

1753 alias="batchSeqLens",

1754 description="Per-batch step-wise sequence lengths for inferWithCache mode. e.g. [[80, 1], [240, 10]]",

1755 )

1756 save_folder: str = Field(

1757 default="sampleInout",

1758 alias="saveFolder",

1759 description="Output folder for sample data",

1760 )

1761 dtype: str = Field(

1762 default="float", description="Data type for saved samples: float or int8"

1763 )

1764

1765 def with_updates(self, **kwargs) -> "SaveSampleConfig":

1766 """Return a copy with updated fields."""

1767 return self.model_copy(update=kwargs)

1768

1769

1770class GroupWiseConfig(BaseModel):

1771 """

1772 @brief Group-wise streaming quantization configuration for large LLMs

1773

1774 @details Configuration for the group-wise (streaming) quantization pipeline used for

1775 LLMs that do not fit fully on GPU. Partitions the model into groups of

1776 transformer blocks and quantizes each group independently.

1777

1778 @param apply bool. Enable group-wise streaming quantization pipeline

1779 @param group_size int. Group size in number of transformer blocks (0 = auto)

1780 @param gpu_budget_gb float. GPU memory budget in GiB for group-wise execution (0 = auto-detect)

1781 @param gpu_safety_margin_gb float. Safety margin in GiB subtracted from detected GPU budget

1782 @param cache_dir str. Directory for group-wise activation/state cache (empty = system temp)

1783 @param keep_cache bool. Retain group-wise cache after run completes (for debugging)

1784 @param partition_policy str. Partitioning policy (e.g. transformer_block, moe_expert_subgroup)

1785 @param expert_groups List[List[int]]. MoE fallback partitioner: list of expert-index lists (List[List[int]]).

1786 @param retain_topology_weights bool. Keep inflated FP weights across groups (false = release after each group for tight memory budgets).

1787 @param checkpoint bool. Save per-group checkpoint so quantization can resume from the last completed group after a crash.

1788 """

1789

1790 model_config = ConfigDict(

1791 populate_by_name=True,

1792 extra="forbid",

1793 )

1794

1795 apply: bool = Field(

1796 default=False, description="Enable group-wise streaming quantization pipeline"

1797 )

1798 group_size: int = Field(

1799 default=0,

1800 alias="groupSize",

1801 description="Group size in number of transformer blocks (0 = auto)",

1802 )

1803 gpu_budget_gb: float = Field(

1804 default=0,

1805 alias="gpuBudgetGb",

1806 description="GPU memory budget in GiB for group-wise execution (0 = auto-detect)",

1807 )

1808 gpu_safety_margin_gb: float = Field(

1809 default=4,

1810 alias="gpuSafetyMarginGb",

1811 description="Safety margin in GiB subtracted from detected GPU budget",

1812 )

1813 cache_dir: str = Field(

1814 default="",

1815 alias="cacheDir",

1816 description="Directory for group-wise activation/state cache (empty = system temp)",

1817 )

1818 keep_cache: bool = Field(

1819 default=False,

1820 alias="keepCache",

1821 description="Retain group-wise cache after run completes (for debugging)",

1822 )

1823 partition_policy: str = Field(

1824 default="transformer_block",

1825 alias="partitionPolicy",

1826 description="Partitioning policy (e.g. transformer_block, moe_expert_subgroup)",

1827 )

1828 expert_groups: Any = Field(

1829 default=[],

1830 alias="expertGroups",

1831 description="MoE fallback partitioner: list of expert-index lists (List[List[int]]).",

1832 )

1833 retain_topology_weights: bool = Field(

1834 default=True,

1835 alias="retainTopologyWeights",

1836 description="Keep inflated FP weights across groups (false = release after each group for tight memory budgets).",

1837 )

1838 checkpoint: bool = Field(

1839 default=False,

1840 description="Save per-group checkpoint so quantization can resume from the last completed group after a crash.",

1841 )

1842

1843 def with_updates(self, **kwargs) -> "GroupWiseConfig":

1844 """Return a copy with updated fields."""

1845 return self.model_copy(update=kwargs)

1846

1847

1848class CompileConfig(BaseModel):

1849 """Unified compilation configuration for Mobilint MXQ compilation."""

1850

1851 model_config = ConfigDict(

1852 populate_by_name=True,

1853 extra="forbid",

1854 )

1855

1856 model_paths: List[str] = Field(

1857 default=[], alias="modelPaths", description="Paths to model files"

1858 )

1859 calib_data_path: List[str] = Field(

1860 default=[], alias="calibDataPaths", description="Paths to calibration datasets"

1861 )

1862 save_paths: List[str] = Field(

1863 default=["./tmp.mxq"],

1864 alias="savePaths",

1865 description="Output MXQ filename/paths",

1866 )

1867 use_random_calib: bool = Field(

1868 default=False, alias="useRandomCalib", description="Use random calibration"

1869 )

1870 inference_scheme: str = Field(

1871 default="single", alias="inferenceScheme", description="NPU inference scheme"

1872 )

1873 cpu_offload: bool = Field(

1874 default=False,

1875 alias="cpuOffload",

1876 description="Enable CPU offload for unsupported operators",

1877 )

1878 force_npu_input_reposition: bool = Field(

1879 default=False,

1880 alias="forceNpuInputReposition",

1881 description="Force input reposition operations to run on NPU instead of CPU",

1882 )

1883 force_npu_output_reposition: bool = Field(

1884 default=False,

1885 alias="forceNpuOutputReposition",

1886 description="Force output reposition operations to run on NPU instead of CPU",

1887 )

1888 optimize_option: int = Field(

1889 default=1,

1890 alias="optimizeOption",

1891 description="Compiler optimization selector",

1892 ge=0,

1893 )

1894 buffer_mode: int = Field(

1895 default=1, alias="bufferMode", description="Buffer serialization mode"

1896 )

1897 input_shape_dict: Any = Field(

1898 default={}, alias="inputShapeDict", description="Dictionary of input shapes"

1899 )

1900 device: str = Field(default="gpu", description="Device for computation")

1901 dtype: str = Field(default="float", description="Data type for computation")

1902 debug: bool = Field(default=False, description="Enable debug mode")

1903 trace: bool = Field(default=False, description="Enable trace mode")

1904 image_channels: int = Field(

1905 default=0,

1906 alias="imageChannels",

1907 description="Number of image channels (0 for auto-detect)",

1908 )

1909 config_version: str = Field(

1910 default="1.0.0", alias="configVersion", description="Config schema version"

1911 )

1912 split_blocks: List[int] = Field(

1913 default=[],

1914 alias="splitBlocks",

1915 description="Multi-MXQ split points by transformer block index",

1916 )

1917 split_parts: int = Field(

1918 default=0,

1919 alias="splitParts",

1920 description="Evenly split transformer blocks into N MXQ parts",

1921 )

1922

1923 uint8_input: Uint8InputConfig = Field(

1924 default_factory=Uint8InputConfig, alias="uint8Input"

1925 )

1926 preprocessing: PreprocessingConfig = Field(default_factory=PreprocessingConfig)

1927 resource_management: ResourceManagementConfig = Field(

1928 default_factory=ResourceManagementConfig, alias="resourceManagement"

1929 )

1930 calibration: CalibrationConfig = Field(default_factory=CalibrationConfig)

1931 bit: BitConfig = Field(default_factory=BitConfig)

1932 hessian_quant: HessianQuantConfig = Field(

1933 default_factory=HessianQuantConfig, alias="hessianQuant"

1934 )

1935 layer_bias_correction: LayerBiasCorrectionConfig = Field(

1936 default_factory=LayerBiasCorrectionConfig, alias="layerBiasCorrection"

1937 )

1938 mod: ModConfig = Field(default_factory=ModConfig)

1939 llm: LlmConfig = Field(default_factory=LlmConfig)

1940 moe: MoeConfig = Field(default_factory=MoeConfig)

1941 equivalent_transformation: EquivalentTransformationConfig = Field(

1942 default_factory=EquivalentTransformationConfig, alias="equivalentTransformation"

1943 )

1944 search_weight_scale: SearchWeightScaleConfig = Field(

1945 default_factory=SearchWeightScaleConfig, alias="searchWeightScale"

1946 )

1947 load_scale: LoadScaleConfig = Field(

1948 default_factory=LoadScaleConfig, alias="loadScale"

1949 )

1950 runtime_options: RuntimeOptions = Field(

1951 default_factory=RuntimeOptions, alias="runtimeOptions"

1952 )

1953 save_sample: SaveSampleConfig = Field(

1954 default_factory=SaveSampleConfig, alias="saveSample"

1955 )

1956 group_wise: GroupWiseConfig = Field(

1957 default_factory=GroupWiseConfig, alias="groupWise"

1958 )

1959

1960 def with_uint8_input(self, **kwargs) -> "CompileConfig":

1961 """Return a copy with uint8_input settings enabled."""

1962 data = {"apply": True, **kwargs}

1963 new_cfg = self.uint8_input.model_copy(update=data)

1964 return self.model_copy(update={"uint8_input": new_cfg})

1965

1966 def with_preprocessing(self, **kwargs) -> "CompileConfig":

1967 """Return a copy with preprocessing settings enabled."""

1968 data = {"apply": True, **kwargs}

1969 new_cfg = self.preprocessing.model_copy(update=data)

1970 return self.model_copy(update={"preprocessing": new_cfg})

1971

1972 def with_hessian_quant(self, **kwargs) -> "CompileConfig":

1973 """Return a copy with hessian_quant settings enabled."""

1974 data = {"apply": True, **kwargs}

1975 new_cfg = self.hessian_quant.model_copy(update=data)

1976 return self.model_copy(update={"hessian_quant": new_cfg})

1977

1978 def with_layer_bias_correction(self, **kwargs) -> "CompileConfig":

1979 """Return a copy with layer_bias_correction settings enabled."""

1980 data = {"apply": True, **kwargs}

1981 new_cfg = self.layer_bias_correction.model_copy(update=data)

1982 return self.model_copy(update={"layer_bias_correction": new_cfg})

1983

1984 def with_mod(self, **kwargs) -> "CompileConfig":

1985 """Return a copy with mod settings enabled."""

1986 data = {"apply": True, **kwargs}

1987 new_cfg = self.mod.model_copy(update=data)

1988 return self.model_copy(update={"mod": new_cfg})

1989

1990 def with_llm(self, **kwargs) -> "CompileConfig":

1991 """Return a copy with llm settings enabled."""

1992 data = {"apply": True, **kwargs}

1993 new_cfg = self.llm.model_copy(update=data)

1994 return self.model_copy(update={"llm": new_cfg})

1995

1996 def with_search_weight_scale(self, **kwargs) -> "CompileConfig":

1997 """Return a copy with search_weight_scale settings enabled."""

1998 data = {"apply": True, **kwargs}

1999 new_cfg = self.search_weight_scale.model_copy(update=data)

2000 return self.model_copy(update={"search_weight_scale": new_cfg})

2001

2002 def with_load_scale(self, **kwargs) -> "CompileConfig":

2003 """Return a copy with load_scale settings enabled."""

2004 data = {"apply": True, **kwargs}

2005 new_cfg = self.load_scale.model_copy(update=data)

2006 return self.model_copy(update={"load_scale": new_cfg})

2007

2008 def with_save_sample(self, **kwargs) -> "CompileConfig":

2009 """Return a copy with save_sample settings enabled."""

2010 data = {"apply": True, **kwargs}

2011 new_cfg = self.save_sample.model_copy(update=data)

2012 return self.model_copy(update={"save_sample": new_cfg})

2013

2014 def with_group_wise(self, **kwargs) -> "CompileConfig":

2015 """Return a copy with group_wise settings enabled."""

2016 data = {"apply": True, **kwargs}

2017 new_cfg = self.group_wise.model_copy(update=data)

2018 return self.model_copy(update={"group_wise": new_cfg})

2019

2020 @classmethod

2021 def from_file(cls, path: Union[str, Path]) -> "CompileConfig":

2022 """Load config from YAML or JSON file."""

2023 path = Path(path)

2024 with open(path) as f:

2025 if path.suffix in (".yaml", ".yml"):

2026 data = yaml.safe_load(f)

2027 else:

2028 data = json.load(f)

2029 data = cls._flatten_grouped_json(data)

2030 return cls.model_validate(data)

2031

2032 @staticmethod

2033 def _flatten_grouped_json(data: dict) -> dict:

2034 """Flatten grouped JSON keys (e.g. quantization.calibration) to flat structure."""

2035 data = data.copy()

2036 if "quantization" in data:

2037 group = data.pop("quantization")

2038 if "calibration" in group:

2039 data["calibration"] = group["calibration"]

2040 if "bit" in group:

2041 data["bit"] = group["bit"]

2042 if "advancedQuantization" in data:

2043 group = data.pop("advancedQuantization")

2044 if "hessianQuant" in group:

2045 data["hessianQuant"] = group["hessianQuant"]

2046 if "layerBiasCorrection" in group:

2047 data["layerBiasCorrection"] = group["layerBiasCorrection"]

2048 if "mod" in group:

2049 data["mod"] = group["mod"]

2050 if "EquivalentTransformation" in group:

2051 data["equivalentTransformation"] = group["EquivalentTransformation"]

2052 if "searchWeightScale" in group:

2053 data["searchWeightScale"] = group["searchWeightScale"]

2054 if "loadScale" in group:

2055 data["loadScale"] = group["loadScale"]

2056 return data

2057

2058 @classmethod

2059 def from_preset(cls, name: str) -> "CompileConfig":

2060 """Load config from a preset."""

2061 from .presets import get_preset

2062

2063 return get_preset(name)

2064

2065 @staticmethod

2066 def _group_to_json(data: dict) -> dict:

2067 """Group flat keys back into nested JSON structure (inverse of _flatten_grouped_json)."""

2068 data = data.copy()

2069 quantization_group = {}

2070 if "calibration" in data:

2071 quantization_group["calibration"] = data.pop("calibration")

2072 if "bit" in data:

2073 quantization_group["bit"] = data.pop("bit")

2074 if quantization_group:

2075 data["quantization"] = quantization_group

2076 advancedQuantization_group = {}

2077 if "hessianQuant" in data:

2078 advancedQuantization_group["hessianQuant"] = data.pop("hessianQuant")

2079 if "layerBiasCorrection" in data:

2080 advancedQuantization_group["layerBiasCorrection"] = data.pop(

2081 "layerBiasCorrection"

2082 )

2083 if "mod" in data:

2084 advancedQuantization_group["mod"] = data.pop("mod")

2085 if "equivalentTransformation" in data:

2086 advancedQuantization_group["EquivalentTransformation"] = data.pop(

2087 "equivalentTransformation"

2088 )

2089 if "searchWeightScale" in data:

2090 advancedQuantization_group["searchWeightScale"] = data.pop(

2091 "searchWeightScale"

2092 )

2093 if "loadScale" in data:

2094 advancedQuantization_group["loadScale"] = data.pop("loadScale")

2095 if advancedQuantization_group:

2096 data["advancedQuantization"] = advancedQuantization_group

2097 return data

2098

2099 def to_file(self, path: Union[str, Path]) -> None:

2100 """Save config to YAML or JSON file."""

2101 path = Path(path)

2102 data = self.model_dump(by_alias=True, exclude_none=True)

2103 data = self._group_to_json(data)

2104 with open(path, "w") as f:

2105 if path.suffix in (".yaml", ".yml"):

2106 yaml.dump(data, f, default_flow_style=False)

2107 else:

2108 json.dump(data, f, indent=2)

qbcompiler.configs.models.BitConfig.LayerOverrides

Definition models.py:739

qbcompiler.configs.models.BitConfig.SaveInfo

Definition models.py:717

qbcompiler.configs.models.BitConfig.Transformer.Activation.Ffn

FFN activation bit-widths (int shorthand sets all sublayers)

Definition models.py:532

qbcompiler.configs.models.BitConfig.Transformer.Activation

Definition models.py:509

qbcompiler.configs.models.BitConfig.Transformer.MixedPrecision.Activation

Definition models.py:676

qbcompiler.configs.models.BitConfig.Transformer.MixedPrecision.Weight

Definition models.py:629

qbcompiler.configs.models.BitConfig.Transformer.MixedPrecision

Definition models.py:619

qbcompiler.configs.models.BitConfig.Transformer.Weight.Ffn

FFN weight bit-widths (int shorthand sets all sublayers)

Definition models.py:587

qbcompiler.configs.models.BitConfig.Transformer.Weight

Definition models.py:564

qbcompiler.configs.models.BitConfig.Transformer

Definition models.py:498

qbcompiler.configs.models.BitConfig

Configuration for bit precision.

Definition models.py:481

qbcompiler.configs.models.BitConfig.with_updates

"BitConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:769

qbcompiler.configs.models.CalibrationConfig.FastDist

Definition models.py:281

qbcompiler.configs.models.CalibrationConfig.GroupLut

Definition models.py:426

qbcompiler.configs.models.CalibrationConfig.Histogram

Definition models.py:297

qbcompiler.configs.models.CalibrationConfig.LayerOverrides

Definition models.py:363

qbcompiler.configs.models.CalibrationConfig.MaxPercentile

Definition models.py:240

qbcompiler.configs.models.CalibrationConfig.OptimizeLut

Definition models.py:458

qbcompiler.configs.models.CalibrationConfig.Statistics

Definition models.py:391

qbcompiler.configs.models.CalibrationConfig

Configuration for calibration during quantization.

Definition models.py:141

qbcompiler.configs.models.CalibrationConfig.with_updates

"CalibrationConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:476

qbcompiler.configs.models.CompileConfig

Unified compilation configuration for Mobilint MXQ compilation.

Definition models.py:1848

qbcompiler.configs.models.CompileConfig.with_llm

"CompileConfig" with_llm(self, **kwargs)

Return a copy with llm settings enabled.

Definition models.py:1990

qbcompiler.configs.models.CompileConfig.preprocessing

PreprocessingConfig preprocessing

Definition models.py:1926

qbcompiler.configs.models.CompileConfig.from_preset

"CompileConfig" from_preset(cls, str name)

Load config from a preset.

Definition models.py:2059

qbcompiler.configs.models.CompileConfig.with_search_weight_scale

"CompileConfig" with_search_weight_scale(self, **kwargs)

Return a copy with search_weight_scale settings enabled.

Definition models.py:1996

qbcompiler.configs.models.CompileConfig.load_scale

LoadScaleConfig load_scale

Definition models.py:1947

qbcompiler.configs.models.CompileConfig.uint8_input

Uint8InputConfig uint8_input

Definition models.py:1923

qbcompiler.configs.models.CompileConfig.search_weight_scale

SearchWeightScaleConfig search_weight_scale

Definition models.py:1944

qbcompiler.configs.models.CompileConfig.hessian_quant

HessianQuantConfig hessian_quant

Definition models.py:1932

qbcompiler.configs.models.CompileConfig.with_layer_bias_correction

"CompileConfig" with_layer_bias_correction(self, **kwargs)

Return a copy with layer_bias_correction settings enabled.

Definition models.py:1978

qbcompiler.configs.models.CompileConfig.layer_bias_correction

LayerBiasCorrectionConfig layer_bias_correction

Definition models.py:1935

qbcompiler.configs.models.CompileConfig.with_load_scale

"CompileConfig" with_load_scale(self, **kwargs)

Return a copy with load_scale settings enabled.

Definition models.py:2002

qbcompiler.configs.models.CompileConfig.save_sample

SaveSampleConfig save_sample

Definition models.py:1953

qbcompiler.configs.models.CompileConfig.with_preprocessing

"CompileConfig" with_preprocessing(self, **kwargs)

Return a copy with preprocessing settings enabled.

Definition models.py:1966

qbcompiler.configs.models.CompileConfig._flatten_grouped_json

dict _flatten_grouped_json(dict data)

Flatten grouped JSON keys (e.g.

Definition models.py:2033

qbcompiler.configs.models.CompileConfig.llm

LlmConfig llm

Definition models.py:1939

qbcompiler.configs.models.CompileConfig.with_mod

"CompileConfig" with_mod(self, **kwargs)

Return a copy with mod settings enabled.

Definition models.py:1984

qbcompiler.configs.models.CompileConfig.with_hessian_quant

"CompileConfig" with_hessian_quant(self, **kwargs)

Return a copy with hessian_quant settings enabled.

Definition models.py:1972

qbcompiler.configs.models.CompileConfig._group_to_json

dict _group_to_json(dict data)

Group flat keys back into nested JSON structure (inverse of _flatten_grouped_json).

Definition models.py:2066

qbcompiler.configs.models.CompileConfig.with_save_sample

"CompileConfig" with_save_sample(self, **kwargs)

Return a copy with save_sample settings enabled.

Definition models.py:2008

qbcompiler.configs.models.CompileConfig.mod

ModConfig mod

Definition models.py:1938

qbcompiler.configs.models.CompileConfig.from_file

"CompileConfig" from_file(cls, Union[str, Path] path)

Load config from YAML or JSON file.

Definition models.py:2021

qbcompiler.configs.models.CompileConfig.to_file

None to_file(self, Union[str, Path] path)

Save config to YAML or JSON file.

Definition models.py:2099

qbcompiler.configs.models.CompileConfig.with_uint8_input

"CompileConfig" with_uint8_input(self, **kwargs)

Return a copy with uint8_input settings enabled.

Definition models.py:1960

qbcompiler.configs.models.EquivalentTransformationConfig.FeedForwardMultiLut

Feed-forward multi-LUT transformation.

Definition models.py:1460

qbcompiler.configs.models.EquivalentTransformationConfig.FlattenQuant

Definition models.py:1573

qbcompiler.configs.models.EquivalentTransformationConfig.HeadOutChRotation

Definition models.py:1495

qbcompiler.configs.models.EquivalentTransformationConfig.InRotation

Definition models.py:1515

qbcompiler.configs.models.EquivalentTransformationConfig.NormConv

Definition models.py:1356

qbcompiler.configs.models.EquivalentTransformationConfig.OptimizeFfn

Definition models.py:1601

qbcompiler.configs.models.EquivalentTransformationConfig.QkRotation

Definition models.py:1555

qbcompiler.configs.models.EquivalentTransformationConfig.Qk

Definition models.py:1384

qbcompiler.configs.models.EquivalentTransformationConfig.SpinR1

Definition models.py:1479

qbcompiler.configs.models.EquivalentTransformationConfig.SpinR2

Definition models.py:1537

qbcompiler.configs.models.EquivalentTransformationConfig.Ud

Definition models.py:1408

qbcompiler.configs.models.EquivalentTransformationConfig.Vo

Definition models.py:1436

qbcompiler.configs.models.EquivalentTransformationConfig

Configuration for equivalent transformation techniques.

Definition models.py:1321

qbcompiler.configs.models.EquivalentTransformationConfig.with_updates

"EquivalentTransformationConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:1617

qbcompiler.configs.models.HessianQuantConfig.Attributes

Definition models.py:809

qbcompiler.configs.models.HessianQuantConfig

Configuration for HessianQuant algorithm.

Definition models.py:774

qbcompiler.configs.models.HessianQuantConfig.with_updates

"HessianQuantConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:845

qbcompiler.configs.models.LayerBiasCorrectionConfig.Attributes

Definition models.py:874

qbcompiler.configs.models.LayerBiasCorrectionConfig

Configuration for calibration-derived layer bias correction.

Definition models.py:850

qbcompiler.configs.models.LayerBiasCorrectionConfig.with_updates

"LayerBiasCorrectionConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:903

qbcompiler.configs.models.LlmConfig.Attributes.Calibration

Definition models.py:1197

qbcompiler.configs.models.LlmConfig.Attributes.Debug

Definition models.py:1257

qbcompiler.configs.models.LlmConfig.Attributes.Runtime

Definition models.py:1221

qbcompiler.configs.models.LlmConfig.Attributes

Definition models.py:1166

qbcompiler.configs.models.LlmConfig

Configuration for Large Language Model (LLM) compilation.

Definition models.py:1140

qbcompiler.configs.models.LlmConfig.with_updates

"LlmConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:1277

qbcompiler.configs.models.LoadScaleConfig

Definition models.py:1664

qbcompiler.configs.models.LoadScaleConfig.with_updates

"LoadScaleConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:1696

qbcompiler.configs.models.ModConfig.Attributes.LearningRates

Definition models.py:985

qbcompiler.configs.models.ModConfig.Attributes.Loss

Definition models.py:1055

qbcompiler.configs.models.ModConfig.Attributes.PostProcessing

Definition models.py:1111

qbcompiler.configs.models.ModConfig.Attributes.Training

Definition models.py:1017

qbcompiler.configs.models.ModConfig.Attributes

Definition models.py:926

qbcompiler.configs.models.ModConfig

Configuration for Minimum Output Difference algorithm.

Definition models.py:908

qbcompiler.configs.models.ModConfig.with_updates

"ModConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:1135

qbcompiler.configs.models.MoeConfig

Sparse MoE expert-selection configuration (calibration only)

Definition models.py:1282

qbcompiler.configs.models.MoeConfig.with_updates

"MoeConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:1316

qbcompiler.configs.models.PreprocessingConfig

Configuration for input preprocessing pipeline.

Definition models.py:47

qbcompiler.configs.models.PreprocessingConfig.with_updates

"PreprocessingConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:82

qbcompiler.configs.models.ResourceManagementConfig.WeightMemory

Weight memory management configuration.

Definition models.py:114

qbcompiler.configs.models.ResourceManagementConfig

Configuration for resource management during model compilation.

Definition models.py:87

qbcompiler.configs.models.ResourceManagementConfig.with_updates

"ResourceManagementConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:136

qbcompiler.configs.models.RuntimeOptions

Runtime options for compilation.

Definition models.py:1701

qbcompiler.configs.models.RuntimeOptions.with_updates

"RuntimeOptions" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:1719

qbcompiler.configs.models.SaveSampleConfig

Sample data generation and saving configuration.

Definition models.py:1724

qbcompiler.configs.models.SaveSampleConfig.with_updates

"SaveSampleConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:1765

qbcompiler.configs.models.SearchWeightScaleConfig.Transformer

Transformer components for weight scale search.

Definition models.py:1640

qbcompiler.configs.models.SearchWeightScaleConfig

Configuration for weight scale search.

Definition models.py:1622

qbcompiler.configs.models.SearchWeightScaleConfig.with_updates

"SearchWeightScaleConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:1659

qbcompiler.configs.models.Uint8InputConfig

Configuration for uint8 input handling.

Definition models.py:13

qbcompiler.configs.models.Uint8InputConfig.with_updates

"Uint8InputConfig" with_updates(self, **kwargs)

Return a copy with updated fields.

Definition models.py:42

---

Generated by  1.14.0