model.h Source File

model.h Source File#

SDK qb Runtime Library: model.h Source File
SDK qb Runtime Library v1.5
MCS001-KR
model.h
Go to the documentation of this file.
1// Copyright ⓒ 2019- Mobilint, Inc. All rights reserved.
5
6#ifndef QBRUNTIME_MODEL_H_
7#define QBRUNTIME_MODEL_H_
8
9#include <cstdint>
10#ifndef _MSC_VER
11#include <experimental/propagate_const>
12#endif
13#include <memory>
14#include <string>
15#include <vector>
16
17#include "qbruntime/export.h"
18#include "qbruntime/future.h"
19#include "qbruntime/model_variant_handle.h"
20#include "qbruntime/ndarray.h"
21#include "qbruntime/npu_data.h"
22#include "qbruntime/pinned_memory.h"
23#include "qbruntime/status_code.h"
24#include "qbruntime/type.h"
25
26namespace mobilint {
27
32
33class Accelerator;
34class ModelImpl;
35
42class QBRUNTIME_EXPORT Model {
43public:
58 static std::unique_ptr<Model> create(const std::string& mxq_path, StatusCode& sc);
59
75 static std::unique_ptr<Model> create(const std::string& mxq_path,
76 const ModelConfig& config, StatusCode& sc);
77
78 Model(const Model& other) = delete;
79 Model(Model&& other) noexcept;
80 Model& operator=(const Model& rhs) = delete;
81 Model& operator=(Model&& rhs) noexcept;
82 ~Model();
83
92 StatusCode launch(Accelerator& acc);
93
102 StatusCode dispose();
103
109 CoreMode getCoreMode() const;
110
119 bool isTarget(CoreId core_id) const;
120
126 std::vector<CoreId> getTargetCores() const;
127
145
156 StatusCode infer(const std::vector<NDArray<float>>& input,
157 std::vector<NDArray<float>>& output);
158
170 std::vector<NDArray<float>> infer(const std::vector<NDArray<float>>& input,
171 StatusCode& sc);
172
184 StatusCode infer(const std::vector<float*>& input,
185 std::vector<std::vector<float>>& output);
186
201 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
202 StatusCode& sc);
203
220 StatusCode infer(const std::vector<float*>& input,
221 std::vector<std::vector<float>>& output,
222 const std::vector<std::vector<int64_t>>& shape);
239 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
240 const std::vector<std::vector<int64_t>>& shape,
241 StatusCode& sc);
242
256 StatusCode infer(const std::vector<NDArray<float>>& input,
257 std::vector<NDArray<float>>& output, uint32_t cache_size);
258
271 StatusCode infer(const std::vector<NDArray<float>>& input,
272 std::vector<NDArray<float>>& output,
273 const std::vector<BatchParam>& params);
274
291 std::vector<NDArray<float>> infer(const std::vector<NDArray<float>>& input,
292 uint32_t cache_size, StatusCode& sc);
293
309 std::vector<NDArray<float>> infer(const std::vector<NDArray<float>>& input,
310 const std::vector<BatchParam>& params,
311 StatusCode& sc);
312
328 StatusCode infer(const std::vector<float*>& input,
329 std::vector<std::vector<float>>& output,
330 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size);
331
350 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
351 const std::vector<std::vector<int64_t>>& shape,
352 uint32_t cache_size, StatusCode& sc);
353
368 StatusCode infer(const std::vector<float*>& input,
369 std::vector<std::vector<float>>& output,
370 const std::vector<std::vector<int64_t>>& shape,
371 const std::vector<BatchParam>& params);
372
390 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
391 const std::vector<std::vector<int64_t>>& shape,
392 const std::vector<BatchParam>& params,
393 StatusCode& sc);
394
396
421
432 StatusCode inferCHW(const std::vector<NDArray<float>>& input,
433 std::vector<NDArray<float>>& output);
434
446 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<float>>& input,
447 StatusCode& sc);
448
460 StatusCode inferCHW(const std::vector<float*>& input,
461 std::vector<std::vector<float>>& output);
462
477 std::vector<std::vector<float>> inferCHW(const std::vector<float*>& input,
478 StatusCode& sc);
479
496 StatusCode inferCHW(const std::vector<float*>& input,
497 std::vector<std::vector<float>>& output,
498 const std::vector<std::vector<int64_t>>& shape);
499
516 std::vector<std::vector<float>> inferCHW(
517 const std::vector<float*>& input, const std::vector<std::vector<int64_t>>& shape,
518 StatusCode& sc);
519
533 StatusCode inferCHW(const std::vector<NDArray<float>>& input,
534 std::vector<NDArray<float>>& output, uint32_t cache_size);
535
552 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<float>>& input,
553 uint32_t cache_size, StatusCode& sc);
554
570 StatusCode inferCHW(const std::vector<float*>& input,
571 std::vector<std::vector<float>>& output,
572 const std::vector<std::vector<int64_t>>& shape,
573 uint32_t cache_size);
574
593 std::vector<std::vector<float>> inferCHW(
594 const std::vector<float*>& input, const std::vector<std::vector<int64_t>>& shape,
595 uint32_t cache_size, StatusCode& sc);
596
598
612
613 StatusCode infer(const std::vector<NDArray<uint8_t>>& input,
614 std::vector<NDArray<float>>& output);
615 std::vector<NDArray<float>> infer(const std::vector<NDArray<uint8_t>>& input,
616 StatusCode& sc);
617 StatusCode infer(const std::vector<uint8_t*>& input,
618 std::vector<std::vector<float>>& output);
619 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
620 StatusCode& sc);
621 StatusCode infer(const std::vector<uint8_t*>& input,
622 std::vector<std::vector<float>>& output,
623 const std::vector<std::vector<int64_t>>& shape);
624 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
625 const std::vector<std::vector<int64_t>>& shape,
626 StatusCode& sc);
627
628 StatusCode infer(const std::vector<NDArray<uint8_t>>& input,
629 std::vector<NDArray<float>>& output, uint32_t cache_size);
630 std::vector<NDArray<float>> infer(const std::vector<NDArray<uint8_t>>& input,
631 uint32_t cache_size, StatusCode& sc);
632 StatusCode infer(const std::vector<uint8_t*>& input,
633 std::vector<std::vector<float>>& output,
634 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size);
635 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
636 const std::vector<std::vector<int64_t>>& shape,
637 uint32_t cache_size, StatusCode& sc);
638
639 StatusCode infer(const std::vector<NDArray<uint8_t>>& input,
640 std::vector<NDArray<float>>& output,
641 const std::vector<BatchParam>& params);
642 std::vector<NDArray<float>> infer(const std::vector<NDArray<uint8_t>>& input,
643 const std::vector<BatchParam>& params,
644 StatusCode& sc);
645 StatusCode infer(const std::vector<uint8_t*>& input,
646 std::vector<std::vector<float>>& output,
647 const std::vector<std::vector<int64_t>>& shape,
648 const std::vector<BatchParam>& params);
649 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
650 const std::vector<std::vector<int64_t>>& shape,
651 const std::vector<BatchParam>& params,
652 StatusCode& sc);
653
655
664 StatusCode inferCHW(const std::vector<NDArray<uint8_t>>& input,
665 std::vector<NDArray<float>>& output);
666 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<uint8_t>>& input,
667 StatusCode& sc);
668 StatusCode inferCHW(const std::vector<uint8_t*>& input,
669 std::vector<std::vector<float>>& output);
670 std::vector<std::vector<float>> inferCHW(const std::vector<uint8_t*>& input,
671 StatusCode& sc);
672 StatusCode inferCHW(const std::vector<uint8_t*>& input,
673 std::vector<std::vector<float>>& output,
674 const std::vector<std::vector<int64_t>>& shape);
675 std::vector<std::vector<float>> inferCHW(
676 const std::vector<uint8_t*>& input,
677 const std::vector<std::vector<int64_t>>& shape, StatusCode& sc);
678
679 StatusCode inferCHW(const std::vector<NDArray<uint8_t>>& input,
680 std::vector<NDArray<float>>& output, uint32_t cache_size);
681 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<uint8_t>>& input,
682 uint32_t cache_size, StatusCode& sc);
683 StatusCode inferCHW(const std::vector<uint8_t*>& input,
684 std::vector<std::vector<float>>& output,
685 const std::vector<std::vector<int64_t>>& shape,
686 uint32_t cache_size);
687 std::vector<std::vector<float>> inferCHW(
688 const std::vector<uint8_t*>& input,
689 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size,
690 StatusCode& sc);
692
711
712 StatusCode infer(const std::vector<NDArray<int8_t>>& input,
713 std::vector<NDArray<int8_t>>& output);
714 std::vector<NDArray<int8_t>> infer(const std::vector<NDArray<int8_t>>& input,
715 StatusCode& sc);
716 StatusCode infer(const std::vector<int8_t*>& input,
717 std::vector<std::vector<int8_t>>& output);
718 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
719 StatusCode& sc);
720 StatusCode infer(const std::vector<int8_t*>& input,
721 std::vector<std::vector<int8_t>>& output,
722 const std::vector<std::vector<int64_t>>& shape);
723 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
724 const std::vector<std::vector<int64_t>>& shape,
725 StatusCode& sc);
726
727 StatusCode infer(const std::vector<NDArray<int8_t>>& input,
728 std::vector<NDArray<int8_t>>& output, uint32_t cache_size);
729 std::vector<NDArray<int8_t>> infer(const std::vector<NDArray<int8_t>>& input,
730 uint32_t cache_size, StatusCode& sc);
731 StatusCode infer(const std::vector<int8_t*>& input,
732 std::vector<std::vector<int8_t>>& output,
733 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size);
734 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
735 const std::vector<std::vector<int64_t>>& shape,
736 uint32_t cache_size, StatusCode& sc);
737
738 StatusCode infer(const std::vector<NDArray<int8_t>>& input,
739 std::vector<NDArray<int8_t>>& output,
740 const std::vector<BatchParam>& params);
741 std::vector<NDArray<int8_t>> infer(const std::vector<NDArray<int8_t>>& input,
742 const std::vector<BatchParam>& params,
743 StatusCode& sc);
744 StatusCode infer(const std::vector<int8_t*>& input,
745 std::vector<std::vector<int8_t>>& output,
746 const std::vector<std::vector<int64_t>>& shape,
747 const std::vector<BatchParam>& params);
748 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
749 const std::vector<std::vector<int64_t>>& shape,
750 const std::vector<BatchParam>& params,
751 StatusCode& sc);
752
754
767 StatusCode inferCHW(const std::vector<NDArray<int8_t>>& input,
768 std::vector<NDArray<int8_t>>& output);
769 std::vector<NDArray<int8_t>> inferCHW(const std::vector<NDArray<int8_t>>& input,
770 StatusCode& sc);
771 StatusCode inferCHW(const std::vector<int8_t*>& input,
772 std::vector<std::vector<int8_t>>& output);
773 std::vector<std::vector<int8_t>> inferCHW(const std::vector<int8_t*>& input,
774 StatusCode& sc);
775 StatusCode inferCHW(const std::vector<int8_t*>& input,
776 std::vector<std::vector<int8_t>>& output,
777 const std::vector<std::vector<int64_t>>& shape);
778 std::vector<std::vector<int8_t>> inferCHW(
779 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
780 StatusCode& sc);
781
782 StatusCode inferCHW(const std::vector<NDArray<int8_t>>& input,
783 std::vector<NDArray<int8_t>>& output, uint32_t cache_size);
784 std::vector<NDArray<int8_t>> inferCHW(const std::vector<NDArray<int8_t>>& input,
785 uint32_t cache_size, StatusCode& sc);
786 StatusCode inferCHW(const std::vector<int8_t*>& input,
787 std::vector<std::vector<int8_t>>& output,
788 const std::vector<std::vector<int64_t>>& shape,
789 uint32_t cache_size);
790 std::vector<std::vector<int8_t>> inferCHW(
791 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
792 uint32_t cache_size, StatusCode& sc);
794
807 std::vector<NDArray<float>> inferToFloat(const std::vector<NDArray<int8_t>>& input,
808 StatusCode& sc);
809 std::vector<std::vector<float>> inferToFloat(const std::vector<int8_t*>& input,
810 StatusCode& sc);
811 std::vector<std::vector<float>> inferToFloat(
812 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
813 StatusCode& sc);
814
815 std::vector<NDArray<float>> inferToFloat(const std::vector<NDArray<int8_t>>& input,
816 uint32_t cache_size, StatusCode& sc);
817 std::vector<std::vector<float>> inferToFloat(
818 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
819 uint32_t cache_size, StatusCode& sc);
821
834 std::vector<NDArray<float>> inferCHWToFloat(const std::vector<NDArray<int8_t>>& input,
835 StatusCode& sc);
836 std::vector<std::vector<float>> inferCHWToFloat(const std::vector<int8_t*>& input,
837 StatusCode& sc);
838 std::vector<std::vector<float>> inferCHWToFloat(
839 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
840 StatusCode& sc);
841
842 std::vector<NDArray<float>> inferCHWToFloat(const std::vector<NDArray<int8_t>>& input,
843 uint32_t cache_size, StatusCode& sc);
844 std::vector<std::vector<float>> inferCHWToFloat(
845 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
846 uint32_t cache_size, StatusCode& sc);
848
871 StatusCode inferBuffer(const std::vector<Buffer>& input, std::vector<Buffer>& output,
872 const std::vector<std::vector<int64_t>>& shape = {},
873 uint32_t cache_size = 0);
874 StatusCode inferBuffer(const std::vector<std::vector<Buffer>>& input,
875 std::vector<std::vector<Buffer>>& output,
876 const std::vector<std::vector<int64_t>>& shape = {},
877 uint32_t cache_size = 0);
879
897 StatusCode inferBufferToFloat(const std::vector<Buffer>& input,
898 std::vector<NDArray<float>>& output,
899 const std::vector<std::vector<int64_t>>& shape = {},
900 uint32_t cache_size = 0);
901 StatusCode inferBufferToFloat(const std::vector<std::vector<Buffer>>& input,
902 std::vector<NDArray<float>>& output,
903 const std::vector<std::vector<int64_t>>& shape = {},
904 uint32_t cache_size = 0);
905 StatusCode inferBufferToFloat(const std::vector<Buffer>& input,
906 std::vector<std::vector<float>>& output,
907 const std::vector<std::vector<int64_t>>& shape = {},
908 uint32_t cache_size = 0);
909 StatusCode inferBufferToFloat(const std::vector<std::vector<Buffer>>& input,
910 std::vector<std::vector<float>>& output,
911 const std::vector<std::vector<int64_t>>& shape = {},
912 uint32_t cache_size = 0);
914
928 StatusCode inferSpeedrun(int variant_idx = 0);
929
984
996 Future<float> inferAsync(const std::vector<NDArray<float>>& input, StatusCode& sc);
997
1009 Future<float> inferAsyncCHW(const std::vector<NDArray<float>>& input, StatusCode& sc);
1010
1021 Future<int8_t> inferAsync(const std::vector<NDArray<int8_t>>& input, StatusCode& sc);
1022
1033 Future<int8_t> inferAsyncCHW(const std::vector<NDArray<int8_t>>& input,
1034 StatusCode& sc);
1035
1046 Future<float> inferAsyncToFloat(const std::vector<NDArray<int8_t>>& input,
1047 StatusCode& sc);
1048
1059 Future<float> inferAsyncCHWToFloat(const std::vector<NDArray<int8_t>>& input,
1060 StatusCode& sc);
1061
1072 Future<float> inferAsync(const std::vector<NDArray<uint8_t>>& input, StatusCode& sc);
1073
1084 Future<float> inferAsyncCHW(const std::vector<NDArray<uint8_t>>& input,
1085 StatusCode& sc);
1086
1088
1110
1111 // Acquire buffer
1112 std::vector<Buffer> acquireInputBuffer(
1113 const std::vector<std::vector<int>>& seqlens = {}) const;
1114 std::vector<Buffer> acquireOutputBuffer(
1115 const std::vector<std::vector<int>>& seqlens = {}) const;
1116 std::vector<std::vector<Buffer>> acquireInputBuffers(
1117 const int batch_size, const std::vector<std::vector<int>>& seqlens = {}) const;
1118 std::vector<std::vector<Buffer>> acquireOutputBuffers(
1119 const int batch_size, const std::vector<std::vector<int>>& seqlens = {}) const;
1120
1121 // Deallocate acquired Input/Output buffer
1122 StatusCode releaseBuffer(std::vector<Buffer>& buffer) const;
1123 StatusCode releaseBuffers(std::vector<std::vector<Buffer>>& buffers) const;
1124
1125 // Reposition single batch
1126 StatusCode repositionInputs(const std::vector<float*>& input,
1127 std::vector<Buffer>& input_buf,
1128 const std::vector<std::vector<int>>& seqlens = {}) const;
1129 StatusCode repositionOutputs(const std::vector<Buffer>& output_buf,
1130 std::vector<float*>& output,
1131 const std::vector<std::vector<int>>& seqlens = {}) const;
1132 StatusCode repositionOutputs(const std::vector<Buffer>& output_buf,
1133 std::vector<std::vector<float>>& output,
1134 const std::vector<std::vector<int>>& seqlens = {}) const;
1135 StatusCode repositionInputs(const std::vector<uint8_t*>& input,
1136 std::vector<Buffer>& input_buf,
1137 const std::vector<std::vector<int>>& seqlens = {}) const;
1138
1139 // Reposition multiple batches
1140 StatusCode repositionInputs(const std::vector<float*>& input,
1141 std::vector<std::vector<Buffer>>& input_buf,
1142 const std::vector<std::vector<int>>& seqlens = {}) const;
1143 StatusCode repositionOutputs(const std::vector<std::vector<Buffer>>& output_buf,
1144 std::vector<float*>& output,
1145 const std::vector<std::vector<int>>& seqlens = {}) const;
1146 StatusCode repositionOutputs(const std::vector<std::vector<Buffer>>& output_buf,
1147 std::vector<std::vector<float>>& output,
1148 const std::vector<std::vector<int>>& seqlens = {}) const;
1149 StatusCode repositionInputs(const std::vector<uint8_t*>& input,
1150 std::vector<std::vector<Buffer>>& input_buf,
1151 const std::vector<std::vector<int>>& seqlens = {}) const;
1153
1165
1177 NPUData acquireInputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
1178 StatusCode& sc);
1179
1200 NPUData acquireInputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
1201 DataType dtype, StatusCode& sc);
1202
1214 NPUData acquireOutputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
1215 StatusCode& sc);
1216
1236 NPUData acquireOutputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
1237 DataType dtype, StatusCode& sc);
1238
1271 StatusCode infer(std::vector<NPUData>& inputs, std::vector<NPUData>& outputs,
1272 uint32_t cache_size = 0);
1273 StatusCode inferCHW(std::vector<NPUData>& inputs, std::vector<NPUData>& outputs,
1274 uint32_t cache_size = 0);
1275
1299 std::vector<NPUData> infer(std::vector<NPUData>& inputs, StatusCode& sc,
1300 uint32_t cache_size = 0);
1301 std::vector<NPUData> inferCHW(std::vector<NPUData>& inputs, StatusCode& sc,
1302 uint32_t cache_size = 0);
1303
1305
1314 int getNumModelVariants() const;
1315
1329 std::unique_ptr<ModelVariantHandle> getModelVariantHandle(int variant_idx,
1330 StatusCode& sc) const;
1331
1337 const std::vector<std::vector<int64_t>>& getModelInputShape() const;
1338
1344 const std::vector<std::vector<int64_t>>& getModelOutputShape() const;
1345
1351 const std::vector<BufferInfo>& getInputBufferInfo() const;
1352
1358 const std::vector<BufferInfo>& getOutputBufferInfo() const;
1359
1365 std::vector<Scale> getInputScale() const;
1366
1372 std::vector<Scale> getOutputScale() const;
1373
1379 DataType getModelInputDataType() const;
1380
1386 DataType getModelOutputDataType() const;
1387
1396 uint32_t getIdentifier() const;
1397
1403 std::string getModelPath() const;
1404
1415 std::vector<std::string> getDeviceNames() const;
1416
1422 std::vector<CacheInfo> getCacheInfos() const;
1423
1430
1442 StatusCode dumpCacheMemory(std::vector<std::vector<int8_t>>& bufs, int cache_id = 0);
1443
1454 std::vector<std::vector<int8_t>> dumpCacheMemory(StatusCode& sc);
1455 std::vector<std::vector<int8_t>> dumpCacheMemory(int cache_id, StatusCode& sc);
1456
1468 StatusCode dumpCacheMemory(const std::string& cache_dir, int cache_id = 0);
1469
1481 StatusCode loadCacheMemory(const std::vector<std::vector<int8_t>>& bufs,
1482 int cache_id = 0);
1483
1495 StatusCode loadCacheMemory(const std::string& cache_dir, int cache_id = 0);
1496
1510 StatusCode dumpCacheMemoryByName(std::vector<int8_t>& buf, const std::string& name,
1511 int cache_id = 0);
1512
1525 std::vector<int8_t> dumpCacheMemoryByName(const std::string& name, StatusCode& sc,
1526 int cache_id = 0);
1527
1542 StatusCode dumpCacheMemoryByLayerHash(std::vector<int8_t>& buf,
1543 const std::string& layer_hash,
1544 int cache_id = 0);
1545
1559 std::vector<int8_t> dumpCacheMemoryByLayerHash(const std::string& layer_hash,
1560 StatusCode& sc, int cache_id = 0);
1561
1575 StatusCode loadCacheMemoryByName(const std::vector<int8_t>& buf,
1576 const std::string& name, int cache_id = 0);
1577
1592 StatusCode loadCacheMemoryByLayerHash(const std::vector<int8_t>& buf,
1593 const std::string& layer_hash,
1594 int cache_id = 0);
1595
1610 StatusCode resetFixedTailCacheMemory(int cache_id = 0);
1611
1625 int filterCacheTail(int cache_size, int tail_size, const std::vector<bool>& mask,
1626 StatusCode& sc);
1627
1641 int moveCacheTail(int num_head, int num_tail, int cache_size, StatusCode& sc);
1642
1670 template <typename T>
1671 std::vector<PinnedMemory<float>> inferPinnedMemory(
1672 const std::vector<PinnedMemory<T>>& inputs, uint32_t cache_size, StatusCode& sc);
1673
1701 template <typename T>
1702 StatusCode inferPinnedMemory(const std::vector<PinnedMemory<T>>& inputs,
1703 std::vector<PinnedMemory<float>>& outputs,
1704 uint32_t cache_size);
1705
1707
1714
1718 StatusCode infer(const std::vector<float*>& input,
1719 std::vector<std::vector<float>>& output, int batch_size);
1720
1724 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
1725 int batch_size, StatusCode& sc);
1726
1728
1729private:
1730 Model();
1731
1732#ifndef _MSC_VER
1733 std::experimental::propagate_const<std::unique_ptr<ModelImpl>> mImpl;
1734#else
1735 std::unique_ptr<ModelImpl> mImpl;
1736#endif
1737
1738 friend class Accelerator;
1739};
1740
1742
1743} // namespace mobilint
1744
1745#endif
mobilint::Accelerator
Represents an accelerator, i.e., an NPU, used for executing models.
Definition acc.h:40
mobilint::Future
Represents a future for retrieving the result of asynchronous inference.
Definition future.h:43
mobilint::ModelConfig
Configures a core mode and core allocation of a model for NPU inference.
Definition type.h:230
mobilint::Model::infer
StatusCode infer(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output, const std::vector< BatchParam > &params)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::inferCHW
std::vector< NPUData > inferCHW(std::vector< NPUData > &inputs, StatusCode &sc, uint32_t cache_size=0)
Acquires an NPUData for the model input at the given index.
mobilint::Model::getModelPath
std::string getModelPath() const
Returns the path to the MXQ model file associated with the Model.
mobilint::Model::inferAsyncToFloat
Future< float > inferAsyncToFloat(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-float asynchronous inference.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, int batch_size, StatusCode &sc)
mobilint::Model::isTarget
bool isTarget(CoreId core_id) const
Checks if the NPU core specified by CoreId is the target of the model. In other words,...
mobilint::Model::create
static std::unique_ptr< Model > create(const std::string &mxq_path, StatusCode &sc)
Creates a Model object from the specified MXQ model file.
mobilint::Model::inferAsyncCHW
Future< float > inferAsyncCHW(const std::vector< NDArray< float > > &input, StatusCode &sc)
Initiates asynchronous inference with input in NCHW (batch N, channels C, height H,...
mobilint::Model::infer
std::vector< NDArray< float > > infer(const std::vector< NDArray< float > > &input, StatusCode &sc)
This overload differs from the above function in that it directly returns the inference results inste...
mobilint::Model::inferAsync
Future< int8_t > inferAsync(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-int8_t asynchronous inference.
mobilint::Model::infer
std::vector< NDArray< float > > infer(const std::vector< NDArray< float > > &input, const std::vector< BatchParam > &params, StatusCode &sc)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::loadCacheMemoryByName
StatusCode loadCacheMemoryByName(const std::vector< int8_t > &buf, const std::string &name, int cache_id=0)
Loads a single KV cache memory from a buffer.
mobilint::Model::inferAsyncCHWToFloat
Future< float > inferAsyncCHWToFloat(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-float asynchronous inference.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape, const std::vector< BatchParam > &params)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::getOutputScale
std::vector< Scale > getOutputScale() const
Returns the output quantization scale(s) of the model.
mobilint::Model::loadCacheMemory
StatusCode loadCacheMemory(const std::vector< std::vector< int8_t > > &bufs, int cache_id=0)
Loads the KV cache memory from buffers.
mobilint::Model::dumpCacheMemory
StatusCode dumpCacheMemory(std::vector< std::vector< int8_t > > &bufs, int cache_id=0)
Dumps the KV cache memory into buffers.
mobilint::Model::inferCHW
StatusCode inferCHW(std::vector< NPUData > &inputs, std::vector< NPUData > &outputs, uint32_t cache_size=0)
Acquires an NPUData for the model input at the given index.
mobilint::Model::getModelOutputShape
const std::vector< std::vector< int64_t > > & getModelOutputShape() const
Returns the output shape of the model.
mobilint::Model::infer
std::vector< NPUData > infer(std::vector< NPUData > &inputs, StatusCode &sc, uint32_t cache_size=0)
Performs inference using NPUData inputs and returns the outputs.
mobilint::Model::getModelVariantHandle
std::unique_ptr< ModelVariantHandle > getModelVariantHandle(int variant_idx, StatusCode &sc) const
Retrieves a handle to the specified model variant.
mobilint::Model::loadCacheMemory
StatusCode loadCacheMemory(const std::string &cache_dir, int cache_id=0)
Loads the KV cache memory from files in the specified directory.
mobilint::Model::launch
StatusCode launch(Accelerator &acc)
Launches the model on the specified Accelerator, which represents the actual NPU.
mobilint::Model::dumpCacheMemoryByLayerHash
StatusCode dumpCacheMemoryByLayerHash(std::vector< int8_t > &buf, const std::string &layer_hash, int cache_id=0)
Dumps a single KV cache memory into a buffer.
mobilint::Model::inferAsync
Future< float > inferAsync(const std::vector< NDArray< uint8_t > > &input, StatusCode &sc)
This overload supports uint8_t-to-float asynchronous inference.
mobilint::Model::getTargetCores
std::vector< CoreId > getTargetCores() const
Returns the NPU cores the model is configured to use.
mobilint::Model::inferCHW
std::vector< std::vector< float > > inferCHW(const std::vector< float * > &input, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
std::vector< std::vector< float > > inferCHW(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::acquireInputNPUData
NPUData acquireInputNPUData(const std::vector< int64_t > &shape, int idx, bool upload, StatusCode &sc)
Acquires an NPUData for the model input at the given index.
mobilint::Model::getInputBufferInfo
const std::vector< BufferInfo > & getInputBufferInfo() const
Returns the input buffer information for the model.
mobilint::Model::moveCacheTail
int moveCacheTail(int num_head, int num_tail, int cache_size, StatusCode &sc)
Moves the tail of the KV cache memory to the end of the head.
mobilint::Model::getModelInputDataType
DataType getModelInputDataType() const
Returns a data type for model inputs.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::infer
StatusCode infer(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output)
Performs inference.
mobilint::Model::getCoreMode
CoreMode getCoreMode() const
Retrieves the core mode of the model.
mobilint::Model::getNumModelVariants
int getNumModelVariants() const
Returns the total number of model variants available in this model.
mobilint::Model::inferAsync
Future< float > inferAsync(const std::vector< NDArray< float > > &input, StatusCode &sc)
Initiates asynchronous inference with input in NHWC (batch N, height H, width W, channels C) or HWC f...
mobilint::Model::dumpCacheMemoryByLayerHash
std::vector< int8_t > dumpCacheMemoryByLayerHash(const std::string &layer_hash, StatusCode &sc, int cache_id=0)
Dumps a single KV cache memory into a buffer.
mobilint::Model::getIdentifier
uint32_t getIdentifier() const
Returns the model's unique identifier.
mobilint::Model::inferCHW
std::vector< NDArray< float > > inferCHW(const std::vector< NDArray< float > > &input, StatusCode &sc)
This overload differs from the above function in that it directly returns the inference results inste...
mobilint::Model::infer
std::vector< NDArray< float > > infer(const std::vector< NDArray< float > > &input, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::getModelOutputDataType
DataType getModelOutputDataType() const
Returns a data type for model outputs.
mobilint::Model::dispose
StatusCode dispose()
Disposes of the model loaded onto the NPU.
mobilint::Model::inferPinnedMemory
std::vector< PinnedMemory< float > > inferPinnedMemory(const std::vector< PinnedMemory< T > > &inputs, uint32_t cache_size, StatusCode &sc)
Performs inference directly on pinned memory buffers.
mobilint::Model::resetFixedTailCacheMemory
StatusCode resetFixedTailCacheMemory(int cache_id=0)
Reset the fixed tail KV cache memory.
mobilint::Model::acquireOutputNPUData
NPUData acquireOutputNPUData(const std::vector< int64_t > &shape, int idx, bool upload, DataType dtype, StatusCode &sc)
Acquires an NPUData with an explicit element type.
mobilint::Model::inferPinnedMemory
StatusCode inferPinnedMemory(const std::vector< PinnedMemory< T > > &inputs, std::vector< PinnedMemory< float > > &outputs, uint32_t cache_size)
Performs inference directly on pinned memory buffers.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::create
static std::unique_ptr< Model > create(const std::string &mxq_path, const ModelConfig &config, StatusCode &sc)
Creates a Model object from the specified MXQ model file and configuration.
mobilint::Model::filterCacheTail
int filterCacheTail(int cache_size, int tail_size, const std::vector< bool > &mask, StatusCode &sc)
Filter the tail of the KV cache memory.
mobilint::Model::getModelInputShape
const std::vector< std::vector< int64_t > > & getModelInputShape() const
Returns the input shape of the model.
mobilint::Model::inferCHW
std::vector< NDArray< float > > inferCHW(const std::vector< NDArray< float > > &input, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::acquireInputNPUData
NPUData acquireInputNPUData(const std::vector< int64_t > &shape, int idx, bool upload, DataType dtype, StatusCode &sc)
Acquires an NPUData with an explicit element type.
mobilint::Model::inferAsyncCHW
Future< int8_t > inferAsyncCHW(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-int8_t asynchronous inference.
mobilint::Model::dumpCacheMemory
StatusCode dumpCacheMemory(const std::string &cache_dir, int cache_id=0)
Dumps KV cache memory to files in the specified directory.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::getOutputBufferInfo
const std::vector< BufferInfo > & getOutputBufferInfo() const
Returns the output buffer information of the model.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::getCacheInfos
std::vector< CacheInfo > getCacheInfos() const
Returns informations of KV-cache of the model.
mobilint::Model::infer
StatusCode infer(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::inferAsyncCHW
Future< float > inferAsyncCHW(const std::vector< NDArray< uint8_t > > &input, StatusCode &sc)
This overload supports uint8_t-to-float asynchronous inference.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
std::vector< std::vector< float > > inferCHW(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, int batch_size)
mobilint::Model::dumpCacheMemory
std::vector< std::vector< int8_t > > dumpCacheMemory(StatusCode &sc)
Dumps the KV cache memory into buffers.
mobilint::Model::dumpCacheMemoryByName
std::vector< int8_t > dumpCacheMemoryByName(const std::string &name, StatusCode &sc, int cache_id=0)
Dumps a single KV cache memory into a buffer.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< float * > &input, std::vector< std::vector< float > > &output)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::getInputScale
std::vector< Scale > getInputScale() const
Returns the input quantization scale(s) of the model.
mobilint::Model::loadCacheMemoryByLayerHash
StatusCode loadCacheMemoryByLayerHash(const std::vector< int8_t > &buf, const std::string &layer_hash, int cache_id=0)
Loads a single KV cache memory from a buffer.
mobilint::Model::infer
StatusCode infer(std::vector< NPUData > &inputs, std::vector< NPUData > &outputs, uint32_t cache_size=0)
Performs inference using NPUData for both inputs and outputs.
mobilint::Model::inferSpeedrun
StatusCode inferSpeedrun(int variant_idx=0)
Development-only API for measuring pure NPU inference speed.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output)
Performs inference.
mobilint::Model::getDeviceNames
std::vector< std::string > getDeviceNames() const
Returns the supported target device name(s) this model can run on.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::dumpCacheMemoryByName
StatusCode dumpCacheMemoryByName(std::vector< int8_t > &buf, const std::string &name, int cache_id=0)
Dumps a single KV cache memory into a buffer.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::dumpCacheMemory
std::vector< std::vector< int8_t > > dumpCacheMemory(int cache_id, StatusCode &sc)
Dumps the KV cache memory into buffers.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, const std::vector< BatchParam > &params, StatusCode &sc)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::acquireOutputNPUData
NPUData acquireOutputNPUData(const std::vector< int64_t > &shape, int idx, bool upload, StatusCode &sc)
Acquires an NPUData for the model output at the given index.
mobilint::NDArray
A class representing an N-dimensional array (NDArray).
Definition ndarray.h:77
mobilint::NPUData
Represents a model input or output tensor that can reside either on the host (CPU) or on the NPU.
Definition npu_data.h:59
mobilint::PinnedMemory
A buffer of NPU-accessible pinned (physically contiguous) memory.
Definition pinned_memory.h:56
mobilint::DataType
DataType
DataType.
Definition type.h:514
mobilint::CoreMode
CoreMode
Defines the core mode for NPU execution.
Definition type.h:165
mobilint::StatusCode
StatusCode
Enumerates status codes for the qbruntime.
Definition status_code.h:28
model_variant_handle.h
npu_data.h
pinned_memory.h
status_code.h
mobilint::CoreId
Represents a unique identifier for an NPU core.
Definition type.h:118
Generated by doxygen 1.14.0