model.h Source File

model.h Source File#

SDK qb Runtime Library: model.h Source File
SDK qb Runtime Library v1.4
MCS001-KR
model.h
Go to the documentation of this file.
1// Copyright ⓒ 2019- Mobilint Inc. All rights reserved.
5
6#ifndef QBRUNTIME_MODEL_H_
7#define QBRUNTIME_MODEL_H_
8
9#include <cstdint>
10#ifndef _MSC_VER
11#include <experimental/propagate_const>
12#endif
13#include <memory>
14#include <string>
15#include <vector>
16
17#include "qbruntime/export.h"
18#include "qbruntime/future.h"
19#include "qbruntime/model_variant_handle.h"
20#include "qbruntime/ndarray.h"
21#include "qbruntime/npu_data.h"
22#include "qbruntime/pinned_memory.h"
23#include "qbruntime/status_code.h"
24#include "qbruntime/type.h"
25
26namespace mobilint {
27
32
33class Accelerator;
34class ModelImpl;
35
42class QBRUNTIME_EXPORT Model {
43public:
58 static std::unique_ptr<Model> create(const std::string& mxq_path, StatusCode& sc);
59
75 static std::unique_ptr<Model> create(const std::string& mxq_path,
76 const ModelConfig& config, StatusCode& sc);
77
78 Model(const Model& other) = delete;
79 Model(Model&& other) noexcept;
80 Model& operator=(const Model& rhs) = delete;
81 Model& operator=(Model&& rhs) noexcept;
82 ~Model();
83
92 StatusCode launch(Accelerator& acc);
93
102 StatusCode dispose();
103
109 CoreMode getCoreMode() const;
110
119 bool isTarget(CoreId core_id) const;
120
126 std::vector<CoreId> getTargetCores() const;
127
145
156 StatusCode infer(const std::vector<NDArray<float>>& input,
157 std::vector<NDArray<float>>& output);
158
170 std::vector<NDArray<float>> infer(const std::vector<NDArray<float>>& input,
171 StatusCode& sc);
172
184 StatusCode infer(const std::vector<float*>& input,
185 std::vector<std::vector<float>>& output);
186
201 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
202 StatusCode& sc);
203
220 StatusCode infer(const std::vector<float*>& input,
221 std::vector<std::vector<float>>& output,
222 const std::vector<std::vector<int64_t>>& shape);
239 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
240 const std::vector<std::vector<int64_t>>& shape,
241 StatusCode& sc);
242
256 StatusCode infer(const std::vector<NDArray<float>>& input,
257 std::vector<NDArray<float>>& output, uint32_t cache_size);
258
271 StatusCode infer(const std::vector<NDArray<float>>& input,
272 std::vector<NDArray<float>>& output,
273 const std::vector<BatchParam>& params);
274
291 std::vector<NDArray<float>> infer(const std::vector<NDArray<float>>& input,
292 uint32_t cache_size, StatusCode& sc);
293
309 std::vector<NDArray<float>> infer(const std::vector<NDArray<float>>& input,
310 const std::vector<BatchParam>& params,
311 StatusCode& sc);
312
328 StatusCode infer(const std::vector<float*>& input,
329 std::vector<std::vector<float>>& output,
330 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size);
331
350 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
351 const std::vector<std::vector<int64_t>>& shape,
352 uint32_t cache_size, StatusCode& sc);
353
368 StatusCode infer(const std::vector<float*>& input,
369 std::vector<std::vector<float>>& output,
370 const std::vector<std::vector<int64_t>>& shape,
371 const std::vector<BatchParam>& params);
372
390 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
391 const std::vector<std::vector<int64_t>>& shape,
392 const std::vector<BatchParam>& params,
393 StatusCode& sc);
394
396
421
432 StatusCode inferCHW(const std::vector<NDArray<float>>& input,
433 std::vector<NDArray<float>>& output);
434
446 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<float>>& input,
447 StatusCode& sc);
448
460 StatusCode inferCHW(const std::vector<float*>& input,
461 std::vector<std::vector<float>>& output);
462
477 std::vector<std::vector<float>> inferCHW(const std::vector<float*>& input,
478 StatusCode& sc);
479
496 StatusCode inferCHW(const std::vector<float*>& input,
497 std::vector<std::vector<float>>& output,
498 const std::vector<std::vector<int64_t>>& shape);
499
516 std::vector<std::vector<float>> inferCHW(
517 const std::vector<float*>& input, const std::vector<std::vector<int64_t>>& shape,
518 StatusCode& sc);
519
533 StatusCode inferCHW(const std::vector<NDArray<float>>& input,
534 std::vector<NDArray<float>>& output, uint32_t cache_size);
535
552 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<float>>& input,
553 uint32_t cache_size, StatusCode& sc);
554
570 StatusCode inferCHW(const std::vector<float*>& input,
571 std::vector<std::vector<float>>& output,
572 const std::vector<std::vector<int64_t>>& shape,
573 uint32_t cache_size);
574
593 std::vector<std::vector<float>> inferCHW(
594 const std::vector<float*>& input, const std::vector<std::vector<int64_t>>& shape,
595 uint32_t cache_size, StatusCode& sc);
596
598
610
611 StatusCode infer(const std::vector<NDArray<uint8_t>>& input,
612 std::vector<NDArray<float>>& output);
613 std::vector<NDArray<float>> infer(const std::vector<NDArray<uint8_t>>& input,
614 StatusCode& sc);
615 StatusCode infer(const std::vector<uint8_t*>& input,
616 std::vector<std::vector<float>>& output);
617 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
618 StatusCode& sc);
619 StatusCode infer(const std::vector<uint8_t*>& input,
620 std::vector<std::vector<float>>& output,
621 const std::vector<std::vector<int64_t>>& shape);
622 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
623 const std::vector<std::vector<int64_t>>& shape,
624 StatusCode& sc);
625
626 StatusCode infer(const std::vector<NDArray<uint8_t>>& input,
627 std::vector<NDArray<float>>& output, uint32_t cache_size);
628 std::vector<NDArray<float>> infer(const std::vector<NDArray<uint8_t>>& input,
629 uint32_t cache_size, StatusCode& sc);
630 StatusCode infer(const std::vector<uint8_t*>& input,
631 std::vector<std::vector<float>>& output,
632 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size);
633 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
634 const std::vector<std::vector<int64_t>>& shape,
635 uint32_t cache_size, StatusCode& sc);
636
637 StatusCode infer(const std::vector<NDArray<uint8_t>>& input,
638 std::vector<NDArray<float>>& output,
639 const std::vector<BatchParam>& params);
640 std::vector<NDArray<float>> infer(const std::vector<NDArray<uint8_t>>& input,
641 const std::vector<BatchParam>& params,
642 StatusCode& sc);
643 StatusCode infer(const std::vector<uint8_t*>& input,
644 std::vector<std::vector<float>>& output,
645 const std::vector<std::vector<int64_t>>& shape,
646 const std::vector<BatchParam>& params);
647 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
648 const std::vector<std::vector<int64_t>>& shape,
649 const std::vector<BatchParam>& params,
650 StatusCode& sc);
651
653
662 StatusCode inferCHW(const std::vector<NDArray<uint8_t>>& input,
663 std::vector<NDArray<float>>& output);
664 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<uint8_t>>& input,
665 StatusCode& sc);
666 StatusCode inferCHW(const std::vector<uint8_t*>& input,
667 std::vector<std::vector<float>>& output);
668 std::vector<std::vector<float>> inferCHW(const std::vector<uint8_t*>& input,
669 StatusCode& sc);
670 StatusCode inferCHW(const std::vector<uint8_t*>& input,
671 std::vector<std::vector<float>>& output,
672 const std::vector<std::vector<int64_t>>& shape);
673 std::vector<std::vector<float>> inferCHW(
674 const std::vector<uint8_t*>& input,
675 const std::vector<std::vector<int64_t>>& shape, StatusCode& sc);
676
677 StatusCode inferCHW(const std::vector<NDArray<uint8_t>>& input,
678 std::vector<NDArray<float>>& output, uint32_t cache_size);
679 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<uint8_t>>& input,
680 uint32_t cache_size, StatusCode& sc);
681 StatusCode inferCHW(const std::vector<uint8_t*>& input,
682 std::vector<std::vector<float>>& output,
683 const std::vector<std::vector<int64_t>>& shape,
684 uint32_t cache_size);
685 std::vector<std::vector<float>> inferCHW(
686 const std::vector<uint8_t*>& input,
687 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size,
688 StatusCode& sc);
690
707
708 StatusCode infer(const std::vector<NDArray<int8_t>>& input,
709 std::vector<NDArray<int8_t>>& output);
710 std::vector<NDArray<int8_t>> infer(const std::vector<NDArray<int8_t>>& input,
711 StatusCode& sc);
712 StatusCode infer(const std::vector<int8_t*>& input,
713 std::vector<std::vector<int8_t>>& output);
714 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
715 StatusCode& sc);
716 StatusCode infer(const std::vector<int8_t*>& input,
717 std::vector<std::vector<int8_t>>& output,
718 const std::vector<std::vector<int64_t>>& shape);
719 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
720 const std::vector<std::vector<int64_t>>& shape,
721 StatusCode& sc);
722
723 StatusCode infer(const std::vector<NDArray<int8_t>>& input,
724 std::vector<NDArray<int8_t>>& output, uint32_t cache_size);
725 std::vector<NDArray<int8_t>> infer(const std::vector<NDArray<int8_t>>& input,
726 uint32_t cache_size, StatusCode& sc);
727 StatusCode infer(const std::vector<int8_t*>& input,
728 std::vector<std::vector<int8_t>>& output,
729 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size);
730 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
731 const std::vector<std::vector<int64_t>>& shape,
732 uint32_t cache_size, StatusCode& sc);
733
734 StatusCode infer(const std::vector<NDArray<int8_t>>& input,
735 std::vector<NDArray<int8_t>>& output,
736 const std::vector<BatchParam>& params);
737 std::vector<NDArray<int8_t>> infer(const std::vector<NDArray<int8_t>>& input,
738 const std::vector<BatchParam>& params,
739 StatusCode& sc);
740 StatusCode infer(const std::vector<int8_t*>& input,
741 std::vector<std::vector<int8_t>>& output,
742 const std::vector<std::vector<int64_t>>& shape,
743 const std::vector<BatchParam>& params);
744 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
745 const std::vector<std::vector<int64_t>>& shape,
746 const std::vector<BatchParam>& params,
747 StatusCode& sc);
748
750
763 StatusCode inferCHW(const std::vector<NDArray<int8_t>>& input,
764 std::vector<NDArray<int8_t>>& output);
765 std::vector<NDArray<int8_t>> inferCHW(const std::vector<NDArray<int8_t>>& input,
766 StatusCode& sc);
767 StatusCode inferCHW(const std::vector<int8_t*>& input,
768 std::vector<std::vector<int8_t>>& output);
769 std::vector<std::vector<int8_t>> inferCHW(const std::vector<int8_t*>& input,
770 StatusCode& sc);
771 StatusCode inferCHW(const std::vector<int8_t*>& input,
772 std::vector<std::vector<int8_t>>& output,
773 const std::vector<std::vector<int64_t>>& shape);
774 std::vector<std::vector<int8_t>> inferCHW(
775 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
776 StatusCode& sc);
777
778 StatusCode inferCHW(const std::vector<NDArray<int8_t>>& input,
779 std::vector<NDArray<int8_t>>& output, uint32_t cache_size);
780 std::vector<NDArray<int8_t>> inferCHW(const std::vector<NDArray<int8_t>>& input,
781 uint32_t cache_size, StatusCode& sc);
782 StatusCode inferCHW(const std::vector<int8_t*>& input,
783 std::vector<std::vector<int8_t>>& output,
784 const std::vector<std::vector<int64_t>>& shape,
785 uint32_t cache_size);
786 std::vector<std::vector<int8_t>> inferCHW(
787 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
788 uint32_t cache_size, StatusCode& sc);
790
803 std::vector<NDArray<float>> inferToFloat(const std::vector<NDArray<int8_t>>& input,
804 StatusCode& sc);
805 std::vector<std::vector<float>> inferToFloat(const std::vector<int8_t*>& input,
806 StatusCode& sc);
807 std::vector<std::vector<float>> inferToFloat(
808 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
809 StatusCode& sc);
810
811 std::vector<NDArray<float>> inferToFloat(const std::vector<NDArray<int8_t>>& input,
812 uint32_t cache_size, StatusCode& sc);
813 std::vector<std::vector<float>> inferToFloat(
814 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
815 uint32_t cache_size, StatusCode& sc);
817
830 std::vector<NDArray<float>> inferCHWToFloat(const std::vector<NDArray<int8_t>>& input,
831 StatusCode& sc);
832 std::vector<std::vector<float>> inferCHWToFloat(const std::vector<int8_t*>& input,
833 StatusCode& sc);
834 std::vector<std::vector<float>> inferCHWToFloat(
835 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
836 StatusCode& sc);
837
838 std::vector<NDArray<float>> inferCHWToFloat(const std::vector<NDArray<int8_t>>& input,
839 uint32_t cache_size, StatusCode& sc);
840 std::vector<std::vector<float>> inferCHWToFloat(
841 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
842 uint32_t cache_size, StatusCode& sc);
844
867 StatusCode inferBuffer(const std::vector<Buffer>& input, std::vector<Buffer>& output,
868 const std::vector<std::vector<int64_t>>& shape = {},
869 uint32_t cache_size = 0);
870 StatusCode inferBuffer(const std::vector<std::vector<Buffer>>& input,
871 std::vector<std::vector<Buffer>>& output,
872 const std::vector<std::vector<int64_t>>& shape = {},
873 uint32_t cache_size = 0);
875
893 StatusCode inferBufferToFloat(const std::vector<Buffer>& input,
894 std::vector<NDArray<float>>& output,
895 const std::vector<std::vector<int64_t>>& shape = {},
896 uint32_t cache_size = 0);
897 StatusCode inferBufferToFloat(const std::vector<std::vector<Buffer>>& input,
898 std::vector<NDArray<float>>& output,
899 const std::vector<std::vector<int64_t>>& shape = {},
900 uint32_t cache_size = 0);
901 StatusCode inferBufferToFloat(const std::vector<Buffer>& input,
902 std::vector<std::vector<float>>& output,
903 const std::vector<std::vector<int64_t>>& shape = {},
904 uint32_t cache_size = 0);
905 StatusCode inferBufferToFloat(const std::vector<std::vector<Buffer>>& input,
906 std::vector<std::vector<float>>& output,
907 const std::vector<std::vector<int64_t>>& shape = {},
908 uint32_t cache_size = 0);
910
924 StatusCode inferSpeedrun(int variant_idx = 0);
925
980
992 Future<float> inferAsync(const std::vector<NDArray<float>>& input, StatusCode& sc);
993
1005 Future<float> inferAsyncCHW(const std::vector<NDArray<float>>& input, StatusCode& sc);
1006
1017 Future<int8_t> inferAsync(const std::vector<NDArray<int8_t>>& input, StatusCode& sc);
1018
1029 Future<int8_t> inferAsyncCHW(const std::vector<NDArray<int8_t>>& input,
1030 StatusCode& sc);
1031
1042 Future<float> inferAsyncToFloat(const std::vector<NDArray<int8_t>>& input,
1043 StatusCode& sc);
1044
1055 Future<float> inferAsyncCHWToFloat(const std::vector<NDArray<int8_t>>& input,
1056 StatusCode& sc);
1057
1068 Future<float> inferAsync(const std::vector<NDArray<uint8_t>>& input, StatusCode& sc);
1069
1080 Future<float> inferAsyncCHW(const std::vector<NDArray<uint8_t>>& input,
1081 StatusCode& sc);
1082
1084
1106
1107 // Acquire buffer
1108 std::vector<Buffer> acquireInputBuffer(
1109 const std::vector<std::vector<int>>& seqlens = {}) const;
1110 std::vector<Buffer> acquireOutputBuffer(
1111 const std::vector<std::vector<int>>& seqlens = {}) const;
1112 std::vector<std::vector<Buffer>> acquireInputBuffers(
1113 const int batch_size, const std::vector<std::vector<int>>& seqlens = {}) const;
1114 std::vector<std::vector<Buffer>> acquireOutputBuffers(
1115 const int batch_size, const std::vector<std::vector<int>>& seqlens = {}) const;
1116
1117 // Deallocate acquired Input/Output buffer
1118 StatusCode releaseBuffer(std::vector<Buffer>& buffer) const;
1119 StatusCode releaseBuffers(std::vector<std::vector<Buffer>>& buffers) const;
1120
1121 // Reposition single batch
1122 StatusCode repositionInputs(const std::vector<float*>& input,
1123 std::vector<Buffer>& input_buf,
1124 const std::vector<std::vector<int>>& seqlens = {}) const;
1125 StatusCode repositionOutputs(const std::vector<Buffer>& output_buf,
1126 std::vector<float*>& output,
1127 const std::vector<std::vector<int>>& seqlens = {}) const;
1128 StatusCode repositionOutputs(const std::vector<Buffer>& output_buf,
1129 std::vector<std::vector<float>>& output,
1130 const std::vector<std::vector<int>>& seqlens = {}) const;
1131 StatusCode repositionInputs(const std::vector<uint8_t*>& input,
1132 std::vector<Buffer>& input_buf,
1133 const std::vector<std::vector<int>>& seqlens = {}) const;
1134
1135 // Reposition multiple batches
1136 StatusCode repositionInputs(const std::vector<float*>& input,
1137 std::vector<std::vector<Buffer>>& input_buf,
1138 const std::vector<std::vector<int>>& seqlens = {}) const;
1139 StatusCode repositionOutputs(const std::vector<std::vector<Buffer>>& output_buf,
1140 std::vector<float*>& output,
1141 const std::vector<std::vector<int>>& seqlens = {}) const;
1142 StatusCode repositionOutputs(const std::vector<std::vector<Buffer>>& output_buf,
1143 std::vector<std::vector<float>>& output,
1144 const std::vector<std::vector<int>>& seqlens = {}) const;
1145 StatusCode repositionInputs(const std::vector<uint8_t*>& input,
1146 std::vector<std::vector<Buffer>>& input_buf,
1147 const std::vector<std::vector<int>>& seqlens = {}) const;
1149
1161
1173 NPUData acquireInputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
1174 StatusCode& sc);
1175
1196 NPUData acquireInputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
1197 DataType dtype, StatusCode& sc);
1198
1210 NPUData acquireOutputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
1211 StatusCode& sc);
1212
1232 NPUData acquireOutputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
1233 DataType dtype, StatusCode& sc);
1234
1264 StatusCode infer(std::vector<NPUData>& inputs, std::vector<NPUData>& outputs,
1265 uint32_t cache_size = 0);
1266 StatusCode inferCHW(std::vector<NPUData>& inputs, std::vector<NPUData>& outputs,
1267 uint32_t cache_size = 0);
1268
1289 std::vector<NPUData> infer(std::vector<NPUData>& inputs, StatusCode& sc,
1290 uint32_t cache_size = 0);
1291 std::vector<NPUData> inferCHW(std::vector<NPUData>& inputs, StatusCode& sc,
1292 uint32_t cache_size = 0);
1293
1295
1304 int getNumModelVariants() const;
1305
1319 std::unique_ptr<ModelVariantHandle> getModelVariantHandle(int variant_idx,
1320 StatusCode& sc) const;
1321
1327 const std::vector<std::vector<int64_t>>& getModelInputShape() const;
1328
1334 const std::vector<std::vector<int64_t>>& getModelOutputShape() const;
1335
1341 const std::vector<BufferInfo>& getInputBufferInfo() const;
1342
1348 const std::vector<BufferInfo>& getOutputBufferInfo() const;
1349
1355 std::vector<Scale> getInputScale() const;
1356
1362 std::vector<Scale> getOutputScale() const;
1363
1369 DataType getModelInputDataType() const;
1370
1376 DataType getModelOutputDataType() const;
1377
1386 uint32_t getIdentifier() const;
1387
1393 std::string getModelPath() const;
1394
1405 std::vector<std::string> getDeviceNames() const;
1406
1412 std::vector<CacheInfo> getCacheInfos() const;
1413
1420
1432 StatusCode dumpCacheMemory(std::vector<std::vector<int8_t>>& bufs, int cache_id = 0);
1433
1444 std::vector<std::vector<int8_t>> dumpCacheMemory(StatusCode& sc);
1445 std::vector<std::vector<int8_t>> dumpCacheMemory(int cache_id, StatusCode& sc);
1446
1458 StatusCode dumpCacheMemory(const std::string& cache_dir, int cache_id = 0);
1459
1471 StatusCode loadCacheMemory(const std::vector<std::vector<int8_t>>& bufs,
1472 int cache_id = 0);
1473
1485 StatusCode loadCacheMemory(const std::string& cache_dir, int cache_id = 0);
1486
1500 int filterCacheTail(int cache_size, int tail_size, const std::vector<bool>& mask,
1501 StatusCode& sc);
1502
1516 int moveCacheTail(int num_head, int num_tail, int cache_size, StatusCode& sc);
1517
1545 template <typename T>
1546 std::vector<PinnedMemory<float>> inferPinnedMemory(
1547 const std::vector<PinnedMemory<T>>& inputs, uint32_t cache_size, StatusCode& sc);
1548
1576 template <typename T>
1577 StatusCode inferPinnedMemory(const std::vector<PinnedMemory<T>>& inputs,
1578 std::vector<PinnedMemory<float>>& outputs,
1579 uint32_t cache_size);
1580
1582
1589
1593 StatusCode infer(const std::vector<float*>& input,
1594 std::vector<std::vector<float>>& output, int batch_size);
1595
1599 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
1600 int batch_size, StatusCode& sc);
1601
1603
1604private:
1605 Model();
1606
1607#ifndef _MSC_VER
1608 std::experimental::propagate_const<std::unique_ptr<ModelImpl>> mImpl;
1609#else
1610 std::unique_ptr<ModelImpl> mImpl;
1611#endif
1612
1613 friend class Accelerator;
1614};
1615
1617
1618} // namespace mobilint
1619
1620#endif
mobilint::Accelerator
Represents an accelerator, i.e., an NPU, used for executing models.
Definition acc.h:40
mobilint::Future
Represents a future for retrieving the result of asynchronous inference.
Definition future.h:43
mobilint::ModelConfig
Configures a core mode and core allocation of a model for NPU inference.
Definition type.h:230
mobilint::Model::infer
StatusCode infer(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output, const std::vector< BatchParam > &params)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::inferCHW
std::vector< NPUData > inferCHW(std::vector< NPUData > &inputs, StatusCode &sc, uint32_t cache_size=0)
Acquires an NPUData for the model input at the given index.
mobilint::Model::getModelPath
std::string getModelPath() const
Returns the path to the MXQ model file associated with the Model.
mobilint::Model::inferAsyncToFloat
Future< float > inferAsyncToFloat(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-float asynchronous inference.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, int batch_size, StatusCode &sc)
mobilint::Model::isTarget
bool isTarget(CoreId core_id) const
Checks if the NPU core specified by CoreId is the target of the model. In other words,...
mobilint::Model::create
static std::unique_ptr< Model > create(const std::string &mxq_path, StatusCode &sc)
Creates a Model object from the specified MXQ model file.
mobilint::Model::inferAsyncCHW
Future< float > inferAsyncCHW(const std::vector< NDArray< float > > &input, StatusCode &sc)
Initiates asynchronous inference with input in NCHW (batch N, channels C, height H,...
mobilint::Model::infer
std::vector< NDArray< float > > infer(const std::vector< NDArray< float > > &input, StatusCode &sc)
This overload differs from the above function in that it directly returns the inference results inste...
mobilint::Model::inferAsync
Future< int8_t > inferAsync(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-int8_t asynchronous inference.
mobilint::Model::infer
std::vector< NDArray< float > > infer(const std::vector< NDArray< float > > &input, const std::vector< BatchParam > &params, StatusCode &sc)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::inferAsyncCHWToFloat
Future< float > inferAsyncCHWToFloat(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-float asynchronous inference.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape, const std::vector< BatchParam > &params)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::getOutputScale
std::vector< Scale > getOutputScale() const
Returns the output quantization scale(s) of the model.
mobilint::Model::loadCacheMemory
StatusCode loadCacheMemory(const std::vector< std::vector< int8_t > > &bufs, int cache_id=0)
Loads the KV cache memory from buffers.
mobilint::Model::dumpCacheMemory
StatusCode dumpCacheMemory(std::vector< std::vector< int8_t > > &bufs, int cache_id=0)
Dumps the KV cache memory into buffers.
mobilint::Model::inferCHW
StatusCode inferCHW(std::vector< NPUData > &inputs, std::vector< NPUData > &outputs, uint32_t cache_size=0)
Acquires an NPUData for the model input at the given index.
mobilint::Model::getModelOutputShape
const std::vector< std::vector< int64_t > > & getModelOutputShape() const
Returns the output shape of the model.
mobilint::Model::infer
std::vector< NPUData > infer(std::vector< NPUData > &inputs, StatusCode &sc, uint32_t cache_size=0)
Performs inference using NPUData inputs and returns the outputs.
mobilint::Model::getModelVariantHandle
std::unique_ptr< ModelVariantHandle > getModelVariantHandle(int variant_idx, StatusCode &sc) const
Retrieves a handle to the specified model variant.
mobilint::Model::loadCacheMemory
StatusCode loadCacheMemory(const std::string &cache_dir, int cache_id=0)
Loads the KV cache memory from files in the specified directory.
mobilint::Model::launch
StatusCode launch(Accelerator &acc)
Launches the model on the specified Accelerator, which represents the actual NPU.
mobilint::Model::inferAsync
Future< float > inferAsync(const std::vector< NDArray< uint8_t > > &input, StatusCode &sc)
This overload supports uint8_t-to-float asynchronous inference.
mobilint::Model::getTargetCores
std::vector< CoreId > getTargetCores() const
Returns the NPU cores the model is configured to use.
mobilint::Model::inferCHW
std::vector< std::vector< float > > inferCHW(const std::vector< float * > &input, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
std::vector< std::vector< float > > inferCHW(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::acquireInputNPUData
NPUData acquireInputNPUData(const std::vector< int64_t > &shape, int idx, bool upload, StatusCode &sc)
Acquires an NPUData for the model input at the given index.
mobilint::Model::getInputBufferInfo
const std::vector< BufferInfo > & getInputBufferInfo() const
Returns the input buffer information for the model.
mobilint::Model::moveCacheTail
int moveCacheTail(int num_head, int num_tail, int cache_size, StatusCode &sc)
Moves the tail of the KV cache memory to the end of the head.
mobilint::Model::getModelInputDataType
DataType getModelInputDataType() const
Returns a data type for model inputs.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::infer
StatusCode infer(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output)
Performs inference.
mobilint::Model::getCoreMode
CoreMode getCoreMode() const
Retrieves the core mode of the model.
mobilint::Model::getNumModelVariants
int getNumModelVariants() const
Returns the total number of model variants available in this model.
mobilint::Model::inferAsync
Future< float > inferAsync(const std::vector< NDArray< float > > &input, StatusCode &sc)
Initiates asynchronous inference with input in NHWC (batch N, height H, width W, channels C) or HWC f...
mobilint::Model::getIdentifier
uint32_t getIdentifier() const
Returns the model's unique identifier.
mobilint::Model::inferCHW
std::vector< NDArray< float > > inferCHW(const std::vector< NDArray< float > > &input, StatusCode &sc)
This overload differs from the above function in that it directly returns the inference results inste...
mobilint::Model::infer
std::vector< NDArray< float > > infer(const std::vector< NDArray< float > > &input, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::getModelOutputDataType
DataType getModelOutputDataType() const
Returns a data type for model outputs.
mobilint::Model::dispose
StatusCode dispose()
Disposes of the model loaded onto the NPU.
mobilint::Model::inferPinnedMemory
std::vector< PinnedMemory< float > > inferPinnedMemory(const std::vector< PinnedMemory< T > > &inputs, uint32_t cache_size, StatusCode &sc)
Performs inference directly on pinned memory buffers.
mobilint::Model::acquireOutputNPUData
NPUData acquireOutputNPUData(const std::vector< int64_t > &shape, int idx, bool upload, DataType dtype, StatusCode &sc)
Acquires an NPUData with an explicit element type.
mobilint::Model::inferPinnedMemory
StatusCode inferPinnedMemory(const std::vector< PinnedMemory< T > > &inputs, std::vector< PinnedMemory< float > > &outputs, uint32_t cache_size)
Performs inference directly on pinned memory buffers.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::create
static std::unique_ptr< Model > create(const std::string &mxq_path, const ModelConfig &config, StatusCode &sc)
Creates a Model object from the specified MXQ model file and configuration.
mobilint::Model::filterCacheTail
int filterCacheTail(int cache_size, int tail_size, const std::vector< bool > &mask, StatusCode &sc)
Filter the tail of the KV cache memory.
mobilint::Model::getModelInputShape
const std::vector< std::vector< int64_t > > & getModelInputShape() const
Returns the input shape of the model.
mobilint::Model::inferCHW
std::vector< NDArray< float > > inferCHW(const std::vector< NDArray< float > > &input, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::acquireInputNPUData
NPUData acquireInputNPUData(const std::vector< int64_t > &shape, int idx, bool upload, DataType dtype, StatusCode &sc)
Acquires an NPUData with an explicit element type.
mobilint::Model::inferAsyncCHW
Future< int8_t > inferAsyncCHW(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-int8_t asynchronous inference.
mobilint::Model::dumpCacheMemory
StatusCode dumpCacheMemory(const std::string &cache_dir, int cache_id=0)
Dumps KV cache memory to files in the specified directory.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::getOutputBufferInfo
const std::vector< BufferInfo > & getOutputBufferInfo() const
Returns the output buffer information of the model.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::getCacheInfos
std::vector< CacheInfo > getCacheInfos() const
Returns informations of KV-cache of the model.
mobilint::Model::infer
StatusCode infer(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::inferAsyncCHW
Future< float > inferAsyncCHW(const std::vector< NDArray< uint8_t > > &input, StatusCode &sc)
This overload supports uint8_t-to-float asynchronous inference.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
std::vector< std::vector< float > > inferCHW(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, int batch_size)
mobilint::Model::dumpCacheMemory
std::vector< std::vector< int8_t > > dumpCacheMemory(StatusCode &sc)
Dumps the KV cache memory into buffers.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< float * > &input, std::vector< std::vector< float > > &output)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::getInputScale
std::vector< Scale > getInputScale() const
Returns the input quantization scale(s) of the model.
mobilint::Model::infer
StatusCode infer(std::vector< NPUData > &inputs, std::vector< NPUData > &outputs, uint32_t cache_size=0)
Performs inference using NPUData for both inputs and outputs.
mobilint::Model::inferSpeedrun
StatusCode inferSpeedrun(int variant_idx=0)
Development-only API for measuring pure NPU inference speed.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output)
Performs inference.
mobilint::Model::getDeviceNames
std::vector< std::string > getDeviceNames() const
Returns the supported target device name(s) this model can run on.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::dumpCacheMemory
std::vector< std::vector< int8_t > > dumpCacheMemory(int cache_id, StatusCode &sc)
Dumps the KV cache memory into buffers.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, const std::vector< BatchParam > &params, StatusCode &sc)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::acquireOutputNPUData
NPUData acquireOutputNPUData(const std::vector< int64_t > &shape, int idx, bool upload, StatusCode &sc)
Acquires an NPUData for the model output at the given index.
mobilint::NDArray
A class representing an N-dimensional array (NDArray).
Definition ndarray.h:77
mobilint::NPUData
Represents a model input or output tensor that can reside either on the host (CPU) or on the NPU.
Definition npu_data.h:51
mobilint::PinnedMemory
A buffer of NPU-accessible pinned (physically contiguous) memory.
Definition pinned_memory.h:56
mobilint::DataType
DataType
DataType.
Definition type.h:510
mobilint::CoreMode
CoreMode
Defines the core mode for NPU execution.
Definition type.h:165
mobilint::StatusCode
StatusCode
Enumerates status codes for the qbruntime.
Definition status_code.h:28
model_variant_handle.h
npu_data.h
pinned_memory.h
status_code.h
mobilint::CoreId
Represents a unique identifier for an NPU core.
Definition type.h:118
Generated by doxygen 1.14.0