model.h Source File

model.h Source File#

SDK qb Runtime Library: model.h Source File
SDK qb Runtime Library v1.3
MCS001-EN
model.h
Go to the documentation of this file.
1// Copyright ⓒ 2019- Mobilint Inc. All rights reserved.
5
6#ifndef QBRUNTIME_MODEL_H_
7#define QBRUNTIME_MODEL_H_
8
9#include <cstdint>
10#ifndef _MSC_VER
11#include <experimental/propagate_const>
12#endif
13#include <memory>
14#include <string>
15#include <vector>
16
17#include "qbruntime/export.h"
18#include "qbruntime/future.h"
19#include "qbruntime/model_variant_handle.h"
20#include "qbruntime/ndarray.h"
21#include "qbruntime/pinned_memory.h"
22#include "qbruntime/status_code.h"
23#include "qbruntime/type.h"
24
25namespace mobilint {
26
31
32class Accelerator;
33class ModelImpl;
34
41class QBRUNTIME_EXPORT Model {
42public:
57 static std::unique_ptr<Model> create(const std::string& mxq_path, StatusCode& sc);
58
74 static std::unique_ptr<Model> create(const std::string& mxq_path,
75 const ModelConfig& config, StatusCode& sc);
76
77 Model(const Model& other) = delete;
78 Model(Model&& other) noexcept;
79 Model& operator=(const Model& rhs) = delete;
80 Model& operator=(Model&& rhs) noexcept;
81 ~Model();
82
91 StatusCode launch(Accelerator& acc);
92
101 StatusCode dispose();
102
108 CoreMode getCoreMode() const;
109
118 bool isTarget(CoreId core_id) const;
119
125 std::vector<CoreId> getTargetCores() const;
126
144
155 StatusCode infer(const std::vector<NDArray<float>>& input,
156 std::vector<NDArray<float>>& output);
157
169 std::vector<NDArray<float>> infer(const std::vector<NDArray<float>>& input,
170 StatusCode& sc);
171
183 StatusCode infer(const std::vector<float*>& input,
184 std::vector<std::vector<float>>& output);
185
200 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
201 StatusCode& sc);
202
219 StatusCode infer(const std::vector<float*>& input,
220 std::vector<std::vector<float>>& output,
221 const std::vector<std::vector<int64_t>>& shape);
238 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
239 const std::vector<std::vector<int64_t>>& shape,
240 StatusCode& sc);
241
255 StatusCode infer(const std::vector<NDArray<float>>& input,
256 std::vector<NDArray<float>>& output, uint32_t cache_size);
257
270 StatusCode infer(const std::vector<NDArray<float>>& input,
271 std::vector<NDArray<float>>& output,
272 const std::vector<BatchParam>& params);
273
290 std::vector<NDArray<float>> infer(const std::vector<NDArray<float>>& input,
291 uint32_t cache_size, StatusCode& sc);
292
308 std::vector<NDArray<float>> infer(const std::vector<NDArray<float>>& input,
309 const std::vector<BatchParam>& params,
310 StatusCode& sc);
311
327 StatusCode infer(const std::vector<float*>& input,
328 std::vector<std::vector<float>>& output,
329 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size);
330
349 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
350 const std::vector<std::vector<int64_t>>& shape,
351 uint32_t cache_size, StatusCode& sc);
352
367 StatusCode infer(const std::vector<float*>& input,
368 std::vector<std::vector<float>>& output,
369 const std::vector<std::vector<int64_t>>& shape,
370 const std::vector<BatchParam>& params);
371
389 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
390 const std::vector<std::vector<int64_t>>& shape,
391 const std::vector<BatchParam>& params,
392 StatusCode& sc);
393
395
420
431 StatusCode inferCHW(const std::vector<NDArray<float>>& input,
432 std::vector<NDArray<float>>& output);
433
445 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<float>>& input,
446 StatusCode& sc);
447
459 StatusCode inferCHW(const std::vector<float*>& input,
460 std::vector<std::vector<float>>& output);
461
476 std::vector<std::vector<float>> inferCHW(const std::vector<float*>& input,
477 StatusCode& sc);
478
495 StatusCode inferCHW(const std::vector<float*>& input,
496 std::vector<std::vector<float>>& output,
497 const std::vector<std::vector<int64_t>>& shape);
498
515 std::vector<std::vector<float>> inferCHW(
516 const std::vector<float*>& input, const std::vector<std::vector<int64_t>>& shape,
517 StatusCode& sc);
518
532 StatusCode inferCHW(const std::vector<NDArray<float>>& input,
533 std::vector<NDArray<float>>& output, uint32_t cache_size);
534
551 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<float>>& input,
552 uint32_t cache_size, StatusCode& sc);
553
569 StatusCode inferCHW(const std::vector<float*>& input,
570 std::vector<std::vector<float>>& output,
571 const std::vector<std::vector<int64_t>>& shape,
572 uint32_t cache_size);
573
592 std::vector<std::vector<float>> inferCHW(
593 const std::vector<float*>& input, const std::vector<std::vector<int64_t>>& shape,
594 uint32_t cache_size, StatusCode& sc);
595
597
609
610 StatusCode infer(const std::vector<NDArray<uint8_t>>& input,
611 std::vector<NDArray<float>>& output);
612 std::vector<NDArray<float>> infer(const std::vector<NDArray<uint8_t>>& input,
613 StatusCode& sc);
614 StatusCode infer(const std::vector<uint8_t*>& input,
615 std::vector<std::vector<float>>& output);
616 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
617 StatusCode& sc);
618 StatusCode infer(const std::vector<uint8_t*>& input,
619 std::vector<std::vector<float>>& output,
620 const std::vector<std::vector<int64_t>>& shape);
621 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
622 const std::vector<std::vector<int64_t>>& shape,
623 StatusCode& sc);
624
625 StatusCode infer(const std::vector<NDArray<uint8_t>>& input,
626 std::vector<NDArray<float>>& output, uint32_t cache_size);
627 std::vector<NDArray<float>> infer(const std::vector<NDArray<uint8_t>>& input,
628 uint32_t cache_size, StatusCode& sc);
629 StatusCode infer(const std::vector<uint8_t*>& input,
630 std::vector<std::vector<float>>& output,
631 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size);
632 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
633 const std::vector<std::vector<int64_t>>& shape,
634 uint32_t cache_size, StatusCode& sc);
635
636 StatusCode infer(const std::vector<NDArray<uint8_t>>& input,
637 std::vector<NDArray<float>>& output,
638 const std::vector<BatchParam>& params);
639 std::vector<NDArray<float>> infer(const std::vector<NDArray<uint8_t>>& input,
640 const std::vector<BatchParam>& params,
641 StatusCode& sc);
642 StatusCode infer(const std::vector<uint8_t*>& input,
643 std::vector<std::vector<float>>& output,
644 const std::vector<std::vector<int64_t>>& shape,
645 const std::vector<BatchParam>& params);
646 std::vector<std::vector<float>> infer(const std::vector<uint8_t*>& input,
647 const std::vector<std::vector<int64_t>>& shape,
648 const std::vector<BatchParam>& params,
649 StatusCode& sc);
650
652
661 StatusCode inferCHW(const std::vector<NDArray<uint8_t>>& input,
662 std::vector<NDArray<float>>& output);
663 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<uint8_t>>& input,
664 StatusCode& sc);
665 StatusCode inferCHW(const std::vector<uint8_t*>& input,
666 std::vector<std::vector<float>>& output);
667 std::vector<std::vector<float>> inferCHW(const std::vector<uint8_t*>& input,
668 StatusCode& sc);
669 StatusCode inferCHW(const std::vector<uint8_t*>& input,
670 std::vector<std::vector<float>>& output,
671 const std::vector<std::vector<int64_t>>& shape);
672 std::vector<std::vector<float>> inferCHW(
673 const std::vector<uint8_t*>& input,
674 const std::vector<std::vector<int64_t>>& shape, StatusCode& sc);
675
676 StatusCode inferCHW(const std::vector<NDArray<uint8_t>>& input,
677 std::vector<NDArray<float>>& output, uint32_t cache_size);
678 std::vector<NDArray<float>> inferCHW(const std::vector<NDArray<uint8_t>>& input,
679 uint32_t cache_size, StatusCode& sc);
680 StatusCode inferCHW(const std::vector<uint8_t*>& input,
681 std::vector<std::vector<float>>& output,
682 const std::vector<std::vector<int64_t>>& shape,
683 uint32_t cache_size);
684 std::vector<std::vector<float>> inferCHW(
685 const std::vector<uint8_t*>& input,
686 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size,
687 StatusCode& sc);
689
706
707 StatusCode infer(const std::vector<NDArray<int8_t>>& input,
708 std::vector<NDArray<int8_t>>& output);
709 std::vector<NDArray<int8_t>> infer(const std::vector<NDArray<int8_t>>& input,
710 StatusCode& sc);
711 StatusCode infer(const std::vector<int8_t*>& input,
712 std::vector<std::vector<int8_t>>& output);
713 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
714 StatusCode& sc);
715 StatusCode infer(const std::vector<int8_t*>& input,
716 std::vector<std::vector<int8_t>>& output,
717 const std::vector<std::vector<int64_t>>& shape);
718 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
719 const std::vector<std::vector<int64_t>>& shape,
720 StatusCode& sc);
721
722 StatusCode infer(const std::vector<NDArray<int8_t>>& input,
723 std::vector<NDArray<int8_t>>& output, uint32_t cache_size);
724 std::vector<NDArray<int8_t>> infer(const std::vector<NDArray<int8_t>>& input,
725 uint32_t cache_size, StatusCode& sc);
726 StatusCode infer(const std::vector<int8_t*>& input,
727 std::vector<std::vector<int8_t>>& output,
728 const std::vector<std::vector<int64_t>>& shape, uint32_t cache_size);
729 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
730 const std::vector<std::vector<int64_t>>& shape,
731 uint32_t cache_size, StatusCode& sc);
732
733 StatusCode infer(const std::vector<NDArray<int8_t>>& input,
734 std::vector<NDArray<int8_t>>& output,
735 const std::vector<BatchParam>& params);
736 std::vector<NDArray<int8_t>> infer(const std::vector<NDArray<int8_t>>& input,
737 const std::vector<BatchParam>& params,
738 StatusCode& sc);
739 StatusCode infer(const std::vector<int8_t*>& input,
740 std::vector<std::vector<int8_t>>& output,
741 const std::vector<std::vector<int64_t>>& shape,
742 const std::vector<BatchParam>& params);
743 std::vector<std::vector<int8_t>> infer(const std::vector<int8_t*>& input,
744 const std::vector<std::vector<int64_t>>& shape,
745 const std::vector<BatchParam>& params,
746 StatusCode& sc);
747
749
762 StatusCode inferCHW(const std::vector<NDArray<int8_t>>& input,
763 std::vector<NDArray<int8_t>>& output);
764 std::vector<NDArray<int8_t>> inferCHW(const std::vector<NDArray<int8_t>>& input,
765 StatusCode& sc);
766 StatusCode inferCHW(const std::vector<int8_t*>& input,
767 std::vector<std::vector<int8_t>>& output);
768 std::vector<std::vector<int8_t>> inferCHW(const std::vector<int8_t*>& input,
769 StatusCode& sc);
770 StatusCode inferCHW(const std::vector<int8_t*>& input,
771 std::vector<std::vector<int8_t>>& output,
772 const std::vector<std::vector<int64_t>>& shape);
773 std::vector<std::vector<int8_t>> inferCHW(
774 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
775 StatusCode& sc);
776
777 StatusCode inferCHW(const std::vector<NDArray<int8_t>>& input,
778 std::vector<NDArray<int8_t>>& output, uint32_t cache_size);
779 std::vector<NDArray<int8_t>> inferCHW(const std::vector<NDArray<int8_t>>& input,
780 uint32_t cache_size, StatusCode& sc);
781 StatusCode inferCHW(const std::vector<int8_t*>& input,
782 std::vector<std::vector<int8_t>>& output,
783 const std::vector<std::vector<int64_t>>& shape,
784 uint32_t cache_size);
785 std::vector<std::vector<int8_t>> inferCHW(
786 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
787 uint32_t cache_size, StatusCode& sc);
789
802 std::vector<NDArray<float>> inferToFloat(const std::vector<NDArray<int8_t>>& input,
803 StatusCode& sc);
804 std::vector<std::vector<float>> inferToFloat(const std::vector<int8_t*>& input,
805 StatusCode& sc);
806 std::vector<std::vector<float>> inferToFloat(
807 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
808 StatusCode& sc);
809
810 std::vector<NDArray<float>> inferToFloat(const std::vector<NDArray<int8_t>>& input,
811 uint32_t cache_size, StatusCode& sc);
812 std::vector<std::vector<float>> inferToFloat(
813 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
814 uint32_t cache_size, StatusCode& sc);
816
829 std::vector<NDArray<float>> inferCHWToFloat(const std::vector<NDArray<int8_t>>& input,
830 StatusCode& sc);
831 std::vector<std::vector<float>> inferCHWToFloat(const std::vector<int8_t*>& input,
832 StatusCode& sc);
833 std::vector<std::vector<float>> inferCHWToFloat(
834 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
835 StatusCode& sc);
836
837 std::vector<NDArray<float>> inferCHWToFloat(const std::vector<NDArray<int8_t>>& input,
838 uint32_t cache_size, StatusCode& sc);
839 std::vector<std::vector<float>> inferCHWToFloat(
840 const std::vector<int8_t*>& input, const std::vector<std::vector<int64_t>>& shape,
841 uint32_t cache_size, StatusCode& sc);
843
866 StatusCode inferBuffer(const std::vector<Buffer>& input, std::vector<Buffer>& output,
867 const std::vector<std::vector<int64_t>>& shape = {},
868 uint32_t cache_size = 0);
869 StatusCode inferBuffer(const std::vector<std::vector<Buffer>>& input,
870 std::vector<std::vector<Buffer>>& output,
871 const std::vector<std::vector<int64_t>>& shape = {},
872 uint32_t cache_size = 0);
874
892 StatusCode inferBufferToFloat(const std::vector<Buffer>& input,
893 std::vector<NDArray<float>>& output,
894 const std::vector<std::vector<int64_t>>& shape = {},
895 uint32_t cache_size = 0);
896 StatusCode inferBufferToFloat(const std::vector<std::vector<Buffer>>& input,
897 std::vector<NDArray<float>>& output,
898 const std::vector<std::vector<int64_t>>& shape = {},
899 uint32_t cache_size = 0);
900 StatusCode inferBufferToFloat(const std::vector<Buffer>& input,
901 std::vector<std::vector<float>>& output,
902 const std::vector<std::vector<int64_t>>& shape = {},
903 uint32_t cache_size = 0);
904 StatusCode inferBufferToFloat(const std::vector<std::vector<Buffer>>& input,
905 std::vector<std::vector<float>>& output,
906 const std::vector<std::vector<int64_t>>& shape = {},
907 uint32_t cache_size = 0);
909
923 StatusCode inferSpeedrun(int variant_idx = 0);
924
979
991 Future<float> inferAsync(const std::vector<NDArray<float>>& input, StatusCode& sc);
992
1004 Future<float> inferAsyncCHW(const std::vector<NDArray<float>>& input, StatusCode& sc);
1005
1016 Future<int8_t> inferAsync(const std::vector<NDArray<int8_t>>& input, StatusCode& sc);
1017
1028 Future<int8_t> inferAsyncCHW(const std::vector<NDArray<int8_t>>& input,
1029 StatusCode& sc);
1030
1041 Future<float> inferAsyncToFloat(const std::vector<NDArray<int8_t>>& input,
1042 StatusCode& sc);
1043
1054 Future<float> inferAsyncCHWToFloat(const std::vector<NDArray<int8_t>>& input,
1055 StatusCode& sc);
1056
1067 Future<float> inferAsync(const std::vector<NDArray<uint8_t>>& input, StatusCode& sc);
1068
1079 Future<float> inferAsyncCHW(const std::vector<NDArray<uint8_t>>& input,
1080 StatusCode& sc);
1081
1083
1105
1106 // Acquire buffer
1107 std::vector<Buffer> acquireInputBuffer(
1108 const std::vector<std::vector<int>>& seqlens = {}) const;
1109 std::vector<Buffer> acquireOutputBuffer(
1110 const std::vector<std::vector<int>>& seqlens = {}) const;
1111 std::vector<std::vector<Buffer>> acquireInputBuffers(
1112 const int batch_size, const std::vector<std::vector<int>>& seqlens = {}) const;
1113 std::vector<std::vector<Buffer>> acquireOutputBuffers(
1114 const int batch_size, const std::vector<std::vector<int>>& seqlens = {}) const;
1115
1116 // Deallocate acquired Input/Output buffer
1117 StatusCode releaseBuffer(std::vector<Buffer>& buffer) const;
1118 StatusCode releaseBuffers(std::vector<std::vector<Buffer>>& buffers) const;
1119
1120 // Reposition single batch
1121 StatusCode repositionInputs(const std::vector<float*>& input,
1122 std::vector<Buffer>& input_buf,
1123 const std::vector<std::vector<int>>& seqlens = {}) const;
1124 StatusCode repositionOutputs(const std::vector<Buffer>& output_buf,
1125 std::vector<float*>& output,
1126 const std::vector<std::vector<int>>& seqlens = {}) const;
1127 StatusCode repositionOutputs(const std::vector<Buffer>& output_buf,
1128 std::vector<std::vector<float>>& output,
1129 const std::vector<std::vector<int>>& seqlens = {}) const;
1130 StatusCode repositionInputs(const std::vector<uint8_t*>& input,
1131 std::vector<Buffer>& input_buf,
1132 const std::vector<std::vector<int>>& seqlens = {}) const;
1133
1134 // Reposition multiple batches
1135 StatusCode repositionInputs(const std::vector<float*>& input,
1136 std::vector<std::vector<Buffer>>& input_buf,
1137 const std::vector<std::vector<int>>& seqlens = {}) const;
1138 StatusCode repositionOutputs(const std::vector<std::vector<Buffer>>& output_buf,
1139 std::vector<float*>& output,
1140 const std::vector<std::vector<int>>& seqlens = {}) const;
1141 StatusCode repositionOutputs(const std::vector<std::vector<Buffer>>& output_buf,
1142 std::vector<std::vector<float>>& output,
1143 const std::vector<std::vector<int>>& seqlens = {}) const;
1144 StatusCode repositionInputs(const std::vector<uint8_t*>& input,
1145 std::vector<std::vector<Buffer>>& input_buf,
1146 const std::vector<std::vector<int>>& seqlens = {}) const;
1148
1157 int getNumModelVariants() const;
1158
1172 std::unique_ptr<ModelVariantHandle> getModelVariantHandle(int variant_idx,
1173 StatusCode& sc) const;
1174
1180 const std::vector<std::vector<int64_t>>& getModelInputShape() const;
1181
1187 const std::vector<std::vector<int64_t>>& getModelOutputShape() const;
1188
1194 const std::vector<BufferInfo>& getInputBufferInfo() const;
1195
1201 const std::vector<BufferInfo>& getOutputBufferInfo() const;
1202
1208 std::vector<Scale> getInputScale() const;
1209
1215 std::vector<Scale> getOutputScale() const;
1216
1222 DataType getModelInputDataType() const;
1223
1229 DataType getModelOutputDataType() const;
1230
1239 uint32_t getIdentifier() const;
1240
1246 std::string getModelPath() const;
1247
1253 std::vector<CacheInfo> getCacheInfos() const;
1254
1261
1273 StatusCode dumpCacheMemory(std::vector<std::vector<int8_t>>& bufs, int cache_id = 0);
1274
1285 std::vector<std::vector<int8_t>> dumpCacheMemory(StatusCode& sc);
1286 std::vector<std::vector<int8_t>> dumpCacheMemory(int cache_id, StatusCode& sc);
1287
1299 StatusCode dumpCacheMemory(const std::string& cache_dir, int cache_id = 0);
1300
1312 StatusCode loadCacheMemory(const std::vector<std::vector<int8_t>>& bufs,
1313 int cache_id = 0);
1314
1326 StatusCode loadCacheMemory(const std::string& cache_dir, int cache_id = 0);
1327
1341 int filterCacheTail(int cache_size, int tail_size, const std::vector<bool>& mask,
1342 StatusCode& sc);
1343
1357 int moveCacheTail(int num_head, int num_tail, int cache_size, StatusCode& sc);
1358
1386 template <typename T>
1387 std::vector<PinnedMemory<float>> inferPinnedMemory(
1388 const std::vector<PinnedMemory<T>>& inputs, uint32_t cache_size, StatusCode& sc);
1389
1417 template <typename T>
1418 StatusCode inferPinnedMemory(const std::vector<PinnedMemory<T>>& inputs,
1419 std::vector<PinnedMemory<float>>& outputs,
1420 uint32_t cache_size);
1421
1423
1430
1434 StatusCode infer(const std::vector<float*>& input,
1435 std::vector<std::vector<float>>& output, int batch_size);
1436
1440 std::vector<std::vector<float>> infer(const std::vector<float*>& input,
1441 int batch_size, StatusCode& sc);
1442
1446 uint64_t getLatencyConsumed(const int npu_op_idx) const;
1447
1451 uint64_t getLatencyFinished(const int npu_op_idx) const;
1452
1454
1455private:
1456 Model();
1457
1458#ifndef _MSC_VER
1459 std::experimental::propagate_const<std::unique_ptr<ModelImpl>> mImpl;
1460#else
1461 std::unique_ptr<ModelImpl> mImpl;
1462#endif
1463
1464 friend class Accelerator;
1465};
1466
1468
1469} // namespace mobilint
1470
1471#endif
mobilint::Accelerator
Represents an accelerator, i.e., an NPU, used for executing models.
Definition acc.h:36
mobilint::Future
Represents a future for retrieving the result of asynchronous inference.
Definition future.h:43
mobilint::ModelConfig
Configures a core mode and core allocation of a model for NPU inference.
Definition type.h:239
mobilint::Model::infer
StatusCode infer(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output, const std::vector< BatchParam > &params)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::getModelPath
std::string getModelPath() const
Returns the path to the MXQ model file associated with the Model.
mobilint::Model::inferAsyncToFloat
Future< float > inferAsyncToFloat(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-float asynchronous inference.
mobilint::Model::getLatencyConsumed
uint64_t getLatencyConsumed(const int npu_op_idx) const
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, int batch_size, StatusCode &sc)
mobilint::Model::isTarget
bool isTarget(CoreId core_id) const
Checks if the NPU core specified by CoreId is the target of the model. In other words,...
mobilint::Model::create
static std::unique_ptr< Model > create(const std::string &mxq_path, StatusCode &sc)
Creates a Model object from the specified MXQ model file.
mobilint::Model::inferAsyncCHW
Future< float > inferAsyncCHW(const std::vector< NDArray< float > > &input, StatusCode &sc)
Initiates asynchronous inference with input in NCHW (batch N, channels C, height H,...
mobilint::Model::infer
std::vector< NDArray< float > > infer(const std::vector< NDArray< float > > &input, StatusCode &sc)
This overload differs from the above function in that it directly returns the inference results inste...
mobilint::Model::inferAsync
Future< int8_t > inferAsync(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-int8_t asynchronous inference.
mobilint::Model::infer
std::vector< NDArray< float > > infer(const std::vector< NDArray< float > > &input, const std::vector< BatchParam > &params, StatusCode &sc)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::inferAsyncCHWToFloat
Future< float > inferAsyncCHWToFloat(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-float asynchronous inference.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape, const std::vector< BatchParam > &params)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::Model::getOutputScale
std::vector< Scale > getOutputScale() const
Returns the output quantization scale(s) of the model.
mobilint::Model::loadCacheMemory
StatusCode loadCacheMemory(const std::vector< std::vector< int8_t > > &bufs, int cache_id=0)
Loads the KV cache memory from buffers.
mobilint::Model::dumpCacheMemory
StatusCode dumpCacheMemory(std::vector< std::vector< int8_t > > &bufs, int cache_id=0)
Dumps the KV cache memory into buffers.
mobilint::Model::getModelOutputShape
const std::vector< std::vector< int64_t > > & getModelOutputShape() const
Returns the output shape of the model.
mobilint::Model::getModelVariantHandle
std::unique_ptr< ModelVariantHandle > getModelVariantHandle(int variant_idx, StatusCode &sc) const
Retrieves a handle to the specified model variant.
mobilint::Model::loadCacheMemory
StatusCode loadCacheMemory(const std::string &cache_dir, int cache_id=0)
Loads the KV cache memory from files in the specified directory.
mobilint::Model::launch
StatusCode launch(Accelerator &acc)
Launches the model on the specified Accelerator, which represents the actual NPU.
mobilint::Model::inferAsync
Future< float > inferAsync(const std::vector< NDArray< uint8_t > > &input, StatusCode &sc)
This overload supports uint8_t-to-float asynchronous inference.
mobilint::Model::getTargetCores
std::vector< CoreId > getTargetCores() const
Returns the NPU cores the model is configured to use.
mobilint::Model::inferCHW
std::vector< std::vector< float > > inferCHW(const std::vector< float * > &input, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
std::vector< std::vector< float > > inferCHW(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::getInputBufferInfo
const std::vector< BufferInfo > & getInputBufferInfo() const
Returns the input buffer information for the model.
mobilint::Model::moveCacheTail
int moveCacheTail(int num_head, int num_tail, int cache_size, StatusCode &sc)
Moves the tail of the KV cache memory to the end of the head.
mobilint::Model::getModelInputDataType
DataType getModelInputDataType() const
Returns a data type for model inputs.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::infer
StatusCode infer(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output)
Performs inference.
mobilint::Model::getCoreMode
CoreMode getCoreMode() const
Retrieves the core mode of the model.
mobilint::Model::getNumModelVariants
int getNumModelVariants() const
Returns the total number of model variants available in this model.
mobilint::Model::inferAsync
Future< float > inferAsync(const std::vector< NDArray< float > > &input, StatusCode &sc)
Initiates asynchronous inference with input in NHWC (batch N, height H, width W, channels C) or HWC f...
mobilint::Model::getIdentifier
uint32_t getIdentifier() const
Returns the model's unique identifier.
mobilint::Model::inferCHW
std::vector< NDArray< float > > inferCHW(const std::vector< NDArray< float > > &input, StatusCode &sc)
This overload differs from the above function in that it directly returns the inference results inste...
mobilint::Model::infer
std::vector< NDArray< float > > infer(const std::vector< NDArray< float > > &input, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::getModelOutputDataType
DataType getModelOutputDataType() const
Returns a data type for model outputs.
mobilint::Model::dispose
StatusCode dispose()
Disposes of the model loaded onto the NPU.
mobilint::Model::inferPinnedMemory
std::vector< PinnedMemory< float > > inferPinnedMemory(const std::vector< PinnedMemory< T > > &inputs, uint32_t cache_size, StatusCode &sc)
Performs inference directly on pinned memory buffers.
mobilint::Model::inferPinnedMemory
StatusCode inferPinnedMemory(const std::vector< PinnedMemory< T > > &inputs, std::vector< PinnedMemory< float > > &outputs, uint32_t cache_size)
Performs inference directly on pinned memory buffers.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::create
static std::unique_ptr< Model > create(const std::string &mxq_path, const ModelConfig &config, StatusCode &sc)
Creates a Model object from the specified MXQ model file and configuration.
mobilint::Model::filterCacheTail
int filterCacheTail(int cache_size, int tail_size, const std::vector< bool > &mask, StatusCode &sc)
Filter the tail of the KV cache memory.
mobilint::Model::getModelInputShape
const std::vector< std::vector< int64_t > > & getModelInputShape() const
Returns the input shape of the model.
mobilint::Model::inferCHW
std::vector< NDArray< float > > inferCHW(const std::vector< NDArray< float > > &input, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::inferAsyncCHW
Future< int8_t > inferAsyncCHW(const std::vector< NDArray< int8_t > > &input, StatusCode &sc)
This overload supports int8_t-to-int8_t asynchronous inference.
mobilint::Model::dumpCacheMemory
StatusCode dumpCacheMemory(const std::string &cache_dir, int cache_id=0)
Dumps KV cache memory to files in the specified directory.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::getOutputBufferInfo
const std::vector< BufferInfo > & getOutputBufferInfo() const
Returns the output buffer information of the model.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::getCacheInfos
std::vector< CacheInfo > getCacheInfos() const
Returns informations of KV-cache of the model.
mobilint::Model::infer
StatusCode infer(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::inferAsyncCHW
Future< float > inferAsyncCHW(const std::vector< NDArray< uint8_t > > &input, StatusCode &sc)
This overload supports uint8_t-to-float asynchronous inference.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
std::vector< std::vector< float > > inferCHW(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size, StatusCode &sc)
This overload supports inference with KV cache.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, StatusCode &sc)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, int batch_size)
mobilint::Model::dumpCacheMemory
std::vector< std::vector< int8_t > > dumpCacheMemory(StatusCode &sc)
Dumps the KV cache memory into buffers.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< float * > &input, std::vector< std::vector< float > > &output)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::getInputScale
std::vector< Scale > getInputScale() const
Returns the input quantization scale(s) of the model.
mobilint::Model::inferSpeedrun
StatusCode inferSpeedrun(int variant_idx=0)
Development-only API for measuring pure NPU inference speed.
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< NDArray< float > > &input, std::vector< NDArray< float > > &output)
Performs inference.
mobilint::Model::infer
StatusCode infer(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape)
This overload is provided for convenience but may result in additional data copies within the qbrunti...
mobilint::Model::getLatencyFinished
uint64_t getLatencyFinished(const int npu_op_idx) const
mobilint::Model::inferCHW
StatusCode inferCHW(const std::vector< float * > &input, std::vector< std::vector< float > > &output, const std::vector< std::vector< int64_t > > &shape, uint32_t cache_size)
This overload supports inference with KV cache.
mobilint::Model::dumpCacheMemory
std::vector< std::vector< int8_t > > dumpCacheMemory(int cache_id, StatusCode &sc)
Dumps the KV cache memory into buffers.
mobilint::Model::infer
std::vector< std::vector< float > > infer(const std::vector< float * > &input, const std::vector< std::vector< int64_t > > &shape, const std::vector< BatchParam > &params, StatusCode &sc)
This overload is supports inference with BatchParam for BatchLLM.
mobilint::NDArray
A class representing an N-dimensional array (NDArray).
Definition ndarray.h:77
mobilint::PinnedMemory
A buffer of NPU-accessible pinned (physically contiguous) memory.
Definition pinned_memory.h:56
mobilint::DataType
DataType
DataType.
Definition type.h:512
mobilint::CoreMode
CoreMode
Defines the core mode for NPU execution.
Definition type.h:174
mobilint::StatusCode
StatusCode
Enumerates status codes for the qbruntime.
Definition status_code.h:26
model_variant_handle.h
pinned_memory.h
status_code.h
mobilint::CoreId
Represents a unique identifier for an NPU core.
Definition type.h:127
Generated by doxygen 1.14.0