type.h Source File

type.h Source File#

SDK qb Runtime Library: type.h Source File
SDK qb Runtime Library v1.1
MCS001-
type.h
Go to the documentation of this file.
1// Copyright ⓒ 2019- Mobilint Inc. All rights reserved.
5
6#ifndef QBRUNTIME_TYPE_H_
7#define QBRUNTIME_TYPE_H_
8
9#include <cstdint>
10#include <string>
11#include <tuple>
12#include <vector>
13
14#include "qbruntime/export.h"
15
16namespace mobilint {
23
29QBRUNTIME_EXPORT std::string getQbRuntimeVersion();
30
36QBRUNTIME_EXPORT std::string getQbRuntimeGitVersion();
37
45QBRUNTIME_EXPORT std::string getQbRuntimeVendor();
46
55QBRUNTIME_EXPORT std::string getQbRuntimeProduct();
56
64enum class Cluster : int32_t {
65 Cluster0 = 1 << 16,
66 Cluster1 = 2 << 16,
67 Error = 0x7FFF'0000,
68};
69
77enum class Core : int32_t {
78 Core0 = 1,
79 Core1 = 2,
80 Core2 = 3,
81 Core3 = 4,
82 All = 0x0000'FFFC,
83 GlobalCore = 0x0000'FFFE,
84 Error = 0x0000'FFFF,
85};
86
94
98struct Scale {
99 std::vector<float> scale_list;
100 float scale = 0.0F;
101 bool is_uniform = false;
102
109 float operator[](int i) const {
110 if (is_uniform) {
111 return scale;
112 }
113 return scale_list[i];
114 }
115};
116
123struct CoreId {
124 Cluster cluster = Cluster::Error;
125 Core core = Core::Error;
126
133 bool operator==(const CoreId& rhs) const {
134 return std::tie(cluster, core) == std::tie(rhs.cluster, rhs.core);
135 }
136
143 bool operator<(const CoreId& rhs) const {
144 return std::tie(cluster, core) < std::tie(rhs.cluster, rhs.core);
145 }
146};
147
153struct Buffer {
154 int8_t* data = nullptr;
155 uint64_t size = 0;
156};
157
170enum class CoreMode : uint8_t {
171 Single = 0,
172 Multi = 1,
173 Global = 2,
174 Global4 = 3,
175 Global8 = 4,
176 Auto = 0xE,
177 Error = 0xF,
178};
179
183struct BufferInfo {
184 // clang-format off
185 uint32_t original_height = 0;
186 uint32_t original_width = 0;
187 uint32_t original_channel = 0;
188 uint32_t reshaped_height = 0;
189 uint32_t reshaped_width = 0;
190 uint32_t reshaped_channel = 0;
191 uint32_t height = 0;
192 uint32_t width = 0;
193 uint32_t channel = 0;
194 uint32_t max_height = 0;
195 uint32_t max_width = 0;
196 uint32_t max_channel = 0;
197 uint32_t max_cache_size = 0;
198 // clang-format on
199
204 uint32_t original_size() const {
205 return original_height * original_width * original_channel;
206 }
207
212 uint32_t reshaped_size() const {
213 return reshaped_height * reshaped_width * reshaped_channel;
214 }
215
220 uint32_t size() const { return height * width * channel; }
221};
222
235class QBRUNTIME_EXPORT ModelConfig {
236public:
241 ModelConfig();
242
255 bool setAutoCoreMode();
256
270 bool setSingleCoreMode(int num_cores);
271
283 bool setSingleCoreMode(std::vector<CoreId> core_ids);
284
297 bool setMultiCoreMode(std::vector<Cluster> clusters = {Cluster::Cluster0,
298 Cluster::Cluster1});
299
313 bool setGlobal4CoreMode(std::vector<Cluster> clusters = {Cluster::Cluster0,
314 Cluster::Cluster1});
315
325 bool setGlobal8CoreMode();
326
334 CoreMode getCoreMode() const { return mCoreMode; }
335
344 CoreAllocationPolicy getCoreAllocationPolicy() const { return mCoreAllocationPolicy; }
345
354 int getNumCores() const { return mNumCores; }
355
371 bool forceSingleNPUBundle(int npu_bundle_index);
372
382 int getForcedNPUBundleIndex() const { return mForcedNPUBundleIndex; }
383
394 const std::vector<CoreId>& getCoreIds() const { return mCoreIds; }
395
396 const std::vector<Cluster>& getClusters() const { return mClusters; }
397
413 void setAsyncPipelineEnabled(bool enable);
414
420 bool getAsyncPipelineEnabled() const { return mAsyncPipelineEnabled; }
421
442 void setActivationSlots(int count);
443
451 int getActivationSlots() const { return mActivationSlots; }
452
453 explicit ModelConfig(int num_cores);
454
455 bool setGlobalCoreMode(std::vector<Cluster> clusters);
456
460 std::vector<uint64_t> early_latencies;
464 std::vector<uint64_t> finish_latencies;
465
466private:
467 CoreMode mCoreMode = CoreMode::Auto;
468 CoreAllocationPolicy mCoreAllocationPolicy = CoreAllocationPolicy::Manual;
469 std::vector<Cluster> mClusters;
470 std::vector<CoreId> mCoreIds;
471 int mNumCores;
472 int mForcedNPUBundleIndex = -1; // -1 means single npu bundle usage is not forced.
473 bool mAsyncPipelineEnabled = false;
474 int mActivationSlots = 1;
475};
476
480enum class LogLevel : char {
481 DEBUG = 1,
482 INFO = 2,
483 WARN = 3,
484 ERR = 4,
485 FATAL = 5,
486 OFF = 6,
487};
488
492enum class CacheType : uint8_t { Default = 0, Batch, Error = 0x0F };
493
497struct CacheInfo {
498 CacheType cache_type = CacheType::Error;
499 std::string name;
500 std::string layer_hash;
501 uint64_t size = 0;
502 size_t num_batches = 0;
503};
504
508enum class DataType : uint8_t { Float32 = 0, Float16, Int8, Uint8, Error = 0x0F };
509
510QBRUNTIME_EXPORT void setLogLevel(LogLevel level);
511
524QBRUNTIME_EXPORT bool startTracingEvents(const char* path);
525
532QBRUNTIME_EXPORT void stopTracingEvents();
533
551QBRUNTIME_EXPORT std::string getModelSummary(const std::string& mxq_path);
552
558QBRUNTIME_EXPORT std::vector<int> getAvailableDeviceNumbers();
559
561
562} // namespace mobilint
563
564#endif
mobilint::ModelConfig::setGlobal8CoreMode
bool setGlobal8CoreMode()
Sets the model to use global8-core mode for inference.
mobilint::ModelConfig::setGlobalCoreMode
bool setGlobalCoreMode(std::vector< Cluster > clusters)
mobilint::ModelConfig::getCoreMode
CoreMode getCoreMode() const
Gets the core mode to be applied to the model.
Definition type.h:334
mobilint::ModelConfig::getAsyncPipelineEnabled
bool getAsyncPipelineEnabled() const
Returns whether the asynchronous pipeline is enabled in this configuration.
Definition type.h:420
mobilint::ModelConfig::setGlobal4CoreMode
bool setGlobal4CoreMode(std::vector< Cluster > clusters={Cluster::Cluster0, Cluster::Cluster1})
Sets the model to use global4-core mode for inference with a specified set of NPU clusters.
mobilint::ModelConfig::setAutoCoreMode
bool setAutoCoreMode()
Sets the model to detect CoreMode automatically.
mobilint::ModelConfig::setAsyncPipelineEnabled
void setAsyncPipelineEnabled(bool enable)
Enables or disables the asynchronous pipeline required for asynchronous inference.
mobilint::ModelConfig::ModelConfig
ModelConfig(int num_cores)
mobilint::ModelConfig::setSingleCoreMode
bool setSingleCoreMode(int num_cores)
Sets the model to use single-core mode for inference with a specified number of local cores.
mobilint::ModelConfig::ModelConfig
ModelConfig()
Default constructor. This default-constructed object is initially set to auto-core mode.
mobilint::ModelConfig::setMultiCoreMode
bool setMultiCoreMode(std::vector< Cluster > clusters={Cluster::Cluster0, Cluster::Cluster1})
Sets the model to use multi-core mode for batch inference.
mobilint::ModelConfig::getActivationSlots
int getActivationSlots() const
Returns activation buffer slot count.
Definition type.h:451
mobilint::ModelConfig::getCoreIds
const std::vector< CoreId > & getCoreIds() const
Returns the list of NPU CoreIds to be used for model inference.
Definition type.h:394
mobilint::ModelConfig::early_latencies
std::vector< uint64_t > early_latencies
Definition type.h:460
mobilint::ModelConfig::finish_latencies
std::vector< uint64_t > finish_latencies
Definition type.h:464
mobilint::ModelConfig::setSingleCoreMode
bool setSingleCoreMode(std::vector< CoreId > core_ids)
Sets the model to use single-core mode for inference with a specific set of NPU local cores.
mobilint::ModelConfig::getCoreAllocationPolicy
CoreAllocationPolicy getCoreAllocationPolicy() const
Gets the core allocation policy to be applied to the model.
Definition type.h:344
mobilint::ModelConfig::forceSingleNPUBundle
bool forceSingleNPUBundle(int npu_bundle_index)
Forces the use of a specific NPU bundle.
mobilint::ModelConfig::getNumCores
int getNumCores() const
Gets the number of cores to be allocated for the model.
Definition type.h:354
mobilint::ModelConfig::getForcedNPUBundleIndex
int getForcedNPUBundleIndex() const
Retrieves the index of the forced NPU bundle.
Definition type.h:382
mobilint::ModelConfig::setActivationSlots
void setActivationSlots(int count)
Sets activation buffer slots for multi-activation supported model.
mobilint::DataType
DataType
DataType.
Definition type.h:508
mobilint::getQbRuntimeProduct
QBRUNTIME_EXPORT std::string getQbRuntimeProduct()
Retrieves product information of the qbruntime.
mobilint::startTracingEvents
QBRUNTIME_EXPORT bool startTracingEvents(const char *path)
Starts event tracing and prepares to save the trace log to a specified file.
mobilint::getAvailableDeviceNumbers
QBRUNTIME_EXPORT std::vector< int > getAvailableDeviceNumbers()
Get the number of available NPU devices.
mobilint::getModelSummary
QBRUNTIME_EXPORT std::string getModelSummary(const std::string &mxq_path)
Generates a structured summary of the specified MXQ model.
mobilint::Cluster
Cluster
Enumerates clusters in the ARIES NPU.
Definition type.h:64
mobilint::stopTracingEvents
QBRUNTIME_EXPORT void stopTracingEvents()
Stops event tracing and writes the recorded trace log.
mobilint::getQbRuntimeVendor
QBRUNTIME_EXPORT std::string getQbRuntimeVendor()
Retrieves the vendor name of the qbruntime.
mobilint::CacheType
CacheType
CacheType.
Definition type.h:492
mobilint::LogLevel
LogLevel
LogLevel.
Definition type.h:480
mobilint::CoreMode
CoreMode
Defines the core mode for NPU execution.
Definition type.h:170
mobilint::getQbRuntimeGitVersion
QBRUNTIME_EXPORT std::string getQbRuntimeGitVersion()
Retrieves the Git commit hash of the qbruntime.
mobilint::getQbRuntimeVersion
QBRUNTIME_EXPORT std::string getQbRuntimeVersion()
Retrieves the version of the qbruntime.
mobilint::CoreAllocationPolicy
CoreAllocationPolicy
Core allocation policy.
Definition type.h:90
mobilint::Core
Core
Enumerates cores within a cluster in the ARIES NPU.
Definition type.h:77
mobilint::Cluster::Cluster1
@ Cluster1
Definition type.h:66
mobilint::Cluster::Error
@ Error
Definition type.h:67
mobilint::Cluster::Cluster0
@ Cluster0
Definition type.h:65
mobilint::CoreMode::Auto
@ Auto
Definition type.h:176
mobilint::CoreMode::Global8
@ Global8
Definition type.h:175
mobilint::CoreMode::Global4
@ Global4
Definition type.h:174
mobilint::CoreMode::Global
@ Global
Definition type.h:173
mobilint::CoreMode::Single
@ Single
Definition type.h:171
mobilint::CoreMode::Multi
@ Multi
Definition type.h:172
mobilint::CoreAllocationPolicy::Auto
@ Auto
Definition type.h:91
mobilint::CoreAllocationPolicy::Manual
@ Manual
Definition type.h:92
mobilint::Core::Core1
@ Core1
Definition type.h:79
mobilint::Core::Core3
@ Core3
Definition type.h:81
mobilint::Core::GlobalCore
@ GlobalCore
Definition type.h:83
mobilint::Core::Core0
@ Core0
Definition type.h:78
mobilint::Core::Error
@ Error
Definition type.h:84
mobilint::Core::All
@ All
Definition type.h:82
mobilint::Core::Core2
@ Core2
Definition type.h:80
mobilint::BufferInfo
Struct representing input/output buffer information.
Definition type.h:183
mobilint::BufferInfo::original_height
uint32_t original_height
Definition type.h:185
mobilint::BufferInfo::max_cache_size
uint32_t max_cache_size
Definition type.h:197
mobilint::BufferInfo::width
uint32_t width
Definition type.h:192
mobilint::BufferInfo::reshaped_channel
uint32_t reshaped_channel
Definition type.h:190
mobilint::BufferInfo::height
uint32_t height
Definition type.h:191
mobilint::BufferInfo::reshaped_height
uint32_t reshaped_height
Definition type.h:188
mobilint::BufferInfo::original_size
uint32_t original_size() const
Returns the total size of the original input/output.
Definition type.h:204
mobilint::BufferInfo::max_channel
uint32_t max_channel
Definition type.h:196
mobilint::BufferInfo::original_channel
uint32_t original_channel
Definition type.h:187
mobilint::BufferInfo::channel
uint32_t channel
Definition type.h:193
mobilint::BufferInfo::reshaped_size
uint32_t reshaped_size() const
Returns the total size of the reshaped input/output.
Definition type.h:212
mobilint::BufferInfo::max_width
uint32_t max_width
Definition type.h:195
mobilint::BufferInfo::size
uint32_t size() const
Returns the total size of the NPU input/output.
Definition type.h:220
mobilint::BufferInfo::original_width
uint32_t original_width
Definition type.h:186
mobilint::BufferInfo::max_height
uint32_t max_height
Definition type.h:194
mobilint::BufferInfo::reshaped_width
uint32_t reshaped_width
Definition type.h:189
mobilint::Buffer
A simple byte-sized buffer.
Definition type.h:153
mobilint::Buffer::size
uint64_t size
Definition type.h:155
mobilint::Buffer::data
int8_t * data
Definition type.h:154
mobilint::CacheInfo
Struct representing KV-cache information.
Definition type.h:497
mobilint::CoreId
Represents a unique identifier for an NPU core.
Definition type.h:123
mobilint::CoreId::operator<
bool operator<(const CoreId &rhs) const
Compares two CoreId objects for ordering.
Definition type.h:143
mobilint::CoreId::operator==
bool operator==(const CoreId &rhs) const
Checks if two CoreId objects are equal.
Definition type.h:133
mobilint::CoreId::cluster
Cluster cluster
Definition type.h:124
mobilint::Scale
Struct for scale values.
Definition type.h:98
mobilint::Scale::operator[]
float operator[](int i) const
Returns the scale value at the specified index.
Definition type.h:109
mobilint::Scale::scale
float scale
Definition type.h:100
mobilint::Scale::is_uniform
bool is_uniform
Definition type.h:101
mobilint::Scale::scale_list
std::vector< float > scale_list
Definition type.h:99
Generated by doxygen 1.14.0