model_variant_handle.h Source File

model_variant_handle.h Source File#

SDK qb Runtime Library: model_variant_handle.h Source File
SDK qb Runtime Library v1.4
MCS001-EN
 
#ifndef QBRUNTIME_MODEL_VARIANT_HANDLE_H_
#define QBRUNTIME_MODEL_VARIANT_HANDLE_H_
 
#include <stdint.h>
 
#include <vector>
 
#include "qbruntime/export.h"
#include "qbruntime/npu_data.h"
#include "qbruntime/status_code.h"
#include "qbruntime/type.h"
 
namespace mobilint {
 
class ModelImpl;
class QBRUNTIME_EXPORT ModelVariantHandle {
public:
    ModelVariantHandle(const ModelVariantHandle& other) = delete;
    ModelVariantHandle(ModelVariantHandle&& other) = delete;
    ModelVariantHandle& operator=(const ModelVariantHandle& rhs) = delete;
    ModelVariantHandle& operator=(ModelVariantHandle&& rhs) noexcept = delete;
    ~ModelVariantHandle();
    int getVariantIdx() const;
    const std::vector<std::vector<int64_t>>& getModelInputShape() const;
    const std::vector<std::vector<int64_t>>& getModelOutputShape() const;
    const std::vector<BufferInfo>& getInputBufferInfo() const;
    const std::vector<BufferInfo>& getOutputBufferInfo() const;
    std::vector<Scale> getInputScale() const;
    std::vector<Scale> getOutputScale() const;
    DataType getModelInputDataType() const;
    DataType getModelOutputDataType() const;
 
    // Acquire buffer
    std::vector<Buffer> acquireInputBuffer(
        const std::vector<std::vector<int>>& seqlens = {}) const;
    std::vector<Buffer> acquireOutputBuffer(
        const std::vector<std::vector<int>>& seqlens = {}) const;
    std::vector<std::vector<Buffer>> acquireInputBuffers(
        int batch_size, const std::vector<std::vector<int>>& seqlens = {}) const;
    std::vector<std::vector<Buffer>> acquireOutputBuffers(
        int batch_size, const std::vector<std::vector<int>>& seqlens = {}) const;
    NPUData acquireInputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
                                DataType dtype, StatusCode& sc) const;
    NPUData acquireOutputNPUData(const std::vector<int64_t>& shape, int idx, bool upload,
                                 DataType dtype, StatusCode& sc) const;
 
    // Deallocate acquired Input/Output buffer
    StatusCode releaseBuffer(std::vector<Buffer>& buffer) const;
    StatusCode releaseBuffers(std::vector<std::vector<Buffer>>& buffers) const;
 
    // Reposition single batch
    StatusCode repositionInputs(const std::vector<float*>& input,
                                std::vector<Buffer>& input_buf,
                                const std::vector<std::vector<int>>& seqlens = {}) const;
    StatusCode repositionOutputs(const std::vector<Buffer>& output_buf,
                                 std::vector<float*>& output,
                                 const std::vector<std::vector<int>>& seqlens = {}) const;
    StatusCode repositionOutputs(const std::vector<Buffer>& output_buf,
                                 std::vector<std::vector<float>>& output,
                                 const std::vector<std::vector<int>>& seqlens = {}) const;
    StatusCode repositionInputs(const std::vector<uint8_t*>& input,
                                std::vector<Buffer>& input_buf,
                                const std::vector<std::vector<int>>& seqlens = {}) const;
 
    // Reposition multiple batches
    StatusCode repositionInputs(const std::vector<float*>& input,
                                std::vector<std::vector<Buffer>>& input_buf,
                                const std::vector<std::vector<int>>& seqlens = {}) const;
    StatusCode repositionOutputs(const std::vector<std::vector<Buffer>>& output_buf,
                                 std::vector<float*>& output,
                                 const std::vector<std::vector<int>>& seqlens = {}) const;
    StatusCode repositionOutputs(const std::vector<std::vector<Buffer>>& output_buf,
                                 std::vector<std::vector<float>>& output,
                                 const std::vector<std::vector<int>>& seqlens = {}) const;
    StatusCode repositionInputs(const std::vector<uint8_t*>& input,
                                std::vector<std::vector<Buffer>>& input_buf,
                                const std::vector<std::vector<int>>& seqlens = {}) const;
 
private:
    ModelVariantHandle(int variant_idx, const ModelImpl& model_impl);
 
    const int mIdx;
    const ModelImpl& mModelImpl;
 
    friend class ModelImpl;
};
 
}  // namespace mobilint
 
#endif  // QBRUNTIME_MODEL_VARIANT_HANDLE_H_