model_variant_handle.py Source File

model_variant_handle.py Source File#

SDK qb Runtime Library: model_variant_handle.py Source File
SDK qb Runtime Library v1.1
MCS001-
model_variant_handle.py
Go to the documentation of this file.
1
4
5from typing import List, Tuple
6
7import qbruntime.qbruntime as _cQbRuntime
8from .type import *
9
10_Shape = Tuple[int, ...]
11
12
15
16
17class ModelVariantHandle:
18 """
19 @brief Handle to a specific variant of a loaded model.
20
21 This class provides access to variant-specific information such as input/output
22 shapes, buffer information, and quantization scales. It also offers APIs for
23 managing inference buffers, consistent with the interface of the `Model` class.
24
25 Objects of this class are obtained via `Model.get_model_variant_handle()`.
26 """
27
28 def __init__(self, _model_variant_handle: _cQbRuntime.ModelVariantHandle):
29 self._model_variant_handle = _model_variant_handle
30 self._output_shape = self.get_model_output_shape()
31
32 @classmethod
33 def from_cpp(cls, _model_variant_handle: _cQbRuntime.ModelVariantHandle):
34 return cls(_model_variant_handle)
35
36 def get_variant_idx(self) -> int:
37 """
38 @brief Returns the index of this model variant.
39
40 @return Index of the model variant.
41 """
42 return self._model_variant_handle.get_variant_idx()
43
44 def get_model_input_shape(self) -> List[_Shape]:
45 """
46 @brief Returns the input shape for this model variant.
47
48 @return model variant's input shape.
49 """
50 return self._model_variant_handle.get_model_input_shape()
51
52 def get_model_output_shape(self) -> List[_Shape]:
53 """
54 @brief Returns the output shape for this model variant.
55
56 @return model variant's output shape.
57 """
58 return self._model_variant_handle.get_model_output_shape()
59
60 def get_input_buffer_info(self) -> List[BufferInfo]:
61 """
62 @brief Returns the input buffer information for this variant.
63
64 @return A list of input buffer information.
65 """
66 return [
67 BufferInfo.from_cpp(bi)
68 for bi in self._model_variant_handle.get_input_buffer_info()
69 ]
70
71 def get_output_buffer_info(self) -> List[BufferInfo]:
72 """
73 @brief Returns the output buffer information for this variant.
74
75 @return A list of output buffer information.
76 """
77 return [
78 BufferInfo.from_cpp(bi)
79 for bi in self._model_variant_handle.get_output_buffer_info()
80 ]
81
82 def get_input_scale(self) -> List[Scale]:
83 """
84 @brief Returns the input quantization scale(s) for this variant.
85
86 @return A list of input scales.
87 """
88 return [Scale.from_cpp(s) for s in self._model_variant_handle.get_input_scale()]
89
90 def get_output_scale(self) -> List[Scale]:
91 """
92 @brief Returns the output quantization scale(s) for this variant.
93
94 @return A list of output scales.
95 """
96 return [
97 Scale.from_cpp(s) for s in self._model_variant_handle.get_output_scale()
98 ]
99
100 def get_model_input_data_type(self) -> DataType:
101 """
102 @brief Returns the input buffer information for the model.
103
104 @return A list of input buffer information.
105 """
106 return DataType(self._model_variant_handle.get_model_input_data_type())
107
108 def get_model_output_data_type(self) -> DataType:
109 """
110 @brief Returns the output buffer information of the model.
111
112 @return A list of output buffer information.
113 """
114 return DataType(self._model_variant_handle.get_model_output_data_type())
115
116 def acquire_input_buffer(self, seqlens: List[List[int]] = []) -> List[Buffer]:
117 """
118 @brief Buffer Management API
119
120 Acquires list of `Buffer` for input.
121 These API is required when calling `Model.infer_buffer()`.
122
123 @note These APIs are intended for advanced use and follow the same buffer
124 management interface as the `Model` class.
125 """
126 return [
127 Buffer(b) for b in self._model_variant_handle.acquire_input_buffer(seqlens)
128 ]
129
130 def acquire_output_buffer(self, seqlens: List[List[int]] = []) -> List[Buffer]:
131 """
132 @brief Buffer Management API
133
134 Acquires list of `Buffer` for output.
135 These API is required when calling `Model.infer_buffer()`.
136
137 @note These APIs are intended for advanced use and follow the same buffer
138 management interface as the `Model` class.
139 """
140 return [
141 Buffer(b) for b in self._model_variant_handle.acquire_output_buffer(seqlens)
142 ]
143
144 def release_buffer(self, buffer: List[Buffer]) -> None:
145 """
146 @brief Buffer Management API
147
148 Deallocate acquired Input/Output buffer
149
150 @note These APIs are intended for advanced use and follow the same buffer
151 management interface as the `Model` class.
152 """
153 self._model_variant_handle.release_buffer([b._buffer for b in buffer])
154
155 def reposition_inputs(
156 self,
157 inputs: List[np.ndarray],
158 input_bufs: List[Buffer],
159 seqlens: List[List[int]] = [],
160 ) -> None:
161 """Reposition input"""
162 inputs = [np.ascontiguousarray(i) for i in inputs]
163 self._model_variant_handle.reposition_inputs(
164 inputs, [buf._buffer for buf in input_bufs], seqlens
165 )
166
167 def reposition_outputs(
168 self,
169 output_bufs: List[Buffer],
170 outputs: List[np.ndarray],
171 seqlens: List[List[int]] = [],
172 ) -> None:
173 """Reposition output"""
174 if len(outputs) != len(self._output_shape):
175 outputs.clear()
176 for shape in self._output_shape:
177 outputs.append(np.empty(shape=shape, dtype=np.float32))
178 else:
179 for oi in range(len(outputs)):
180 outputs[oi] = np.ascontiguousarray(outputs[oi])
181 self._model_variant_handle.reposition_outputs(
182 [buf._buffer for buf in output_bufs], outputs, seqlens
183 )
184
185
186
qbruntime.model_variant_handle.ModelVariantHandle
Handle to a specific variant of a loaded model.
Definition model_variant_handle.py:17
qbruntime.model_variant_handle.ModelVariantHandle.release_buffer
None release_buffer(self, List[Buffer] buffer)
Buffer Management API.
Definition model_variant_handle.py:144
qbruntime.model_variant_handle.ModelVariantHandle.get_output_buffer_info
List[BufferInfo] get_output_buffer_info(self)
Returns the output buffer information for this variant.
Definition model_variant_handle.py:71
qbruntime.model_variant_handle.ModelVariantHandle._model_variant_handle
_model_variant_handle
Definition model_variant_handle.py:29
qbruntime.model_variant_handle.ModelVariantHandle.reposition_outputs
None reposition_outputs(self, List[Buffer] output_bufs, List[np.ndarray] outputs, List[List[int]] seqlens=[])
Reposition output.
Definition model_variant_handle.py:172
qbruntime.model_variant_handle.ModelVariantHandle.get_model_input_data_type
DataType get_model_input_data_type(self)
Returns the input buffer information for the model.
Definition model_variant_handle.py:100
qbruntime.model_variant_handle.ModelVariantHandle.get_model_input_shape
List[_Shape] get_model_input_shape(self)
Returns the input shape for this model variant.
Definition model_variant_handle.py:44
qbruntime.model_variant_handle.ModelVariantHandle.get_input_scale
List[Scale] get_input_scale(self)
Returns the input quantization scale(s) for this variant.
Definition model_variant_handle.py:82
qbruntime.model_variant_handle.ModelVariantHandle._output_shape
List[_Shape] _output_shape
Definition model_variant_handle.py:30
qbruntime.model_variant_handle.ModelVariantHandle.get_variant_idx
int get_variant_idx(self)
Returns the index of this model variant.
Definition model_variant_handle.py:36
qbruntime.model_variant_handle.ModelVariantHandle.get_model_output_shape
List[_Shape] get_model_output_shape(self)
Returns the output shape for this model variant.
Definition model_variant_handle.py:52
qbruntime.model_variant_handle.ModelVariantHandle.get_model_output_data_type
DataType get_model_output_data_type(self)
Returns the output buffer information of the model.
Definition model_variant_handle.py:108
qbruntime.model_variant_handle.ModelVariantHandle.get_output_scale
List[Scale] get_output_scale(self)
Returns the output quantization scale(s) for this variant.
Definition model_variant_handle.py:90
qbruntime.model_variant_handle.ModelVariantHandle.acquire_input_buffer
List[Buffer] acquire_input_buffer(self, List[List[int]] seqlens=[])
Buffer Management API.
Definition model_variant_handle.py:116
qbruntime.model_variant_handle.ModelVariantHandle.reposition_inputs
None reposition_inputs(self, List[np.ndarray] inputs, List[Buffer] input_bufs, List[List[int]] seqlens=[])
Reposition input.
Definition model_variant_handle.py:160
qbruntime.model_variant_handle.ModelVariantHandle.get_input_buffer_info
List[BufferInfo] get_input_buffer_info(self)
Returns the input buffer information for this variant.
Definition model_variant_handle.py:60
qbruntime.model_variant_handle.ModelVariantHandle.acquire_output_buffer
List[Buffer] acquire_output_buffer(self, List[List[int]] seqlens=[])
Buffer Management API.
Definition model_variant_handle.py:130
qbruntime.type.Buffer
A simple byte-sized buffer.
Definition type.py:171
Generated by doxygen 1.14.0