Skip to content

Commit 46cd113

Browse files
zhupengyangzhupengyang
authored
[QualcommQnn] add tanh, unsqueeze, equal, slice, cast (#9296)
1 parent 1b7a0a2 commit 46cd113

9 files changed

Lines changed: 269 additions & 72 deletions

File tree

‎lite/backends/nnadapter/nnadapter/include/nnadapter/optimizer/convert_datalayout_nchw_to_nhwc.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ class NCHW2NHWCDataLayoutConverter {
3939
void ConvertBatchNormalization(core::Operation* operation);
4040
void ConvertCast(core::Operation* operation);
4141
void ConvertClip(core::Operation* operation);
42+
void ConvertComparisons(core::Operation* operation);
4243
void ConvertConv2DTranspose(core::Operation* operation);
4344
void ConvertCumSum(core::Operation* operation);
4445
void ConvertElementwise(core::Operation* operation);
@@ -59,12 +60,14 @@ class NCHW2NHWCDataLayoutConverter {
5960
void ConvertResizeNearest(core::Operation* operation);
6061
void ConvertResizeLinear(core::Operation* operation);
6162
void ConvertShape(core::Operation* operation);
63+
void ConvertSlice(core::Operation* operation);
6264
void ConvertSoftmax(core::Operation* operation);
6365
void ConvertSplit(core::Operation* operation);
6466
void ConvertSqueeze(core::Operation* operation);
6567
void ConvertStack(core::Operation* operation);
6668
void ConvertTranspose(core::Operation* operation);
6769
void ConvertMatMul(core::Operation* operation);
70+
void ConvertUnsqueeze(core::Operation* operation);
6871

6972
private:
7073
core::Model* model_{nullptr};

‎lite/backends/nnadapter/nnadapter/src/optimizer/convert_datalayout_nchw_to_nhwc.cc‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,81 @@ void NCHW2NHWCDataLayoutConverter::ConvertConv2DTranspose(
421421
SetOperationLayout(operation, 3);
422422
}
423423

424+
void NCHW2NHWCDataLayoutConverter::ConvertComparisons(
425+
core::Operation* operation) {
426+
auto& input_operands = operation->input_operands;
427+
auto& output_operands = operation->output_operands;
428+
auto input_count = input_operands.size();
429+
auto output_count = output_operands.size();
430+
NNADAPTER_CHECK_EQ(input_count, 2);
431+
NNADAPTER_CHECK_EQ(output_count, 1);
432+
auto output_operand = output_operands[0];
433+
auto output_dimensions_count = output_operand->type.dimensions.count;
434+
// Force to align the dimorder vector of all of input operands
435+
std::vector<int32_t> reference_permutation;
436+
core::Operand* reference_operand = nullptr;
437+
for (size_t i = 0; i < input_count; i++) {
438+
auto input_operand = input_operands[i];
439+
if (!IsConstantOperand(input_operand)) {
440+
auto input_permutation = GetPermutation(input_operand);
441+
if (input_permutation.size() > reference_permutation.size()) {
442+
reference_permutation = input_permutation;
443+
reference_operand = input_operand;
444+
}
445+
}
446+
}
447+
if (reference_permutation.empty()) {
448+
// All of input operands are constant
449+
SetPermutation(output_operand,
450+
IdentityPermutation(output_dimensions_count));
451+
} else {
452+
auto reference_dimensions_count = reference_operand->type.dimensions.count;
453+
for (size_t i = 0; i < input_count; i++) {
454+
auto input_operand = input_operands[i];
455+
auto input_dimensions_count = input_operand->type.dimensions.count;
456+
if (!IsConstantOperand(input_operand)) {
457+
auto input_permutation = GetPermutation(input_operand);
458+
auto transpose_input_permutation = MultiplyPermutation(
459+
InversePermutation(input_permutation), reference_permutation);
460+
if (!IsIdentityPermutation(transpose_input_permutation)) {
461+
auto transpose_input_operand = AppendTransposeOperation(
462+
model_, input_operand, transpose_input_permutation);
463+
UpdateOperationInputOperands(
464+
{operation}, input_operand, transpose_input_operand);
465+
SetPermutation(transpose_input_operand, reference_permutation);
466+
}
467+
} else {
468+
if (IsIdentityPermutation(reference_permutation)) {
469+
// Ignore
470+
} else if (input_dimensions_count == reference_permutation.size()) {
471+
TransposeOperand(input_operand, reference_permutation);
472+
} else {
473+
// Expand shape with 1
474+
std::vector<int32_t> origin_reference_dimensions(
475+
reference_dimensions_count);
476+
TransposeDimensions(reference_operand->type.dimensions.data,
477+
InversePermutation(reference_permutation),
478+
&origin_reference_dimensions[0]);
479+
std::vector<int32_t> expanded_input_dimensions;
480+
for (uint32_t j = 0, k = 0; j < reference_dimensions_count; j++) {
481+
if (origin_reference_dimensions[j] ==
482+
input_operand->type.dimensions.data[k] &&
483+
k < input_dimensions_count) {
484+
expanded_input_dimensions.push_back(
485+
input_operand->type.dimensions.data[k]);
486+
++k;
487+
} else {
488+
expanded_input_dimensions.push_back(1);
489+
}
490+
}
491+
}
492+
}
493+
}
494+
TransposeOperand(output_operand, reference_permutation);
495+
SetPermutation(output_operand, reference_permutation);
496+
}
497+
}
498+
424499
void NCHW2NHWCDataLayoutConverter::ConvertCumSum(core::Operation* operation) {
425500
auto& input_operands = operation->input_operands;
426501
auto& output_operands = operation->output_operands;
@@ -679,6 +754,26 @@ void NCHW2NHWCDataLayoutConverter::ConvertShape(core::Operation* operation) {
679754
SetPermutation(output_operand, IdentityPermutation(output_dimensions_count));
680755
}
681756

757+
void NCHW2NHWCDataLayoutConverter::ConvertSlice(core::Operation* operation) {
758+
auto& input_operands = operation->input_operands;
759+
auto& output_operands = operation->output_operands;
760+
auto input_count = input_operands.size();
761+
auto output_count = output_operands.size();
762+
NNADAPTER_CHECK_EQ(input_count, 5);
763+
NNADAPTER_CHECK_EQ(output_count, 1);
764+
// Recalculate the axis according to the dimorder vector of the input operand
765+
auto axes_operand = input_operands[1];
766+
int axes_count = axes_operand->length / sizeof(int32_t);
767+
int* axes = reinterpret_cast<int32_t*>(axes_operand->buffer);
768+
auto input_permutation = GetPermutation(input_operands[0]);
769+
for (int i = 0; i < axes_count; i++) {
770+
axes[i] = TransposeAxis(axes[i], input_permutation);
771+
}
772+
auto output_operand = output_operands[0];
773+
TransposeOperand(output_operand, input_permutation);
774+
SetPermutation(output_operand, input_permutation);
775+
}
776+
682777
void NCHW2NHWCDataLayoutConverter::ConvertFill(core::Operation* operation) {
683778
auto& input_operands = operation->input_operands;
684779
auto& output_operands = operation->output_operands;
@@ -937,6 +1032,32 @@ void NCHW2NHWCDataLayoutConverter::ConvertTranspose(
9371032
SetPermutation(output_operand, input_permutation);
9381033
}
9391034

1035+
void NCHW2NHWCDataLayoutConverter::ConvertUnsqueeze(
1036+
core::Operation* operation) {
1037+
auto& input_operands = operation->input_operands;
1038+
auto& output_operands = operation->output_operands;
1039+
auto input_count = input_operands.size();
1040+
auto output_count = output_operands.size();
1041+
NNADAPTER_CHECK_EQ(input_count, 2);
1042+
NNADAPTER_CHECK_EQ(output_count, 1);
1043+
auto input_operand = input_operands[0];
1044+
int input_dimensions_count = input_operand->type.dimensions.count;
1045+
auto output_operand = output_operands[0];
1046+
auto output_dimensions_count = output_operand->type.dimensions.count;
1047+
// Force to restore the dimorder vector of the input operand
1048+
auto input_permutation = GetPermutation(input_operand);
1049+
auto transpose_input_permutation = InversePermutation(input_permutation);
1050+
if (!IsIdentityPermutation(transpose_input_permutation)) {
1051+
auto transpose_input_operand = AppendTransposeOperation(
1052+
model_, input_operand, transpose_input_permutation);
1053+
UpdateOperationInputOperands(
1054+
{operation}, input_operand, transpose_input_operand);
1055+
SetPermutation(transpose_input_operand,
1056+
IdentityPermutation(input_dimensions_count));
1057+
}
1058+
SetPermutation(output_operand, IdentityPermutation(output_dimensions_count));
1059+
}
1060+
9401061
void NCHW2NHWCDataLayoutConverter::Apply(core::Model* model) {
9411062
model_ = model;
9421063
// Initialize the permutation of model input operands
@@ -987,6 +1108,14 @@ void NCHW2NHWCDataLayoutConverter::Apply(core::Model* model) {
9871108
case NNADAPTER_CUM_SUM:
9881109
ConvertCumSum(operation);
9891110
break;
1111+
case NNADAPTER_EQUAL:
1112+
case NNADAPTER_GREATER:
1113+
case NNADAPTER_GREATER_EQUAL:
1114+
case NNADAPTER_LESS:
1115+
case NNADAPTER_LESS_EQUAL:
1116+
case NNADAPTER_NOT_EQUAL:
1117+
ConvertComparisons(operation);
1118+
break;
9901119
case NNADAPTER_FILL:
9911120
ConvertFill(operation);
9921121
break;
@@ -1045,6 +1174,9 @@ void NCHW2NHWCDataLayoutConverter::Apply(core::Model* model) {
10451174
case NNADAPTER_SHAPE:
10461175
ConvertShape(operation);
10471176
break;
1177+
case NNADAPTER_SLICE:
1178+
ConvertSlice(operation);
1179+
break;
10481180
case NNADAPTER_SOFTMAX:
10491181
ConvertSoftmax(operation);
10501182
break;
@@ -1060,6 +1192,9 @@ void NCHW2NHWCDataLayoutConverter::Apply(core::Model* model) {
10601192
case NNADAPTER_TRANSPOSE:
10611193
ConvertTranspose(operation);
10621194
break;
1195+
case NNADAPTER_UNSQUEEZE:
1196+
ConvertUnsqueeze(operation);
1197+
break;
10631198
default:
10641199
NNADAPTER_LOG(FATAL)
10651200
<< "Missing the processing of "

‎lite/backends/nnadapter/nnadapter/src/optimizer/convert_quantization_symm_to_asymm.cc‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -82,12 +82,18 @@ NNADAPTER_EXPORT void ConvertQuantizationSymmToAsymm(core::Model* model) {
8282
switch (operation->type) {
8383
case NNADAPTER_ADD:
8484
case NNADAPTER_DIV:
85+
case NNADAPTER_EQUAL:
8586
case NNADAPTER_FULLY_CONNECTED:
8687
case NNADAPTER_GATHER:
88+
case NNADAPTER_GREATER:
89+
case NNADAPTER_GREATER_EQUAL:
90+
case NNADAPTER_LESS:
91+
case NNADAPTER_LESS_EQUAL:
8792
case NNADAPTER_MAT_MUL:
8893
case NNADAPTER_MAX:
8994
case NNADAPTER_MIN:
9095
case NNADAPTER_MUL:
96+
case NNADAPTER_NOT_EQUAL:
9197
case NNADAPTER_POW:
9298
case NNADAPTER_SUB: {
9399
ConvertOperandSymmToAsymm(input_operands[0], 128);
@@ -96,25 +102,27 @@ NNADAPTER_EXPORT void ConvertQuantizationSymmToAsymm(core::Model* model) {
96102
} break;
97103
case NNADAPTER_AVERAGE_POOL_2D:
98104
case NNADAPTER_BATCH_NORMALIZATION:
105+
case NNADAPTER_CAST:
106+
case NNADAPTER_CHANNEL_SHUFFLE:
107+
case NNADAPTER_CLIP:
99108
case NNADAPTER_CUM_SUM:
109+
case NNADAPTER_FILL_LIKE:
110+
case NNADAPTER_FLATTEN:
111+
case NNADAPTER_HARD_SIGMOID:
112+
case NNADAPTER_HARD_SWISH:
113+
case NNADAPTER_LEAKY_RELU:
100114
case NNADAPTER_MAX_POOL_2D:
101115
case NNADAPTER_RELU:
102116
case NNADAPTER_RELU6:
103117
case NNADAPTER_RESHAPE:
104118
case NNADAPTER_RESIZE_NEAREST:
105119
case NNADAPTER_RESIZE_LINEAR:
120+
case NNADAPTER_SLICE:
121+
case NNADAPTER_SQUEEZE:
106122
case NNADAPTER_SWISH:
107123
case NNADAPTER_TANH:
108-
case NNADAPTER_FLATTEN:
109124
case NNADAPTER_TRANSPOSE:
110-
case NNADAPTER_HARD_SIGMOID:
111-
case NNADAPTER_HARD_SWISH:
112-
case NNADAPTER_LEAKY_RELU:
113-
case NNADAPTER_SQUEEZE:
114-
case NNADAPTER_CLIP:
115-
case NNADAPTER_CHANNEL_SHUFFLE:
116-
case NNADAPTER_SLICE:
117-
case NNADAPTER_FILL_LIKE: {
125+
case NNADAPTER_UNSQUEEZE: {
118126
ConvertOperandSymmToAsymm(input_operands[0], 128);
119127
ConvertOperandSymmToAsymm(output_operands[0], 128);
120128
PropagateAsymmZeroPoint(input_operands[0], output_operands[0]);

‎lite/kernels/nnadapter/converter/all.h‎

Lines changed: 25 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ REGISTER_CONVERTER(bmm, ConvertBmm, "intel_openvino");
2626
REGISTER_CONVERTER(cast,
2727
ConvertCast,
2828
"huawei_ascend_npu,cambricon_mlu,huawei_kirin_npu,nvidia_"
29-
"tensorrt,intel_openvino");
29+
"tensorrt,intel_openvino,qualcomm_qnn");
3030
REGISTER_CONVERTER(clip,
3131
ConvertClip,
3232
"huawei_ascend_npu,cambricon_mlu,verisilicon_timvx,huawei_"
@@ -92,10 +92,10 @@ REGISTER_CONVERTER(reshape2,
9292
"npu,amlogic_npu,imagination_nna,verisilicon_timvx,"
9393
"kunlunxin_xtcl,cambricon_mlu,android_nnapi,nvidia_tensorrt,"
9494
"intel_openvino,qualcomm_qnn,google_xnnpack");
95-
REGISTER_CONVERTER(
96-
unsqueeze,
97-
ConvertUnsqueeze,
98-
"huawei_ascend_npu,cambricon_mlu,nvidia_tensorrt,intel_openvino");
95+
REGISTER_CONVERTER(unsqueeze,
96+
ConvertUnsqueeze,
97+
"huawei_ascend_npu,cambricon_mlu,nvidia_tensorrt,intel_"
98+
"openvino,qualcomm_qnn");
9999
REGISTER_CONVERTER(
100100
unsqueeze2,
101101
ConvertUnsqueeze,
@@ -215,7 +215,7 @@ REGISTER_CONVERTER(tanh,
215215
"rockchip_npu,mediatek_apu,huawei_kirin_npu,huawei_ascend_"
216216
"npu,amlogic_npu,cambricon_mlu,verisilicon_timvx,kunlunxin_"
217217
"xtcl,android_nnapi,intel_openvino,nvidia_tensorrt,"
218-
"eeasytech_npu");
218+
"eeasytech_npu,qualcomm_qnn");
219219
REGISTER_CONVERTER(abs,
220220
ConvertUnaryActivations,
221221
"huawei_ascend_npu,huawei_kirin_npu,intel_openvino");
@@ -262,29 +262,30 @@ REGISTER_CONVERTER(
262262
REGISTER_CONVERTER(equal,
263263
ConvertComparisons,
264264
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu,intel_"
265-
"openvino,nvidia_tensorrt");
265+
"openvino,nvidia_tensorrt,qualcomm_qnn");
266266
REGISTER_CONVERTER(expand_v2,
267267
ConvertExpandV2,
268268
"huawei_ascend_npu,cambricon_mlu,intel_openvino");
269-
REGISTER_CONVERTER(not_equal,
270-
ConvertComparisons,
271-
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu");
272-
REGISTER_CONVERTER(greater_than,
273-
ConvertComparisons,
274-
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu");
275269
REGISTER_CONVERTER(
276-
greater_equal,
270+
not_equal,
277271
ConvertComparisons,
278-
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu,intel_openvino");
279-
REGISTER_CONVERTER(less_than,
280-
ConvertComparisons,
281-
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu");
282-
REGISTER_CONVERTER(less_equal,
283-
ConvertComparisons,
284-
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu");
285-
REGISTER_CONVERTER(less_than,
272+
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu,qualcomm_qnn");
273+
REGISTER_CONVERTER(
274+
greater_than,
275+
ConvertComparisons,
276+
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu,qualcomm_qnn");
277+
REGISTER_CONVERTER(greater_equal,
286278
ConvertComparisons,
287-
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu");
279+
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu,intel_"
280+
"openvino,qualcomm_qnn");
281+
REGISTER_CONVERTER(
282+
less_than,
283+
ConvertComparisons,
284+
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu,qualcomm_qnn");
285+
REGISTER_CONVERTER(
286+
less_equal,
287+
ConvertComparisons,
288+
"huawei_ascend_npu,huawei_kirin_npu,cambricon_mlu,qualcomm_qnn");
288289
REGISTER_CONVERTER(
289290
reduce_mean,
290291
ConvertReduce,
@@ -321,7 +322,7 @@ REGISTER_CONVERTER(
321322
REGISTER_CONVERTER(slice,
322323
ConvertSlice,
323324
"huawei_ascend_npu,verisilicon_timvx,cambricon_mlu,nvidia_"
324-
"tensorrt,intel_openvino");
325+
"tensorrt,intel_openvino,qualcomm_qnn");
325326
REGISTER_CONVERTER(strided_slice,
326327
ConvertStridedSlice,
327328
"huawei_ascend_npu,huawei_kirin_npu,nvidia_tensorrt");

‎lite/tests/kernels/activation_compute_test.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,8 @@ TEST(Activation_tanh, precision) {
697697
abs_error = 2e-5;
698698
#elif defined(NNADAPTER_WITH_NVIDIA_TENSORRT)
699699
abs_error = 2e-5;
700+
#elif defined(NNADAPTER_WITH_QUALCOMM_QNN)
701+
abs_error = 2e-5;
700702
#elif defined(NNADAPTER_WITH_INTEL_OPENVINO)
701703
abs_error = 1e-5;
702704
for (auto& dims : test_dims) {

0 commit comments

Comments
 (0)