Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ class CustomPluginCreater : public OpConverter {
CHECK(creator);

// set attrs
std::vector<nvinfer1::PluginField> plugindatas;
std::vector<nvinfer1::PluginField> plugin_datas;
auto &op_attrs_names = OpMetaInfoHelper::GetAttrs(op_info);
auto &attrs = op_desc.GetAttrMap();

Expand All @@ -74,73 +74,73 @@ class CustomPluginCreater : public OpConverter {
for (auto &attr_name_and_type : op_attrs_names) {
auto attr_name =
attr_name_and_type.substr(0, attr_name_and_type.find_first_of(":"));
nvinfer1::PluginField plugindata;
nvinfer1::PluginField plugin_data;

// NOTE: to avoid string rewrite by iterator, deep copy here
std::vector<char> plugin_attr_name(attr_name.length() + 1, 0);
snprintf(plugin_attr_name.data(),
attr_name.length() + 1,
"%s",
attr_name.c_str());
plugindata.name = plugin_attr_name.data();
plugin_data.name = plugin_attr_name.data();

if (op_desc.GetAttrType(attr_name) == framework::proto::AttrType::INT) {
int_attrs.push_back(PADDLE_GET_CONST(int, attrs.at(attr_name)));
plugindata.data = &int_attrs.back();
plugindata.type = nvinfer1::PluginFieldType::kINT32;
plugindata.length = 1;
plugin_data.data = &int_attrs.back();
plugin_data.type = nvinfer1::PluginFieldType::kINT32;
plugin_data.length = 1;
} else if (op_desc.GetAttrType(attr_name) ==
framework::proto::AttrType::FLOAT) {
float_attrs.push_back(PADDLE_GET_CONST(float, attrs.at(attr_name)));
plugindata.data = &float_attrs.back();
plugindata.type = nvinfer1::PluginFieldType::kFLOAT32;
plugindata.length = 1;
plugin_data.data = &float_attrs.back();
plugin_data.type = nvinfer1::PluginFieldType::kFLOAT32;
plugin_data.length = 1;
} else if (op_desc.GetAttrType(attr_name) ==
framework::proto::AttrType::BOOLEAN) {
int_attrs.push_back(PADDLE_GET_CONST(bool, attrs.at(attr_name)));
plugindata.data = &int_attrs.back();
plugindata.type = nvinfer1::PluginFieldType::kINT32;
plugindata.length = 1;
plugin_data.data = &int_attrs.back();
plugin_data.type = nvinfer1::PluginFieldType::kINT32;
plugin_data.length = 1;
} else if (op_desc.GetAttrType(attr_name) ==
framework::proto::AttrType::STRING) {
string_attrs.push_back(
PADDLE_GET_CONST(std::string, attrs.at(attr_name)));
plugindata.data = string_attrs.back().data();
plugindata.type = nvinfer1::PluginFieldType::kCHAR;
plugindata.length =
plugin_data.data = string_attrs.back().data();
plugin_data.type = nvinfer1::PluginFieldType::kCHAR;
plugin_data.length =
string_attrs.back().size() + 1; // string ends with ‘\0’
} else if (op_desc.GetAttrType(attr_name) ==
framework::proto::AttrType::INTS) {
ints_attrs.push_back(
PADDLE_GET_CONST(std::vector<int>, attrs.at(attr_name)));
plugindata.data = ints_attrs.back().data();
plugindata.type = nvinfer1::PluginFieldType::kINT32;
plugindata.length = ints_attrs.back().size();
plugin_data.data = ints_attrs.back().data();
plugin_data.type = nvinfer1::PluginFieldType::kINT32;
plugin_data.length = ints_attrs.back().size();
} else if (op_desc.GetAttrType(attr_name) ==
framework::proto::AttrType::FLOATS) {
floats_attrs.push_back(
PADDLE_GET_CONST(std::vector<float>, attrs.at(attr_name)));
plugindata.data = floats_attrs.back().data();
plugindata.type = nvinfer1::PluginFieldType::kFLOAT32;
plugindata.length = floats_attrs.back().size();
plugin_data.data = floats_attrs.back().data();
plugin_data.type = nvinfer1::PluginFieldType::kFLOAT32;
plugin_data.length = floats_attrs.back().size();
} else if (op_desc.GetAttrType(attr_name) ==
framework::proto::AttrType::BOOLEANS) {
auto bools_attr =
PADDLE_GET_CONST(std::vector<bool>, attrs.at(attr_name));
std::vector<int> convert_to_ints_attr;
for (bool i : bools_attr) convert_to_ints_attr.push_back(i);
ints_attrs.push_back(convert_to_ints_attr);
plugindata.data = ints_attrs.back().data();
plugindata.type = nvinfer1::PluginFieldType::kINT32;
plugindata.length = ints_attrs.back().size();
plugin_data.data = ints_attrs.back().data();
plugin_data.type = nvinfer1::PluginFieldType::kINT32;
plugin_data.length = ints_attrs.back().size();
} else {
CHECK(false) << "UNKNOWN PluginFieldType.";
}
plugindatas.push_back(plugindata);
plugin_datas.push_back(plugin_data);
}

nvinfer1::PluginFieldCollection plugin_fc{(int32_t)plugindatas.size(),
plugindatas.data()};
nvinfer1::PluginFieldCollection plugin_fc{(int32_t)plugin_datas.size(),
plugin_datas.data()};

auto *plugin = creator->createPlugin(op_desc.Type().c_str(), &plugin_fc);
CHECK(plugin);
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/inference/tensorrt/convert/layer_norm_op.cc
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ class LayerNormOpConverter : public OpConverter {
#endif
#if IS_TRT_VERSION_LT(8600)
// For dynamic shape & trt<8.6,
// the shape of mean and variance will be determine in configuPlugin.
// the shape of mean and variance will be determine in configurePlugin.
auto* X = engine_->GetITensor(op_desc.Input("X").front());
auto* Bias_v = scope.FindVar(op_desc.Input("Bias").front());
auto* Scale_v = scope.FindVar(op_desc.Input("Scale").front());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ class LayerNormShiftPartitionOpConverter : public OpConverter {
PADDLE_ENFORCE_EQ(bias_weight.get().count,
scale_weight.get().count,
platform::errors::InvalidArgument(
"The num between bias_weight and cale_weight should "
"The num between bias_weight and scale_weight should "
"be equal. (%d vs %d)",
bias_weight.get().count,
scale_weight.get().count));
Expand Down
6 changes: 3 additions & 3 deletions paddle/fluid/inference/tensorrt/convert/op_converter.h
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class OpConverter {
1UL,
platform::errors::InvalidArgument(
"The input op's Input(\"Y\")."
"size() should equal to 1, but reveceid "
"size() should equal to 1, but received "
"Input(\"Y\").size() = %u.",
op_desc.Input("Y").size()));
int op_type_len = op_desc.Type().size();
Expand Down Expand Up @@ -179,7 +179,7 @@ class OpConverter {
(*it)(op, scope, test_mode);

size_t output_num = op_desc.OutputNames().size();
// only one out settensordynamicRange
// only one out SetTensorDynamicRange
if (op_desc.HasAttr("out_threshold")) {
float out_scale =
PADDLE_GET_CONST(float, op_desc.GetAttr("out_threshold"));
Expand All @@ -202,7 +202,7 @@ class OpConverter {
VLOG(1) << "Set out scale = " << out_scale << " for tensor "
<< output_name << ".";
}
// outs settensordynamicRange
// outs SetTensorDynamicRange
for (size_t i = 0; i < output_num; ++i) {
if (op_desc.HasAttr("out_" + std::to_string(i) + "_threshold")) {
float out_scale = PADDLE_GET_CONST(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ class PrelnEmbEltwiseLayerNormOpConverter : public OpConverter {
slice_stride_dims); // unuseful slice_start_dims
slice_layer->setInput(1, *start_tensor);
slice_layer->setInput(2, *size_tensor);
slice_layer->setName(("Embeltwise_slice_layer (Output: slice_max_seqlen " +
slice_layer->setName(("EmbEltwise_slice_layer (Output: slice_max_seqlen " +
op_desc.Output("Out")[0] + ")")
.c_str());
engine_->SetTensorDynamicRange(slice_layer->getOutput(0), 1.0f);
Expand All @@ -114,7 +114,7 @@ class PrelnEmbEltwiseLayerNormOpConverter : public OpConverter {
shape_dim.nbDims = 1;
shape_dim.d[0] = -1;
reshape_layer->setReshapeDimensions(shape_dim);
reshape_layer->setName(("Embeltwise_reshape_layer (Output: max_seqlen " +
reshape_layer->setName(("EmbEltwise_reshape_layer (Output: max_seqlen " +
op_desc.Output("Out")[0] + ")")
.c_str());
engine_->SetTensorDynamicRange(reshape_layer->getOutput(0), 1.0f);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class QuantizeLinearOpConverter : public OpConverter {
// Create constant layer for scale
PADDLE_ENFORCE_NOT_NULL(
scale_var,
platform::errors::NotFound("Can not find %s presistale var in scope.",
platform::errors::NotFound("Can not find %s presistable var in scope.",
op_desc.Input("Scale")[0]));
auto* scale_t = scale_var->GetMutable<phi::DenseTensor>();
int n_scale = scale_t->numel();
Expand Down
6 changes: 3 additions & 3 deletions paddle/fluid/inference/tensorrt/convert/range_op.cc
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,15 @@ class RangeOpConverter : public OpConverter {
auto output_name = op_desc.Output("Out")[0];

auto zero_tensor = Add1DConstantLayer(0, output_name + "_zero_tensor_");
auto fquotient_tensor = FloorDiv(Sub(start, end), step);
auto f_quotient_tensor = FloorDiv(Sub(start, end), step);
if (start->getType() == nvinfer1::DataType::kFLOAT) {
auto* cast_int32_layer =
TRT_ENGINE_ADD_LAYER(engine_, Identity, *fquotient_tensor);
TRT_ENGINE_ADD_LAYER(engine_, Identity, *f_quotient_tensor);
cast_int32_layer->setOutputType(0, nvinfer1::DataType::kINT32);
cast_int32_layer->getOutput(0)->setType(nvinfer1::DataType::kINT32);
quotient_tensor = cast_int32_layer->getOutput(0);
} else {
quotient_tensor = fquotient_tensor;
quotient_tensor = f_quotient_tensor;
}
auto number_tensor = Max(Sub(zero_tensor, quotient_tensor), zero_tensor);
auto* start1 = engine_->GetITensor(op_desc.Input("Start")[0]);
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/inference/tensorrt/convert/reshape_op.cc
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ class ReshapeOpConverter : public OpConverter {
layer->getOutput(0)->getDimensions().nbDims,
0,
platform::errors::InvalidArgument(
"Errors occures in Paddle-TRT reshape2 op, try to use C++ Api "
"Errors occurs in Paddle-TRT reshape2 op, try to use C++ Api "
"config.Exp_DisableTensorRtOPs({\"reshape2\"})\n; or Python Api "
"config.exp_disable_tensorrt_ops([\"reshape2\"]) to forbid "
"reshape2 op into "
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/inference/tensorrt/convert/set_value_op.cc
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ class SetValueConverter : public OpConverter {
platform::errors::InvalidArgument(
"ValueTensor‘s rank not equal to Input's rank, "
"you should try use C++ API "
"config.exp_disable_tensorrt_ops({\"%s\"}) to forbind this op "
"config.exp_disable_tensorrt_ops({\"%s\"}) to forbid this op "
"enter into TRT, "
"please find the %s's real name from .pdmodel or shape.txt",
output_name,
Expand Down
24 changes: 14 additions & 10 deletions paddle/fluid/inference/tensorrt/convert/skip_layernorm.cc
Original file line number Diff line number Diff line change
Expand Up @@ -67,35 +67,39 @@ class SkipLayerNormOpConverter : public OpConverter {

if ((x_rank == 2 && y_rank == 4) || (y_rank == 2 && x_rank == 4)) {
if (x_rank == 2 && y_rank == 4) {
auto* reshape_before_skiplayn =
auto* reshape_before_skip_layer_n =
TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input1);
std::vector<nvinfer1::ITensor*> reshape_before_tensor;
reshape_before_tensor.push_back(GetEleTensorOfShape(Shape(input1), 0));
reshape_before_tensor.push_back(GetEleTensorOfShape(Shape(input1), 1));
reshape_before_tensor.push_back(Add1DConstantLayer(1));
reshape_before_tensor.push_back(Add1DConstantLayer(1));
reshape_before_skiplayn->setInput(1, *Concat(reshape_before_tensor));
reshape_before_skiplayn->setName(
("reshape_before_skiplayn(Output: " + output_name + ")").c_str());
input1 = reshape_before_skiplayn->getOutput(0);
reshape_before_skip_layer_n->setInput(1,
*Concat(reshape_before_tensor));
reshape_before_skip_layer_n->setName(
("reshape_before_skip_layer_n(Output: " + output_name + ")")
.c_str());
input1 = reshape_before_skip_layer_n->getOutput(0);

if (enable_int8) {
if (op_desc.HasAttr("X")) {
engine_->SetTensorDynamicRange(input1, x_scale);
}
}
} else {
auto* reshape_before_skiplayn =
auto* reshape_before_skip_layer_n =
TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input2);
std::vector<nvinfer1::ITensor*> reshape_before_tensor;
reshape_before_tensor.push_back(GetEleTensorOfShape(Shape(input2), 0));
reshape_before_tensor.push_back(GetEleTensorOfShape(Shape(input2), 1));
reshape_before_tensor.push_back(Add1DConstantLayer(1));
reshape_before_tensor.push_back(Add1DConstantLayer(1));
reshape_before_skiplayn->setInput(1, *Concat(reshape_before_tensor));
reshape_before_skiplayn->setName(
("reshape_before_skiplayn(Output: " + output_name + ")").c_str());
input2 = reshape_before_skiplayn->getOutput(0);
reshape_before_skip_layer_n->setInput(1,
*Concat(reshape_before_tensor));
reshape_before_skip_layer_n->setName(
("reshape_before_skip_layer_n(Output: " + output_name + ")")
.c_str());
input2 = reshape_before_skip_layer_n->getOutput(0);

if (enable_int8) {
if (op_desc.HasAttr("Y")) {
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/inference/tensorrt/convert/slice_op.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class SliceOpConverter : public OpConverter {
void operator()(const framework::proto::OpDesc& op,
const framework::Scope& scope,
bool test_mode) override {
// This OP is implemented by trt dynamic shpae plugin.
// This OP is implemented by trt dynamic shape plugin.
// Dynamic shape plugin requires TRT version greater than 6.0.
VLOG(4) << "convert slice op to tensorrt layer";
framework::OpDesc op_desc(op, nullptr);
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/inference/tensorrt/convert/softmax_op.cc
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ class SoftMaxOpConverter : public OpConverter {
uint32_t axes = std::max(0, input_dims - 3);
// TODO(cryoco): Poor workaround. Fix padded dims problem when TRT layers
// support Nd.
// Tips: Dynammic shape alreay fixes.
// Tips: Dynamic shape already fixes.
int padded_dims = 0;
int explicit_batch = 0;
if (engine_->with_dynamic_shape()) explicit_batch = 1;
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/inference/tensorrt/convert/sparse_fc_op.cc
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ class SparseFcOpConverter : public OpConverter {
PADDLE_ENFORCE_NOT_NULL(
Y_v,
platform::errors::NotFound(
"Can not find %s presistale var of sparse_fc in scope.", w_name));
"Can not find %s presistable var of sparse_fc in scope.", w_name));
auto* Y_t = Y_v->GetMutable<phi::DenseTensor>();
int x_num_col_dims =
op_desc.HasAttr("x_num_col_dims")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class TransLayerNormOpConverter : public OpConverter {
nvinfer1::ILayer* layernorm_layer = nullptr;
if (engine_->with_dynamic_shape()) {
// For dynamic shape,
// the shape of mean and variance will be determine in configuPlugin.
// the shape of mean and variance will be determine in configurePlugin.
std::vector<int64_t> mean_shape{1};
std::vector<int64_t> variance_shape{1};
bool with_fp16 =
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/inference/tensorrt/convert/ut_helper.h
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ class TRTConvertValidation {
std::unique_ptr<framework::OpDesc> op_desc_;
const std::unordered_set<std::string>& parameters_;
framework::Scope& scope_;
// The ITensor of trt does not cotain the batch size,
// The ITensor of trt does not contain the batch size,
// bug, in most cases, we need to set batch size for
// fluid's tensor shape. This variable indicates
// whether to add batch size to tensor shape of fluid.
Expand Down