Ryanunderhill/rel 1.1.0 (#2651)

* Add missig env variables for mac pipeline test (#2595)
* Java API for onnxruntime (#2215)
* Rename automl python tools folder to featurizer_ops. (#2593)
* Make sure fenced tensor could not reuse other tensor. (#2561)
* Add support for opset 11 in reshape fusion (#2592)
* Support opset 11 subgraph of Squad model in Embed Layer Normalization (#2605)
* Allow providers to be set for InferenceSession at construction (#2606)
* EmbedLayerNormalization Fusion For Dynamic Squad Model Opset 10 (#2613)
* Improve Embed Layer Norm Fusion for SQuAD with static input shape  (#2621)
* Improve cuda expand() opeator's performance. (#2624)
* Cuda pad optimize when no padding is needed. (#2625)
* Shortcut cuda Pad() when no padding is needed.
* Improve performance of resize() in Nearest mode (#2626)
* Optimize cuda scatter() on 2D compatible. (#2628)
* Optimize cuda scatter() on 2D compatible.
* fix float16 comparison in initializer (#2629)
* epsilon attribute for layernormalization fusion (#2639)
* Fix memory exception in Layer Norm Fusion (#2644)

Author: RyanUnderhill

onnxruntime/core/framework/allocation_planner.cc

@@ -325,6 +325,7 @@ class PlannerImpl {
     auto p_required_buffer_shape = context_.GetShape(output_arg);
     if (nullptr == p_required_buffer_shape) return false;
     auto& required_memory_info = AllocPlan(output_arg.Name()).location;
+    if (HasFence(&output_arg)) return false;
 
     for (auto it = freelist_.begin(); it != freelist_.end(); ++it) {
       size_t reusable = static_cast<size_t>(it->ml_value);

onnxruntime/core/optimizer/embed_layer_norm_fusion.cc

@@ -153,7 +153,6 @@ Status LayerNormFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level,
       continue;
     }
     nodes_to_remove.push_back(add2_node);
-
     // Traceback the add node to find reduceMean --> add
     const Node* p_reduce_mean2 = nullptr;
 
@@ -255,6 +254,15 @@ Status LayerNormFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level,
                                           layer_norm_input_defs,
                                           {}, {}, kOnnxDomain);
 
+    // Get constant "epsilon" from "Add2" node if available. Else, default value will be used.
+    const ONNX_NAMESPACE::TensorProto* tensor_proto = graph_utils::GetConstantInitializer(graph, add2_node.MutableInputDefs()[1]->Name());
+    if (tensor_proto != nullptr) {
+      if (tensor_proto->data_type() == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
+        auto initializer = onnxruntime::make_unique<Initializer>(*tensor_proto);
+        layer_norm_node.AddAttribute("epsilon", initializer->data<float>()[0]);
+      }
+    }
+
     // Assign provider to this new node. Provider should be same as the provider for old node.
     layer_norm_node.SetExecutionProviderType(reduce_mean_node.GetExecutionProviderType());
 

onnxruntime/core/optimizer/layer_norm_fusion.cc

@@ -72,7 +72,7 @@ bool ReshapeFusion::Fuse_Subgraph1(Node& reshape, Graph& graph, const logging::L
   }
   const Node& concat = *p_concat;
 
-  if (!graph_utils::IsSupportedOptypeVersionAndDomain(concat, "Concat", {1, 4})) {
+  if (!graph_utils::IsSupportedOptypeVersionAndDomain(concat, "Concat", {1, 4, 11})) {
     return false;
   }
 
@@ -83,8 +83,8 @@ bool ReshapeFusion::Fuse_Subgraph1(Node& reshape, Graph& graph, const logging::L
 
   // path 1: [Root] --> Shape --> Gather(indices=0) --> Unsqueeze (axes=0) --> Concat [input 0]
   std::vector<graph_utils::EdgeEndToMatch> parent_path{
-      {0, 0, "Unsqueeze", {1}, kOnnxDomain},
-      {0, 0, "Gather", {1}, kOnnxDomain},
+      {0, 0, "Unsqueeze", {1, 11}, kOnnxDomain},
+      {0, 0, "Gather", {1, 11}, kOnnxDomain},
       {0, 0, "Shape", {1}, kOnnxDomain}};
 
   std::vector<const Node::EdgeEnd*> edges;
@@ -114,8 +114,8 @@ bool ReshapeFusion::Fuse_Subgraph1(Node& reshape, Graph& graph, const logging::L
 
   // path 2: [Root] --> Shape --> Gather(indices=1) --> Unsqueeze (axes=0) --> Concat [input 1]
   std::vector<graph_utils::EdgeEndToMatch> parent_path2 {
-      {0, 1, "Unsqueeze", {1}, kOnnxDomain},
-      {0, 0, "Gather", {1}, kOnnxDomain},
+      {0, 1, "Unsqueeze", {1, 11}, kOnnxDomain},
+      {0, 0, "Gather", {1, 11}, kOnnxDomain},
       {0, 0, "Shape", {1}, kOnnxDomain}};
 
   if (!graph_utils::FindPath(concat, true, parent_path2, edges, logger)) {

onnxruntime/core/optimizer/reshape_fusion.cc

@@ -63,7 +63,7 @@ bool IsInitializerWithExpectedValue(const Graph& graph, const NodeArg& input_arg
     }
   } else if (data_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) {
     const MLFloat16* val = init_const->data<MLFloat16>();
-    float diff = std::abs(math::halfToFloat(val[0].val) - static_cast<float>(expected_value));
+    float diff = std::abs(math::halfToFloat(val[0].val) - math::halfToFloat(math::floatToHalf(expected_value)));
     if (diff > FLT_EPSILON) {
       return false;
     }

onnxruntime/core/optimizer/utils.cc

@@ -5,69 +5,105 @@
 #include "expand_impl.h"
 #include "core/providers/cpu/tensor/utils.h"
 
+using std::vector;
+
 namespace onnxruntime {
 namespace cuda {
 
+// Logically expanded y could just be a view of x.
+static void CalcEffectiveDims(vector<int64_t>& x_dims, vector<int64_t>& y_dims) {
+  vector<int64_t> x_reverse;
+  vector<int64_t> y_reverse;
+
+  int xi = gsl::narrow_cast<int>(x_dims.size()) - 1;
+  for (int yi = gsl::narrow_cast<int>(y_dims.size()) - 1; yi >= 0; --yi, --xi) {
+    int64_t xdim = (xi >= 0) ? x_dims[xi] : 1;
+    int64_t ydim = y_dims[yi];
+    if (xdim == ydim || xdim == 1) {
+      x_reverse.push_back(xdim);
+      y_reverse.push_back(ydim);
+    }
+    else { // xdim < ydim && xdim > 1, split
+      ydim /= xdim;
+      x_reverse.push_back(xdim);
+      y_reverse.push_back(xdim);
+      x_reverse.push_back(1);
+      y_reverse.push_back(ydim);
+    }
+  }
+
+  x_dims.clear();
+  y_dims.clear();
+  x_dims.push_back(1);
+  y_dims.push_back(1);
+  // compact the dims, remove (x=1, y=1), merge (x=1, y1*y2...)
+  for (int i = gsl::narrow_cast<int>(y_reverse.size()) - 1; i >= 0; --i) {
+    if (x_reverse[i] == 1) {
+      if (y_reverse[i] == 1) {
+        continue;
+      }
+      if (x_dims.back() == 1) {
+        y_dims.back() *= y_reverse[i];
+      }
+      else {
+        x_dims.push_back(1);
+        y_dims.push_back(y_reverse[i]);
+      }
+    }
+    else { // x_reverse[i] == y_reverse[i]
+      if (x_dims.back() == y_dims.back()) {
+        x_dims.back() *= x_reverse[i];
+        y_dims.back() *= y_reverse[i];
+      }
+      else {
+        x_dims.push_back(x_reverse[i]);
+        y_dims.push_back(y_reverse[i]);
+      }
+    }
+  }
+}
+
 Status Expand::ComputeInternal(OpKernelContext* ctx) const {
-  const auto& input0 = *ctx->Input<Tensor>(0);
-  const auto& input1 = *ctx->Input<Tensor>(1);
+  const auto& input_data_tensor = *ctx->Input<Tensor>(0);
+  const auto& input_shape_tensor = *ctx->Input<Tensor>(1);
 
   // new shape to be expanded to
-  const auto* p_shape = input1.template Data<int64_t>();
-  std::vector<int64_t> output_dims{p_shape, p_shape + input1.Shape().Size()};
+  const auto* p_shape = input_shape_tensor.template Data<int64_t>();
+  std::vector<int64_t> output_dims{p_shape, p_shape + input_shape_tensor.Shape().Size()};
   TensorShape output_shape(output_dims);
 
-  ORT_RETURN_IF_ERROR(ComputeOutputShape(Node().Name(), input0.Shape(), output_dims, output_shape));
-  auto rank = output_shape.NumDimensions();
+  ORT_RETURN_IF_ERROR(ComputeOutputShape(Node().Name(), input_data_tensor.Shape(), output_dims, output_shape));
   auto& output_tensor = *ctx->Output(0, output_shape);
-
   if (0 == output_shape.Size()) {
     return Status::OK();
   }
 
-  auto input_shape = input0.Shape().GetDims();
+  output_dims = output_shape.GetDims();
+  auto input_dims = input_data_tensor.Shape().GetDims();
 
-  // pad input_dims with 1 to make ranks match
-  for (size_t i = 0; i < rank - input_shape.size(); i++) {
-    input_shape.insert(input_shape.begin(), 1);
-  }
+  CalcEffectiveDims(input_dims, output_dims);
+  int rank = gsl::narrow_cast<int>(output_dims.size());
 
-  // create fast_divmod using dimension values
-  CudaAsyncBuffer<fast_divmod> fdm_input_dims(this, rank);
-  CudaAsyncBuffer<fast_divmod> fdm_output_dims(this, rank);
-  CudaAsyncBuffer<fast_divmod> fdm_output_subdim_size(this, rank);
-  {
-    auto in_span = fdm_input_dims.CpuSpan();
-    auto out_span = fdm_output_dims.CpuSpan();
-    auto sdm_span = fdm_output_subdim_size.CpuSpan();
-    auto subdim_size = output_shape.Size();
-    for (size_t i = 0; i < rank; i++) {
-      in_span[i] = fast_divmod(static_cast<int>(input_shape[i]));
-      out_span[i] = fast_divmod(static_cast<int>(output_shape[i]));
-      // output_shape[i] won't be 0 here, it's covered in (0 == output_shape.Size())
-      // a null output will be returned for that case
-      subdim_size /= output_shape[i];
-      sdm_span[i] = static_cast<int>(subdim_size);
-    }
+  CudaAsyncBuffer<fast_divmod> fdm_output_strides(this, rank);
+  ORT_ENFORCE(CalculateFdmStrides(fdm_output_strides.CpuSpan(), output_dims));
+
+  CudaAsyncBuffer<int64_t> input_view_strides(this, rank);
+  TensorPitches::Calculate(input_view_strides.CpuSpan(), input_dims);
+  for (int i = 0; i < rank; ++i) {
+    if (input_dims[i] == 1) input_view_strides.CpuSpan()[i] = 0;
   }
-  ORT_RETURN_IF_ERROR(fdm_input_dims.CopyToGpu());
-  ORT_RETURN_IF_ERROR(fdm_output_dims.CopyToGpu());
-  ORT_RETURN_IF_ERROR(fdm_output_subdim_size.CopyToGpu());
-
-  ExpandImpl(
-      input0.DataType()->Size(),
-      output_shape.NumDimensions(),
-      output_shape.Size(),
-      input0.Shape().Size(),
-      input0.DataRaw(),
-      output_tensor.MutableDataRaw(),
-      fdm_input_dims.GpuPtr(),
-      fdm_output_dims.GpuPtr(),
-      fdm_output_subdim_size.GpuPtr());
 
-  return Status::OK();
+  return ExpandImpl(
+      input_data_tensor.DataType()->Size(),
+      gsl::narrow_cast<int>(output_shape.Size()),
+      gsl::narrow_cast<int>(input_data_tensor.Shape().Size()),
+      input_data_tensor.DataRaw(),
+      output_tensor.MutableDataRaw(),
+      fdm_output_strides,
+      input_view_strides);
 }
 
+
 ONNX_OPERATOR_KERNEL_EX(
     Expand,
     kOnnxDomain,