|
| 1 | +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. |
| 2 | +// |
| 3 | +// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +// you may not use this file except in compliance with the License. |
| 5 | +// You may obtain a copy of the License at |
| 6 | +// |
| 7 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +// |
| 9 | +// Unless required by applicable law or agreed to in writing, software |
| 10 | +// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +// See the License for the specific language governing permissions and |
| 13 | +// limitations under the License. |
| 14 | + |
| 15 | +#include "paddle/fluid/eager/backward.h" |
| 16 | +#include <queue> |
| 17 | + |
| 18 | +#include "paddle/fluid/eager/autograd_meta.h" |
| 19 | +#include "paddle/fluid/eager/grad_node_info.h" |
| 20 | +#include "paddle/fluid/eager/grad_tensor_holder.h" |
| 21 | +#include "paddle/fluid/eager/utils.h" |
| 22 | + |
| 23 | +#include "paddle/fluid/platform/enforce.h" |
| 24 | +#include "paddle/fluid/platform/errors.h" |
| 25 | + |
| 26 | +#include "glog/logging.h" |
| 27 | + |
| 28 | +namespace egr { |
| 29 | + |
| 30 | +std::unordered_map<GradNodeBase*, int> getInDegreeMap( |
| 31 | + const std::queue<GradNodeBase*>& init_queue) { |
| 32 | + // Calculate in_degree for each node |
| 33 | + // We can completely remove this pass, if in_degree were set during forward |
| 34 | + // pass |
| 35 | + std::unordered_map<GradNodeBase*, int> node_in_degree_map; |
| 36 | + |
| 37 | + // Copy nodes |
| 38 | + std::queue<GradNodeBase*> queue = init_queue; |
| 39 | + std::unordered_set<GradNodeBase*> visited; |
| 40 | + |
| 41 | + // Visit each node exactly once in any order |
| 42 | + while (!queue.empty()) { |
| 43 | + GradNodeBase* node = queue.front(); |
| 44 | + queue.pop(); |
| 45 | + |
| 46 | + if (visited.count(node)) { |
| 47 | + continue; |
| 48 | + } |
| 49 | + visited.insert(node); |
| 50 | + |
| 51 | + // Find and append next nodes |
| 52 | + const std::vector<std::vector<Edge>>& edges = node->GetEdges(); |
| 53 | + for (const auto& edge_list : edges) { |
| 54 | + for (const Edge& edge : edge_list) { |
| 55 | + GradNodeBase* next_node = edge.GetMutableGradNode().get(); |
| 56 | + // Update in_degree |
| 57 | + if (!node_in_degree_map.count(next_node)) |
| 58 | + node_in_degree_map[next_node] = 0; |
| 59 | + node_in_degree_map[next_node]++; |
| 60 | + queue.push(next_node); |
| 61 | + } |
| 62 | + } |
| 63 | + } |
| 64 | + |
| 65 | + return node_in_degree_map; |
| 66 | +} |
| 67 | + |
| 68 | +void RunBackward(const std::vector<egr::EagerTensor>& tensors, |
| 69 | + const std::vector<egr::EagerTensor>& grad_tensors, |
| 70 | + bool retain_graph) { |
| 71 | + VLOG(6) << "Start Backward"; |
| 72 | + // *Gradient Hook should happen at node-level |
| 73 | + // *Inplace version check should perform at node-level |
| 74 | + // *Cross-batch accumulation happens at forward pass |
| 75 | + |
| 76 | + /* --- Initialization --- */ |
| 77 | + // 1. Init queue with starting nodes |
| 78 | + // 2. Prepare initial input buffers |
| 79 | + std::queue<GradNodeBase*> queue; |
| 80 | + std::unordered_map<GradNodeBase*, std::unique_ptr<GradTensorHolder>> |
| 81 | + node_input_buffers_dict; |
| 82 | + for (size_t i = 0; i < tensors.size(); i++) { |
| 83 | + const egr::EagerTensor& tensor = tensors[i]; |
| 84 | + |
| 85 | + AutogradMeta* auto_grad_meta = EagerUtils::unsafe_autograd_meta(tensor); |
| 86 | + // Get grad input info from target tensors |
| 87 | + auto input_info = auto_grad_meta->OutRankInfo(); |
| 88 | + |
| 89 | + VLOG(2) << "Out Rank of Tensor is slot: " << input_info.first |
| 90 | + << ", rank: " << input_info.second; |
| 91 | + // Get target GradNodeBase from target tensors |
| 92 | + GradNodeBase* grad_node = auto_grad_meta->GetMutableGradNode().get(); |
| 93 | + |
| 94 | + PADDLE_ENFORCE(grad_node, |
| 95 | + paddle::platform::errors::Fatal( |
| 96 | + "Detected null grad_node." |
| 97 | + "Grad Node is nullptr for grad input tensor %d", |
| 98 | + i)); |
| 99 | + // Prepare GradTensorHolder |
| 100 | + if (!node_input_buffers_dict.count(grad_node)) { |
| 101 | + VLOG(6) << "Create Value for grad input tensor " << i; |
| 102 | + node_input_buffers_dict[grad_node] = |
| 103 | + std::make_unique<GradTensorHolder>(grad_node->InputMeta()); |
| 104 | + } |
| 105 | + |
| 106 | + if (grad_tensors.size() > 0) { |
| 107 | + PADDLE_ENFORCE( |
| 108 | + grad_tensors.size() == tensors.size(), |
| 109 | + paddle::platform::errors::Fatal( |
| 110 | + "Detected size mismatch between tensors and grad_tensors" |
| 111 | + "grad_tensors should either have " |
| 112 | + "size = 0 or same size as tensors")); |
| 113 | + // Feed given tensor if it's provided |
| 114 | + VLOG(6) << "Fill grad input tensor " << i << "with give grad tensor"; |
| 115 | + node_input_buffers_dict[grad_node]->add( |
| 116 | + input_info.first, input_info.second, grad_tensors[i]); |
| 117 | + |
| 118 | + } else { |
| 119 | + VLOG(6) << "Fill grad input tensor " << i << " with 1.0"; |
| 120 | + // Initialize tensor with 1.0 |
| 121 | + // Forward Tensor "tensor" is passed to indicate tensortype, datatype and |
| 122 | + // dims |
| 123 | + // GradTensorHolder will initialize another tensor with same tensortype, |
| 124 | + // datatype and dims but filled with 1.0 |
| 125 | + node_input_buffers_dict[grad_node]->add( |
| 126 | + input_info.first, input_info.second, tensor, true /*fill_one=true*/); |
| 127 | + } |
| 128 | + |
| 129 | + // Prepare queue |
| 130 | + queue.push(grad_node); |
| 131 | + } |
| 132 | + |
| 133 | + VLOG(6) << "Update In degree Map for backward"; |
| 134 | + // 3. Compute in_degree for each node |
| 135 | + std::unordered_map<GradNodeBase*, int> node_in_degree_map = |
| 136 | + getInDegreeMap(queue); |
| 137 | + |
| 138 | + /* --- Topological Visit --- */ |
| 139 | + // 1. Pop queue |
| 140 | + // 2. Run node |
| 141 | + // |- node(grads) |
| 142 | + // |- Prepare for next node |
| 143 | + // 3. Update queue |
| 144 | + VLOG(6) << "Run Backward"; |
| 145 | + while (!queue.empty()) { |
| 146 | + GradNodeBase* node = queue.front(); |
| 147 | + queue.pop(); |
| 148 | + |
| 149 | + // Run node: This is where Hook happens |
| 150 | + PADDLE_ENFORCE( |
| 151 | + node_input_buffers_dict.count(node), |
| 152 | + paddle::platform::errors::Fatal( |
| 153 | + "Unable to find next node in the InputBuufer" |
| 154 | + "Trying to run Node without configuring its GradTensorHolder")); |
| 155 | + |
| 156 | + std::unique_ptr<GradTensorHolder> node_input_buffer = |
| 157 | + std::move(node_input_buffers_dict[node]); |
| 158 | + VLOG(6) << "Run Backward Kernel with input_buffer"; |
| 159 | + // Run Backward Node and get outputs |
| 160 | + std::vector<std::vector<egr::EagerTensor>> grad_output_tensors = |
| 161 | + (*node)(node_input_buffer->Buffers()); |
| 162 | + // TODO(jiabin): Should we erase it or find a more efficient way. |
| 163 | + node_input_buffers_dict.erase(node); |
| 164 | + |
| 165 | + // Prepare GradTensorHolder for next node |
| 166 | + const std::vector<std::vector<Edge>>& edges = node->GetEdges(); |
| 167 | + |
| 168 | + PADDLE_ENFORCE(edges.size() == grad_output_tensors.size() || edges.empty(), |
| 169 | + paddle::platform::errors::Fatal( |
| 170 | + "Number of edges should be either empty ( for leaf node " |
| 171 | + ") or the same as number of output grad tensors")); |
| 172 | + |
| 173 | + for (size_t i = 0; i < edges.size(); i++) { |
| 174 | + for (size_t j = 0; j < edges[i].size(); j++) { |
| 175 | + const Edge& edge = edges[i][j]; |
| 176 | + auto edge_rank = edge.GetEdgeRankInfo(); |
| 177 | + // Since we make edge has as same rank as bwd outputs, we indexing them |
| 178 | + // with |
| 179 | + // the same rank(i, j) |
| 180 | + VLOG(6) << "Get Edge with slot: " << i << ", rank: " << j; |
| 181 | + egr::EagerTensor& grad_output_tensor = grad_output_tensors[i][j]; |
| 182 | + if (!grad_output_tensor.defined() || |
| 183 | + !grad_output_tensor.initialized()) { |
| 184 | + VLOG(6) << "We get grad_output_tensor with slot: " << i |
| 185 | + << ", rank: " << j << " as uninitialized or undefined tensor"; |
| 186 | + } |
| 187 | + GradNodeBase* next_node = edge.GetMutableGradNode().get(); |
| 188 | + |
| 189 | + if (!node_input_buffers_dict.count(next_node)) { |
| 190 | + node_input_buffers_dict[next_node] = |
| 191 | + std::make_unique<GradTensorHolder>(next_node->InputMeta()); |
| 192 | + } |
| 193 | + VLOG(6) << "Sum grad inputs for edge slot: " << edge_rank.first |
| 194 | + << ", rank: " << edge_rank.second; |
| 195 | + node_input_buffers_dict[next_node]->add( |
| 196 | + edge_rank.first, edge_rank.second, grad_output_tensor); |
| 197 | + |
| 198 | + // Update queue |
| 199 | + node_in_degree_map[next_node]--; |
| 200 | + PADDLE_ENFORCE(node_in_degree_map[next_node] >= 0, |
| 201 | + paddle::platform::errors::Fatal( |
| 202 | + "Detected in-degree value smaller than zero." |
| 203 | + "Node's in-degree cannot be negative")); |
| 204 | + if (node_in_degree_map[next_node] == 0) { |
| 205 | + queue.emplace(std::move(next_node)); |
| 206 | + } |
| 207 | + } |
| 208 | + } |
| 209 | + } |
| 210 | +} |
| 211 | + |
| 212 | +} // namespace egr |
0 commit comments