Skip to content

Commit 828c5b3

Browse files
abhinavaroraabhinavarora
authored
Adding Adadelta optimization operator (#4576)
* Adding Adadelta optimization operator * Making inputs and outputs conform to naming convention * Removing type alias from header files * Fixing Adadelta documentation in comments * Addressing code review feedback
1 parent 1172f24 commit 828c5b3

File tree

4 files changed

+300
-0
lines changed

4 files changed

+300
-0
lines changed

‎paddle/operators/adadelta_op.cc‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#include "paddle/operators/adadelta_op.h"
16+
17+
namespace paddle {
18+
namespace operators {
19+
20+
class AdadeltaOp : public framework::OperatorWithKernel {
21+
public:
22+
using framework::OperatorWithKernel::OperatorWithKernel;
23+
24+
protected:
25+
void InferShape(framework::InferShapeContextBase *ctx) const override {
26+
PADDLE_ENFORCE(ctx->HasInput("Param"),
27+
"Input(Param) of AdadeltaOp should not be null.");
28+
PADDLE_ENFORCE(ctx->HasInput("Grad"),
29+
"Input(Grad) of AdadeltaOp should not be null.");
30+
PADDLE_ENFORCE(ctx->HasInput("AvgSquaredGrad"),
31+
"Input(AvgSquaredGrad) of AdadeltaOp should not be null.");
32+
PADDLE_ENFORCE(ctx->HasInput("AvgSquaredUpdate"),
33+
"Input(AvgSquaredUpdate) of AdadeltaOp should not be null.");
34+
35+
PADDLE_ENFORCE(ctx->HasOutput("ParamOut"),
36+
"Output(ParamOut) of AdadeltaOp should not be null.");
37+
PADDLE_ENFORCE(
38+
ctx->HasOutput("AvgSquaredGradOut"),
39+
"Output(AvgSquaredGradOut) of AdadeltaOp should not be null.");
40+
PADDLE_ENFORCE(
41+
ctx->HasOutput("AvgSquaredUpdateOut"),
42+
"Output(AvgSquaredUpdateOut) of AdadeltaOp should not be null.");
43+
44+
auto param_dim = ctx->GetInputDim("Param");
45+
PADDLE_ENFORCE_EQ(
46+
param_dim, ctx->GetInputDim("Grad"),
47+
"param and grad input of AdadeltaOp should have same dimension");
48+
PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("AvgSquaredGrad"),
49+
"Param and AvgSquaredGrad input of AdadeltaOp "
50+
"should have same dimension");
51+
PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("AvgSquaredUpdate"),
52+
"Param and AvgSquaredUpdate input of AdadeltaOp "
53+
"should have same dimension");
54+
55+
ctx->SetOutputDim("ParamOut", param_dim);
56+
ctx->SetOutputDim("AvgSquaredGradOut", param_dim);
57+
ctx->SetOutputDim("AvgSquaredUpdateOut", param_dim);
58+
}
59+
};
60+
61+
class AdadeltaOpMaker : public framework::OpProtoAndCheckerMaker {
62+
public:
63+
AdadeltaOpMaker(framework::OpProto *proto,
64+
framework::OpAttrChecker *op_checker)
65+
: OpProtoAndCheckerMaker(proto, op_checker) {
66+
AddInput("Param", "(Tensor) Input parameter");
67+
AddInput("Grad", "(Tensor) Input gradient");
68+
AddInput("AvgSquaredGrad",
69+
"(Tensor) Input expectation of squared gradient");
70+
AddInput("AvgSquaredUpdate",
71+
"(Tensor) Input expectation of squared parameter updates");
72+
73+
AddOutput("ParamOut", "(Tensor) Output parameter");
74+
AddOutput("AvgSquaredGradOut",
75+
"(Tensor) Output expectation of squared gradient");
76+
AddOutput("AvgSquaredUpdateOut",
77+
"(Tensor) Output expectation of squared parameter updates");
78+
79+
AddAttr<float>("rho",
80+
"(float, default 0.95) Exponential decay rate "
81+
"for squared gradients.")
82+
.SetDefault(0.95f);
83+
AddAttr<float>("epsilon",
84+
"(float, default 1.0e-6) Constant for "
85+
"numerical stability")
86+
.SetDefault(1.0e-6f);
87+
AddComment(R"DOC(
88+
Adadelta Updates Operator.
89+
90+
This implements the Adadelta optimizer[1]. Adadelta is a per-dimension
91+
adaptive learning rate method for gradient descent.
92+
93+
Adadelta updates:
94+
95+
avg_squared_grad_out = rho * avg_squared_grad + (1 - rho) * grad * grad
96+
param_update = - sqrt((avg_squared_update + epsilon) /
97+
(avg_squared_grad_out + epsilon)) * grad
98+
avg_squared_update_out = rho * avg_squared_update + (1 - rho) * param_update**2
99+
param_out = param + param_update
100+
101+
References:
102+
[1] ADADELTA: An Adaptive Learning Rate Method
103+
https://arxiv.org/abs/1212.5701
104+
105+
)DOC");
106+
}
107+
};
108+
109+
} // namespace operators
110+
} // namespace paddle
111+
112+
namespace ops = paddle::operators;
113+
REGISTER_OP_WITHOUT_GRADIENT(adadelta, ops::AdadeltaOp, ops::AdadeltaOpMaker);
114+
REGISTER_OP_CPU_KERNEL(
115+
adadelta, ops::AdadeltaOpKernel<paddle::platform::CPUPlace, float>);

‎paddle/operators/adadelta_op.cu‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#define EIGEN_USE_GPU
16+
#include "paddle/operators/adadelta_op.h"
17+
18+
namespace ops = paddle::operators;
19+
REGISTER_OP_GPU_KERNEL(
20+
adadelta, ops::AdadeltaOpKernel<paddle::platform::GPUPlace, float>);

‎paddle/operators/adadelta_op.h‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#pragma once
16+
#include "paddle/framework/eigen.h"
17+
#include "paddle/framework/op_registry.h"
18+
19+
namespace paddle {
20+
namespace operators {
21+
22+
template <typename Place, typename T>
23+
class AdadeltaOpKernel : public framework::OpKernel<T> {
24+
public:
25+
void Compute(const framework::ExecutionContext& ctx) const override {
26+
auto param_out_tensor = ctx.Output<framework::Tensor>("ParamOut");
27+
auto avg_squared_grad_out_tensor =
28+
ctx.Output<framework::Tensor>("AvgSquaredGradOut");
29+
auto avg_squared_update_out_tensor =
30+
ctx.Output<framework::Tensor>("AvgSquaredUpdateOut");
31+
32+
param_out_tensor->mutable_data<T>(ctx.GetPlace());
33+
avg_squared_grad_out_tensor->mutable_data<T>(ctx.GetPlace());
34+
avg_squared_update_out_tensor->mutable_data<T>(ctx.GetPlace());
35+
36+
float rho = ctx.Attr<float>("rho");
37+
float epsilon = ctx.Attr<float>("epsilon");
38+
39+
auto param = framework::EigenVector<T>::Flatten(
40+
*ctx.Input<framework::Tensor>("Param"));
41+
auto grad = framework::EigenVector<T>::Flatten(
42+
*ctx.Input<framework::Tensor>("Grad"));
43+
// Squared gradient accumulator
44+
auto avg_squared_grad = framework::EigenVector<T>::Flatten(
45+
*ctx.Input<framework::Tensor>("AvgSquaredGrad"));
46+
// Squared updates accumulator
47+
auto avg_squared_update = framework::EigenVector<T>::Flatten(
48+
*ctx.Input<framework::Tensor>("AvgSquaredUpdate"));
49+
auto param_out = framework::EigenVector<T>::Flatten(*param_out_tensor);
50+
auto avg_squared_grad_out =
51+
framework::EigenVector<T>::Flatten(*avg_squared_grad_out_tensor);
52+
auto avg_squared_update_out =
53+
framework::EigenVector<T>::Flatten(*avg_squared_update_out_tensor);
54+
auto place = ctx.GetEigenDevice<Place>();
55+
56+
avg_squared_grad_out.device(place) =
57+
rho * avg_squared_grad + (1 - rho) * grad.square();
58+
auto update =
59+
-((avg_squared_update + epsilon) / (avg_squared_grad_out + epsilon))
60+
.sqrt() *
61+
grad;
62+
avg_squared_update_out.device(place) =
63+
rho * avg_squared_update + (1 - rho) * update.square();
64+
param_out.device(place) = param + update;
65+
}
66+
};
67+
68+
} // namespace operators
69+
} // namespace paddle

‎python/paddle/v2/framework/tests/test_adadelta_op.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import unittest
2+
import numpy as np
3+
from op_test import OpTest
4+
5+
6+
class TestAdadeltaOp1(OpTest):
7+
def setUp(self):
8+
self.op_type = "adadelta"
9+
param = np.random.uniform(-1, 1, (102, 105)).astype("float32")
10+
grad = np.random.uniform(-1, 1, (102, 105)).astype("float32")
11+
# The squared gradient is positive
12+
avg_squared_grad = np.random.random((102, 105)).astype("float32")
13+
# The squared update is positive
14+
avg_squared_update = np.random.random((102, 105)).astype("float32")
15+
16+
rho = 0.95
17+
epsilon = 1e-6
18+
19+
self.inputs = {
20+
'Param': param,
21+
'Grad': grad,
22+
'AvgSquaredGrad': avg_squared_grad,
23+
'AvgSquaredUpdate': avg_squared_update
24+
}
25+
26+
self.attrs = {'rho': rho, 'epsilon': epsilon}
27+
28+
avg_squared_grad_out = rho * avg_squared_grad + \
29+
(1 - rho) * np.square(grad)
30+
update = -np.multiply(
31+
np.sqrt(
32+
np.divide(avg_squared_update + epsilon, avg_squared_grad_out +
33+
epsilon)), grad)
34+
35+
avg_squared_update_out = rho * avg_squared_update + \
36+
(1 - rho) * np.square(update)
37+
38+
param_out = param + update
39+
40+
self.outputs = {
41+
'ParamOut': param_out,
42+
'AvgSquaredGradOut': avg_squared_grad_out,
43+
'AvgSquaredUpdateOut': avg_squared_update_out
44+
}
45+
46+
def test_check_output(self):
47+
self.check_output()
48+
49+
50+
class TestAdadeltaOp2(OpTest):
51+
'''Test Adadelta op with default attribute values
52+
'''
53+
54+
def setUp(self):
55+
self.op_type = "adadelta"
56+
param = np.random.uniform(-1, 1, (102, 105)).astype("float32")
57+
grad = np.random.uniform(-1, 1, (102, 105)).astype("float32")
58+
# The squared gradient is positive
59+
avg_squared_grad = np.random.random((102, 105)).astype("float32")
60+
# The squared update is positive
61+
avg_squared_update = np.random.random((102, 105)).astype("float32")
62+
63+
rho = 0.95
64+
epsilon = 1e-6
65+
66+
self.inputs = {
67+
'Param': param,
68+
'Grad': grad,
69+
'AvgSquaredGrad': avg_squared_grad,
70+
'AvgSquaredUpdate': avg_squared_update
71+
}
72+
73+
avg_squared_grad_out = rho * avg_squared_grad + \
74+
(1 - rho) * np.square(grad)
75+
update = -np.multiply(
76+
np.sqrt(
77+
np.divide(avg_squared_update + epsilon, avg_squared_grad_out +
78+
epsilon)), grad)
79+
80+
avg_squared_update_out = rho * avg_squared_update + \
81+
(1 - rho) * np.square(update)
82+
83+
param_out = param + update
84+
85+
self.outputs = {
86+
'ParamOut': param_out,
87+
'AvgSquaredGradOut': avg_squared_grad_out,
88+
'AvgSquaredUpdateOut': avg_squared_update_out
89+
}
90+
91+
def test_check_output(self):
92+
self.check_output()
93+
94+
95+
if __name__ == "__main__":
96+
unittest.main()

0 commit comments

Comments
 (0)