Skip to content

Refine understand_sentiment_dynamic_lstm #7004

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Refine understand_sentiment_dynamic_lstm #7004

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 70 additions & 13 deletions python/paddle/v2/fluid/tests/book/test_understand_sentiment_dynamic_lstm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import math
import numpy as np
import paddle.v2 as paddle
import paddle.v2.fluid as fluid
from paddle.v2.fluid.param_attr import ParamAttr
from paddle.v2.fluid.initializer import NormalInitializer


def stacked_lstm_net(data,
Expand All @@ -9,32 +12,82 @@ def stacked_lstm_net(data,
class_dim=2,
emb_dim=128,
hid_dim=512,
stacked_num=3):
stacked_num=3,
batch_size=100):
assert stacked_num % 2 == 1

emb = fluid.layers.embedding(input=data, size=[input_dim, emb_dim])
# add bias attr

# TODO(qijun) linear act
fc1 = fluid.layers.fc(input=emb, size=hid_dim)
lstm1, cell1 = fluid.layers.dynamic_lstm(input=fc1, size=hid_dim)
emb = fluid.layers.embedding(
input=data,
size=[input_dim, emb_dim],
param_attr=ParamAttr(initializer=NormalInitializer(
Copy link
Contributor

@dzhwinter dzhwinter Dec 25, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

loc default value is .0.

loc=0., scale=1.0 / math.sqrt(input_dim))))

fc1 = fluid.layers.fc(input=emb,
size=hid_dim,
bias_attr=ParamAttr(initializer=NormalInitializer(
Copy link
Contributor

@dzhwinter dzhwinter Dec 25, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Create a ParamAttr, and share it between layers.

loc=0., scale=0.)),
param_attr=ParamAttr(
name='fc1',
initializer=NormalInitializer(
loc=0., scale=1.0 / math.sqrt(emb_dim))))
lstm1, cell1 = fluid.layers.dynamic_lstm(
input=fc1,
size=hid_dim,
candidate_activation='relu',
bias_attr=ParamAttr(initializer=NormalInitializer(
Copy link
Contributor

@dzhwinter dzhwinter Dec 25, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same above

loc=0., scale=0.)),
param_attr=ParamAttr(initializer=NormalInitializer(
loc=0., scale=1.0 / math.sqrt(emb_dim))))

inputs = [fc1, lstm1]

for i in range(2, stacked_num + 1):
fc = fluid.layers.fc(input=inputs, size=hid_dim)
fc = fluid.layers.fc(input=inputs,
Copy link
Contributor

@dzhwinter dzhwinter Dec 25, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please format these lines

size=hid_dim,
bias_attr=ParamAttr(initializer=NormalInitializer(
loc=0., scale=0.)),
param_attr=[
ParamAttr(
learning_rate=1e-3,
initializer=NormalInitializer(
loc=0., scale=1.0 /
math.sqrt(hid_dim))), ParamAttr(
learning_rate=1.,
initializer=NormalInitializer(
loc=0., scale=0.))
])
lstm, cell = fluid.layers.dynamic_lstm(
input=fc, size=hid_dim, is_reverse=(i % 2) == 0)
input=fc,
size=hid_dim,
is_reverse=(i % 2) == 0,
candidate_activation='relu',
bias_attr=ParamAttr(initializer=NormalInitializer(
Copy link
Contributor

@dzhwinter dzhwinter Dec 25, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same

loc=0., scale=0.)),
param_attr=ParamAttr(initializer=NormalInitializer(
loc=0., scale=1.0 / math.sqrt(emb_dim))))
inputs = [fc, lstm]

fc_last = fluid.layers.sequence_pool(input=inputs[0], pool_type='max')
lstm_last = fluid.layers.sequence_pool(input=inputs[1], pool_type='max')

prediction = fluid.layers.fc(input=[fc_last, lstm_last],
size=class_dim,
act='softmax')
prediction = fluid.layers.fc(
input=[fc_last, lstm_last],
size=class_dim,
bias_attr=ParamAttr(initializer=NormalInitializer(
Copy link
Contributor

@dzhwinter dzhwinter Dec 25, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And these lines.

loc=0., scale=0.)),
param_attr=[
ParamAttr(
learning_rate=1e-3,
initializer=NormalInitializer(
loc=0., scale=1.0 / math.sqrt(hid_dim))), ParamAttr(
learning_rate=1.,
initializer=NormalInitializer(
loc=0., scale=0.))
],
act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=label)
avg_cost = fluid.layers.mean(x=cost)
avg_cost = fluid.layers.scale(x=avg_cost, scale=float(batch_size))
adam_optimizer = fluid.optimizer.Adam(learning_rate=0.002)
adam_optimizer.minimize(avg_cost)
accuracy = fluid.evaluator.Accuracy(input=prediction, label=label)
Expand Down Expand Up @@ -69,7 +122,11 @@ def main():
name="words", shape=[1], dtype="int64", lod_level=1)
label = fluid.layers.data(name="label", shape=[1], dtype="int64")
cost, accuracy, acc_out = stacked_lstm_net(
data, label, input_dim=dict_dim, class_dim=class_dim)
data,
label,
input_dim=dict_dim,
class_dim=class_dim,
batch_size=BATCH_SIZE)

train_data = paddle.batch(
paddle.reader.shuffle(
Expand Down