Skip to content

Add progressbar for datasets downloading #33302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 3, 2021
Merged

Add progressbar for datasets downloading #33302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1bd8790
add progressbar for datasets download
LielinJiang Jun 2, 2021
b4a85e9
rm unused code
LielinJiang Jun 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions python/paddle/dataset/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import paddle.dataset
import six.moves.cPickle as pickle
import glob
import paddle

__all__ = []

Expand Down Expand Up @@ -95,16 +96,19 @@ def download(url, module_name, md5sum, save_name=None):
chunk_size = 4096
total_length = int(total_length)
total_iter = total_length / chunk_size + 1
log_interval = total_iter / 20 if total_iter > 20 else 1
log_interval = total_iter // 20 if total_iter > 20 else 1
log_index = 0
bar = paddle.hapi.progressbar.ProgressBar(
total_iter, name='item')
for data in r.iter_content(chunk_size=chunk_size):
if six.PY2:
data = six.b(data)
f.write(data)
log_index += 1
bar.update(log_index, {})
if log_index % log_interval == 0:
sys.stderr.write(".")
sys.stdout.flush()
bar.update(log_index)

except Exception as e:
# re-try
continue
Expand Down
22 changes: 12 additions & 10 deletions python/paddle/hapi/progressbar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ def __init__(self,
width=30,
verbose=1,
start=True,
file=sys.stdout):
file=sys.stdout,
name='step'):
self._num = num
if isinstance(num, int) and num <= 0:
raise TypeError('num should be None or integer (> 0)')
Expand All @@ -47,6 +48,7 @@ def __init__(self,
if start:
self._start = time.time()
self._last_update = 0
self.name = name

self._dynamic_display = (
(hasattr(self.file, 'isatty') and
Expand Down Expand Up @@ -74,7 +76,7 @@ def start(self):
self.file.flush()
self._start = time.time()

def update(self, current_num, values=None):
def update(self, current_num, values={}):
now = time.time()

if current_num:
Expand All @@ -83,11 +85,11 @@ def update(self, current_num, values=None):
time_per_unit = 0

if time_per_unit >= 1 or time_per_unit == 0:
fps = ' - %.0fs/%s' % (time_per_unit, 'step')
fps = ' - %.0fs/%s' % (time_per_unit, self.name)
elif time_per_unit >= 1e-3:
fps = ' - %.0fms/%s' % (time_per_unit * 1e3, 'step')
fps = ' - %.0fms/%s' % (time_per_unit * 1e3, self.name)
else:
fps = ' - %.0fus/%s' % (time_per_unit * 1e6, 'step')
fps = ' - %.0fus/%s' % (time_per_unit * 1e6, self.name)

info = ''
if self._verbose == 1:
Expand All @@ -102,7 +104,7 @@ def update(self, current_num, values=None):
if self._num is not None:
numdigits = int(np.log10(self._num)) + 1

bar_chars = ('step %' + str(numdigits) + 'd/%d [') % (
bar_chars = (self.name + ' %' + str(numdigits) + 'd/%d [') % (
current_num, self._num)
prog = float(current_num) / self._num
prog_width = int(self._width * prog)
Expand All @@ -116,7 +118,7 @@ def update(self, current_num, values=None):
bar_chars += ('.' * (self._width - prog_width))
bar_chars += ']'
else:
bar_chars = 'step %3d' % current_num
bar_chars = self.name + ' %3d' % current_num

self._total_width = len(bar_chars)
sys.stdout.write(bar_chars)
Expand Down Expand Up @@ -162,10 +164,10 @@ def update(self, current_num, values=None):
elif self._verbose == 2 or self._verbose == 3:
if self._num:
numdigits = int(np.log10(self._num)) + 1
count = ('step %' + str(numdigits) + 'd/%d') % (current_num,
self._num)
count = (self.name + ' %' + str(numdigits) + 'd/%d') % (
current_num, self._num)
else:
count = 'step %3d' % current_num
count = self.name + ' %3d' % current_num
info = count + info

for k, val in values:
Expand Down