Skip to content

Commit 7b09143

Browse files
authored
libpython: Support benchmarks of non-parallel runs better (#1733)
* Function for possibly non-parallel repeated runs for writing benchmark scripts. * Better documentation of non-parallel runs in resolution-changing benchmark. * CLI for joining JSON result files from multiple benchmarks and plotting from a file. * CLI which is using argparse with subcommands (subparsers) is extensible and more can be added in the future.
1 parent 3e3a881 commit 7b09143

File tree

9 files changed

+374
-11
lines changed

9 files changed

+374
-11
lines changed

python/grass/benchmark/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ include $(MODULE_TOPDIR)/include/Make/Python.make
55

66
DSTDIR = $(ETC)/python/grass/benchmark
77

8-
MODULES = plots results runners
8+
MODULES = app plots results runners __main__
99

1010
PYFILES := $(patsubst %,$(DSTDIR)/%.py,$(MODULES) __init__)
1111
PYCFILES := $(patsubst %,$(DSTDIR)/%.pyc,$(MODULES) __init__)

python/grass/benchmark/__init__.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,36 @@
1+
# MODULE: grass.benchmark
2+
#
3+
# AUTHOR(S): Vaclav Petras <wenzeslaus gmail com>
4+
#
5+
# PURPOSE: Benchmarking for GRASS GIS modules
6+
#
7+
# COPYRIGHT: (C) 2021 Vaclav Petras, and by the GRASS Development Team
8+
#
9+
# This program is free software under the GNU General Public
10+
# License (>=v2). Read the file COPYING that comes with GRASS
11+
# for details.
12+
113
"""Benchmarking for GRASS GIS modules
214
315
This subpackage of the grass package is experimental and the API can change anytime.
416
The API of the package is defined by what is imported in the top-level ``__init__.py``
517
file of the subpackage.
18+
19+
The functions in the Python API raise exceptions, although calls of other functions from
20+
the grass package may call grass.script.fatal and exit
21+
(see :func:`grass.script.core.set_raise_on_error` for changing the behavior).
22+
This applies to the CLI interface of this subpackage too except that raised usage
23+
exceptions originating in the CLI code result in *sys.exit* with an error message, not
24+
traceback. Messages and other user-visible texts in this package are not translatable.
625
"""
726

827
from .plots import nprocs_plot, num_cells_plot
928
from .results import (
1029
join_results,
30+
join_results_from_files,
1131
load_results,
1232
load_results_from_file,
1333
save_results,
1434
save_results_to_file,
1535
)
16-
from .runners import benchmark_nprocs, benchmark_resolutions
36+
from .runners import benchmark_nprocs, benchmark_resolutions, benchmark_single

python/grass/benchmark/__main__.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# MODULE: grass.benchmark
2+
#
3+
# AUTHOR(S): Vaclav Petras <wenzeslaus gmail com>
4+
#
5+
# PURPOSE: Benchmarking for GRASS GIS modules
6+
#
7+
# COPYRIGHT: (C) 2021 Vaclav Petras, and by the GRASS Development Team
8+
#
9+
# This program is free software under the GNU General Public
10+
# License (>=v2). Read the file COPYING that comes with GRASS
11+
# for details.
12+
13+
14+
"""The main file for executing using python -m"""
15+
16+
from grass.benchmark.app import main
17+
18+
if __name__ == "__main__":
19+
main()

python/grass/benchmark/app.py

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
# MODULE: grass.benchmark
2+
#
3+
# AUTHOR(S): Vaclav Petras <wenzeslaus gmail com>
4+
#
5+
# PURPOSE: Benchmarking for GRASS GIS modules
6+
#
7+
# COPYRIGHT: (C) 2021 Vaclav Petras, and by the GRASS Development Team
8+
#
9+
# This program is free software under the GNU General Public
10+
# License (>=v2). Read the file COPYING that comes with GRASS
11+
# for details.
12+
13+
14+
"""CLI for the benchmark package"""
15+
16+
import argparse
17+
import sys
18+
from pathlib import Path
19+
20+
from grass.benchmark import (
21+
join_results_from_files,
22+
load_results_from_file,
23+
num_cells_plot,
24+
save_results_to_file,
25+
)
26+
27+
28+
class CliUsageError(ValueError):
29+
"""Raised when error is in the command line arguments.
30+
31+
Used when the error is discovered only after argparse parsed the arguments.
32+
"""
33+
34+
# ArgumentError from argparse may work too, but it is not documented and
35+
# takes a reference argument which we don't have access to after the parse step.
36+
pass
37+
38+
39+
def join_results_cli(args):
40+
"""Translate CLI parser result to API calls."""
41+
if args.prefixes and len(args.results) != len(args.prefixes):
42+
raise CliUsageError(
43+
f"Number of prefixes ({len(args.prefixes)}) needs to be the same"
44+
f" as the number of input result files ({len(args.results)})"
45+
)
46+
results = join_results_from_files(
47+
source_filenames=args.results,
48+
prefixes=args.prefixes,
49+
)
50+
save_results_to_file(results, args.output)
51+
52+
53+
def plot_cells_cli(args):
54+
"""Translate CLI parser result to API calls."""
55+
results = load_results_from_file(args.input)
56+
num_cells_plot(
57+
results.results,
58+
filename=args.output,
59+
title=args.title,
60+
show_resolution=args.resolutions,
61+
)
62+
63+
64+
def get_executable_name():
65+
"""Get name of the executable and module.
66+
67+
This is a workaround for Python issue:
68+
argparse support for "python -m module" in help
69+
https://bugs.python.org/issue22240
70+
"""
71+
executable = Path(sys.executable).stem
72+
return f"{executable} -m grass.benchmark"
73+
74+
75+
class ExtendAction(argparse.Action):
76+
"""Support for agrparse action="extend" before Python 3.8
77+
78+
Each parser instance needs the action to be registered.
79+
"""
80+
81+
# pylint: disable=too-few-public-methods
82+
def __call__(self, parser, namespace, values, option_string=None):
83+
items = getattr(namespace, self.dest) or []
84+
items.extend(values)
85+
setattr(namespace, self.dest, items)
86+
87+
88+
def add_subcommand_parser(subparsers, name, description):
89+
"""Add parser for a subcommand into subparsers."""
90+
# help is in parent's help, description in subcommand's help.
91+
return subparsers.add_parser(name, help=description, description=description)
92+
93+
94+
def add_subparsers(parser, dest):
95+
"""Add subparsers in a unified way.
96+
97+
Uses title 'subcommands' for the list of commands
98+
(instead of the 'positional' which is the default).
99+
100+
The *dest* should be 'command', 'subcommand', etc. with appropriate nesting.
101+
"""
102+
if sys.version_info < (3, 7):
103+
# required as parameter is only in >=3.7.
104+
return parser.add_subparsers(title="subcommands", dest=dest)
105+
return parser.add_subparsers(title="subcommands", required=True, dest=dest)
106+
107+
108+
def add_results_subcommand(parent_subparsers):
109+
"""Add results subcommand."""
110+
main_parser = add_subcommand_parser(
111+
parent_subparsers, "results", description="Manipulate results"
112+
)
113+
main_subparsers = add_subparsers(main_parser, dest="subcommand")
114+
115+
join = main_subparsers.add_parser("join", help="Join results")
116+
join.add_argument("results", help="Files with results", nargs="*", metavar="file")
117+
join.add_argument("output", help="Output file", metavar="output_file")
118+
if sys.version_info < (3, 8):
119+
join.register("action", "extend", ExtendAction)
120+
join.add_argument(
121+
"--prefixes",
122+
help="Add prefixes to result labels per file",
123+
action="extend",
124+
nargs="*",
125+
metavar="text",
126+
)
127+
join.set_defaults(handler=join_results_cli)
128+
129+
130+
def add_plot_subcommand(parent_subparsers):
131+
"""Add plot subcommand."""
132+
main_parser = add_subcommand_parser(
133+
parent_subparsers, "plot", description="Plot results"
134+
)
135+
main_subparsers = add_subparsers(main_parser, dest="subcommand")
136+
137+
join = main_subparsers.add_parser("cells", help="Plot for variable number of cells")
138+
join.add_argument("input", help="file with results (JSON)", metavar="input_file")
139+
join.add_argument(
140+
"output", help="output file (e.g., PNG)", nargs="?", metavar="output_file"
141+
)
142+
join.add_argument(
143+
"--title",
144+
help="Title for the plot",
145+
metavar="text",
146+
)
147+
join.add_argument(
148+
"--resolutions",
149+
help="Use resolutions for x axis instead of cell count",
150+
action="store_true",
151+
)
152+
join.set_defaults(handler=plot_cells_cli)
153+
154+
155+
def define_arguments():
156+
"""Define top level parser and create subparsers."""
157+
parser = argparse.ArgumentParser(
158+
description="Process results from module benchmarks.",
159+
prog=get_executable_name(),
160+
)
161+
subparsers = add_subparsers(parser, dest="command")
162+
163+
add_results_subcommand(subparsers)
164+
add_plot_subcommand(subparsers)
165+
166+
return parser
167+
168+
169+
def main(args=None):
170+
"""Define and parse command line parameters then run the appropriate handler."""
171+
parser = define_arguments()
172+
args = parser.parse_args(args)
173+
try:
174+
args.handler(args)
175+
except CliUsageError as error:
176+
# Report a usage error and exit.
177+
sys.exit(f"ERROR: {error}")
178+
179+
180+
if __name__ == "__main__":
181+
main()

python/grass/benchmark/plots.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def get_pyplot(to_file):
4040
def nprocs_plot(results, filename=None):
4141
"""Plot results from a multiple nprocs (thread) benchmarks.
4242
43-
*results* is a list of individual results from separate benchmars.
43+
*results* is a list of individual results from separate benchmarks.
4444
One result is required to have attributes: *nprocs*, *times*, *label*.
4545
The *nprocs* attribute is a list of all processing elements
4646
(cores, threads, processes) used in the benchmark.
@@ -76,10 +76,10 @@ def nprocs_plot(results, filename=None):
7676
plt.show()
7777

7878

79-
def num_cells_plot(results, filename=None, show_resolution=False):
79+
def num_cells_plot(results, filename=None, title=None, show_resolution=False):
8080
"""Plot results from a multiple raster grid size benchmarks.
8181
82-
*results* is a list of individual results from separate benchmars
82+
*results* is a list of individual results from separate benchmarks
8383
with one result being similar to the :func:`nprocs_plot` function.
8484
The result is required to have *times* and *label* attributes
8585
and may have an *all_times* attribute.
@@ -116,6 +116,12 @@ def num_cells_plot(results, filename=None, show_resolution=False):
116116
else:
117117
plt.xlabel("Number of cells")
118118
plt.ylabel("Time [s]")
119+
if title:
120+
plt.title(title)
121+
elif show_resolution:
122+
plt.title("Execution time by resolution")
123+
else:
124+
plt.title("Execution time by cell count")
119125
if filename:
120126
plt.savefig(filename)
121127
else:

python/grass/benchmark/results.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,3 +93,11 @@ def join_results(results, prefixes=None):
9393
result.label = f"{prefix}: {result.label}"
9494
joined.append(result)
9595
return joined
96+
97+
98+
def join_results_from_files(source_filenames, prefixes):
99+
"""Join multiple files into one results object."""
100+
to_merge = []
101+
for result_file in source_filenames:
102+
to_merge.append(load_results_from_file(result_file))
103+
return join_results(to_merge, prefixes=prefixes)

0 commit comments

Comments
 (0)