trex/benchmark.py at main · mooch443/trex · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
#!/usr/bin/env python3
import pandas as pd
import matplotlib.pyplot as plt
import re

# Read the CSV file generated by the benchmark.
df = pd.read_csv("/Users/tristan/trex/Application/beta/benchmark_samples.csv")

# Add a new column "Parameter" by extracting the text between '(' and ')' in the Mode column.
# For example, FAST_SETTING(track_max_speed) becomes "track_max_speed"
def extract_parameter(mode_str):
    match = re.search(r'\(([^)]+)\)', mode_str)
    return match.group(1) if match else ""

df["Parameter"] = df["Mode"].apply(extract_parameter)

# Create a new grouping column 'Group' where Modes starting with 'SETTING(' get their own group.
def determine_group(mode_str):
    if mode_str.startswith("SETTING("):
        return "SETTING"
    else:
        return "Other"

df["Group"] = df["Mode"].apply(determine_group)

# Print overall summary statistics.
print("Overall summary statistics:")
print(df.groupby("Mode")["Time"].agg(["count", "mean", "min", "max"]))

# Print summary statistics by group.
print("\nSummary statistics by Group:")
print(df.groupby("Group")["Time"].agg(["count", "mean", "min", "max"]))

# For each Group and Parameter combination, produce a separate plot.
group_param_combinations = df.groupby(["Group", "Parameter"]).size().reset_index().rename(columns={0:"count"})
for _, row in group_param_combinations.iterrows():
    group_name = row["Group"]
    param = row["Parameter"]
    df_subset = df[(df["Group"] == group_name) & (df["Parameter"] == param)]
    if df_subset.empty:
        continue
    title = f"Benchmark Time Distribution for {group_name} - {param}"
    filename = f"/Users/tristan/trex/Application/beta/benchmark_results_{group_name}_{param}.png"

    # Prepare data for plotting for this combination.
    modes = sorted(df_subset["Mode"].unique())
    data = [df_subset.loc[df_subset["Mode"] == mode, "Time"].values for mode in modes]

    plt.figure(figsize=(12, 10))
    if group_name == "SETTING":
        ylim = None  # No fixed limit
    elif "track_max_speed" in param:
        ylim = (0.02, 0.3)
    else:
        ylim = (0.1, 0.3)

    if ylim:
        plt.ylim(ylim)

    # Plot the boxplot
    boxplot = plt.boxplot(data, labels=modes, showfliers=False)

    # Add visual marks for out-of-view data points
    for i, mode_data in enumerate(data):
        upper_limit = ylim[1] if ylim else None
        if upper_limit:
            out_of_view = mode_data[mode_data > upper_limit]
            if len(out_of_view) > 0:
                # Place the mark just below the upper limit
                plt.text(i + 1, upper_limit, "▲",
                        ha="center", va="top", color="red", fontsize=12)

    plt.xticks(rotation=45, ha="right")
    plt.ylabel("Time (μs per call)")
    plt.title(title)
    plt.tight_layout()
    plt.savefig(filename)
    plt.show()

    # Print summary statistics for this group and parameter.
    summary = df_subset.groupby("Mode")["Time"].agg(["count", "mean", "min", "max"])
    print(f"\nSummary statistics for {group_name} - {param}:")
    print(summary)