Skip to content

plotting

plotting

Utilities for building charts and visualizations of LLMeter results

These tools depend on Plotly, which you can either install separately or via the llmeter[plotting] extra.

boxplot_by_dimension

boxplot_by_dimension(result, dimension, name=None, **box_kwargs)

Create a box plot for a given dimension from responses in a Run Result.

Parameters:

Name Type Description Default
result Result

The Result object containing the response data

required
dimension str

The dimension to plot as histogram values

required
**box_kwargs Any

Additional keyword arguments to pass to go.Box

{}

Returns:

Type Description
Box

plotly.graph_objects.Box: Box plot of the specified dimension

Example

result = Result(...) fig = boxplot_by_dimension(result, "time_to_first_token") fig.show()

Source code in llmeter/plotting/plotting.py
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
def boxplot_by_dimension(
    result: Result, dimension: str, name: str | None = None, **box_kwargs: Any
) -> go.Box:
    """
    Create a box plot for a given dimension from responses in a Run Result.

    Args:
        result (Result): The Result object containing the response data
        dimension (str): The dimension to plot as histogram values
        **box_kwargs: Additional keyword arguments to pass to go.Box

    Returns:
        plotly.graph_objects.Box: Box plot of the specified dimension

    Example:
        >>> result = Result(...)
        >>> fig = boxplot_by_dimension(result, "time_to_first_token")
        >>> fig.show()
    """
    x = result.get_dimension(dimension)

    return go.Box(x=x, name=name or result.run_name, **box_kwargs)

histogram_by_dimension

histogram_by_dimension(result, dimension, filter_dimension=None, filter_value=None, **histogram_kwargs)

Create a histogram plot for a given dimension from the result data.

Parameters:

Name Type Description Default
result Result

The Result object containing the response data

required
dimension str

The dimension to plot as histogram values

required
filter_dimension str | None

Dimension to filter the data on. Defaults to None.

None
filter_value Any

Value to filter the filter_dimension by. Defaults to None.

None
**histogram_kwargs Any

Additional keyword arguments to pass to go.Histogram

{}

Returns:

Type Description
Histogram

plotly.graph_objects.Histogram: Histogram plot of the specified dimension

Example

result = Result(...) fig = histogram_by_dimension(result, "time_to_first_token") fig.show()

Source code in llmeter/plotting/plotting.py
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def histogram_by_dimension(
    result: Result,
    dimension: str,
    filter_dimension: str | None = None,
    filter_value: Any = None,
    **histogram_kwargs: Any,
) -> go.Histogram:
    """
    Create a histogram plot for a given dimension from the result data.

    Args:
        result (Result): The Result object containing the response data
        dimension (str): The dimension to plot as histogram values
        filter_dimension (str | None, optional): Dimension to filter the data on. Defaults to None.
        filter_value (Any, optional): Value to filter the filter_dimension by. Defaults to None.
        **histogram_kwargs: Additional keyword arguments to pass to go.Histogram

    Returns:
        plotly.graph_objects.Histogram: Histogram plot of the specified dimension

    Example:
        >>> result = Result(...)
        >>> fig = histogram_by_dimension(result, "time_to_first_token")
        >>> fig.show()
    """
    x = result.get_dimension(
        dimension, filter_dimension=filter_dimension, filter_value=filter_value
    )
    name = histogram_kwargs.pop("name", None) or result.run_name
    histnorm = histogram_kwargs.pop("histnorm", None) or "probability"

    return go.Histogram(x=x, name=name, histnorm=histnorm, **histogram_kwargs)

plot_heatmap

plot_heatmap(result, dimension, n_bins_x, n_bins_y, output_path=None, show_scatter=False)

Generate a heatmap visualization of token counts and a specified dimension.

Parameters:

Name Type Description Default
result Result

The Result object containing the response data

required
dimension str

The dimension to plot as the z-axis/color values

required
n_bins_x int

Number of bins for the x-axis histogram

required
n_bins_y int

Number of bins for the y-axis histogram

required
show_scatter bool

Whether to overlay scatter points. Defaults to False.

False

Returns:

Type Description
Figure

plotly.graph_objs._figure.Figure: The generated heatmap figure

Raises:

Type Description
ValueError

If the specified dimension is not found in the result data

Source code in llmeter/plotting/plotting.py
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
def plot_heatmap(
    result: Result,
    dimension: str,
    n_bins_x: int | None,
    n_bins_y: int | None,
    output_path: Path | None = None,
    show_scatter=False,
) -> go.Figure:
    """
    Generate a heatmap visualization of token counts and a specified dimension.

    Args:
        result (Result): The Result object containing the response data
        dimension (str): The dimension to plot as the z-axis/color values
        n_bins_x (int): Number of bins for the x-axis histogram
        n_bins_y (int): Number of bins for the y-axis histogram
        show_scatter (bool, optional): Whether to overlay scatter points. Defaults to False.

    Returns:
        plotly.graph_objs._figure.Figure: The generated heatmap figure

    Raises:
        ValueError: If the specified dimension is not found in the result data
    """

    num_tokens_input_list = result.get_dimension("num_tokens_input")
    num_tokens_output_list = result.get_dimension("num_tokens_output")
    try:
        z = result.get_dimension(dimension)
    except AttributeError:
        raise ValueError(f"Dimension {dimension} not found in result")

    fig = px.density_heatmap(
        x=num_tokens_input_list,
        y=num_tokens_output_list,
        z=z,
        marginal_x="histogram",
        marginal_y="histogram",
        template="plotly_white",
        histfunc="avg",
        nbinsx=n_bins_x,
        nbinsy=n_bins_y,
        labels={
            "x": "Number of input tokens",
            "y": "Number of output tokens",
        },
        title=f'LLMeter: average {dimension.replace("_", " ").capitalize()}',
        width=800,
        height=800,
    )
    fig.layout["coloraxis"]["colorbar"]["title"] = "average"  # type: ignore
    fig.update_coloraxes(colorbar_tickformat=".1s")
    fig.update_coloraxes(colorbar_ticksuffix="s")
    fig.update_layout(coloraxis_colorscale="oryel")
    fig.update_traces(marker_color="grey", selector=dict(type="histogram"))
    fig.update_traces(texttemplate="%{z:.3s}s <br> %{x:.2s}, %{y:.2s}")
    fig.update_traces(
        texttemplate="%{y:.3s}", selector=dict(type="histogram", bingroup="x")
    )
    fig.update_traces(
        texttemplate="%{x:.3s}", selector=dict(type="histogram", bingroup="y")
    )
    if show_scatter:
        fig.add_trace(
            go.Scatter(
                x=num_tokens_input_list,
                y=num_tokens_output_list,
                mode="markers",
                showlegend=False,
                marker=dict(
                    symbol="x",
                    opacity=0.7,
                    color="white",
                    size=8,
                    line=dict(width=1),
                ),
            )
        )
    return fig

plot_load_test_results

plot_load_test_results(load_test_result, log_scale=True)

Generate a collection of plots visualizing different metrics from a load test result.

Parameters:

Name Type Description Default
load_test_result LoadTestResult

The load test result object containing the data to plot

required
log_scale bool

Whether to use logarithmic scale for the plots. Defaults to True.

True

Returns:

Name Type Description
dict dict[str, Figure]

Dictionary containing the following plots: - time_to_first_token: Figure showing time to first token vs number of clients - time_to_last_token: Figure showing time to last token vs number of clients - requests_per_minute: Figure showing requests per minute vs number of clients - error_rate: Figure showing error rate vs number of clients - average_input_tokens_clients: Figure showing average input tokens per minute vs clients - average_output_tokens_clients: Figure showing average output tokens per minute vs clients

Source code in llmeter/plotting/plotting.py
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
def plot_load_test_results(
    load_test_result: LoadTestResult,
    log_scale=True,
) -> dict[str, go.Figure]:
    """
    Generate a collection of plots visualizing different metrics from a load test result.

    Args:
        load_test_result (LoadTestResult): The load test result object containing the data to plot
        log_scale (bool, optional): Whether to use logarithmic scale for the plots. Defaults to True.

    Returns:
        dict: Dictionary containing the following plots:
            - time_to_first_token: Figure showing time to first token vs number of clients
            - time_to_last_token: Figure showing time to last token vs number of clients
            - requests_per_minute: Figure showing requests per minute vs number of clients
            - error_rate: Figure showing error rate vs number of clients
            - average_input_tokens_clients: Figure showing average input tokens per minute vs clients
            - average_output_tokens_clients: Figure showing average output tokens per minute vs clients
    """
    f1 = latency_clients_fig(
        load_test_result, "time_to_first_token", log_scale=log_scale
    )
    f2 = latency_clients_fig(
        load_test_result, "time_to_last_token", log_scale=log_scale
    )
    f3 = rpm_clients_fig(load_test_result, log_scale=log_scale)
    f4 = error_clients_fig(load_test_result, log_scale=log_scale)
    f5 = average_input_tokens_clients_fig(load_test_result, log_scale=log_scale)
    f6 = average_output_tokens_clients_fig(load_test_result, log_scale=log_scale)

    return {
        "time_to_first_token": f1,
        "time_to_last_token": f2,
        "requests_per_minute": f3,
        "error_rate": f4,
        "average_input_tokens_clients": f5,
        "average_output_tokens_clients": f6,
    }

scatter_histogram_2d

scatter_histogram_2d(result, x_dimension, y_dimension, n_bins_x, n_bins_y)

Generate a scatter plot with histograms for the given dimensions.

Parameters:

Name Type Description Default
result Result

The Result object containing the data.

required
x_dimension str

The dimension to be plotted on the x-axis.

required
y_dimension str

The dimension to be plotted on the y-axis.

required
n_bins_x int

The number of bins for the x-axis histogram.

required
n_bins_y int

The number of bins for the y-axis histogram.

required

Returns:

Type Description
Figure

plotly.graph_objs._figure.Figure: The generated scatter plot figure.

Source code in llmeter/plotting/plotting.py
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
def scatter_histogram_2d(
    result: Result, x_dimension: str, y_dimension: str, n_bins_x: int, n_bins_y: int
) -> go.Figure:
    """
    Generate a scatter plot with histograms for the given dimensions.

    Args:
        result (Result): The Result object containing the data.
        x_dimension (str): The dimension to be plotted on the x-axis.
        y_dimension (str): The dimension to be plotted on the y-axis.
        n_bins_x (int): The number of bins for the x-axis histogram.
        n_bins_y (int): The number of bins for the y-axis histogram.

    Returns:
        plotly.graph_objs._figure.Figure: The generated scatter plot figure.
    """

    try:
        x = result.get_dimension(x_dimension)
        y = result.get_dimension(y_dimension)
    except AttributeError:
        raise ValueError(f"Invalid dimension: {x_dimension} or {y_dimension}")

    fig = px.scatter(
        x=x,
        y=y,
        marginal_x="histogram",
        marginal_y="histogram",
        template="plotly_white",
        range_x=[0, max(x for x in x if x is not None) * 1.1],
        range_y=[0, max(y for y in y if y is not None) * 1.1],
        labels={
            "x": x_dimension.replace("_", " ").capitalize(),
            "y": y_dimension.replace("_", " ").capitalize(),
        },
        width=800,
        height=800,
    )
    fig.update_traces(nbinsx=n_bins_x, autobinx=True, selector={"type": "histogram"})
    fig.update_traces(nbinsy=n_bins_y, autobinx=True, selector={"type": "histogram"})
    return fig