Runner

def add_callback(self, callback: Callback):
    """
    Add a callback to the runner's list of callbacks.

    Args:
        callback (Callback): The callback to be added.
    """
    if self.callbacks is None:
        self.callbacks = [callback]
    else:
        self.callbacks.append(callback)

async def run(
    self,
    *,  # Prevent mistakes with this long arg list by allowing only keyword-arg based passing
    # Explicitly name and re-document the args for ease of use of this important public method
    endpoint: Endpoint | dict | None = None,
    output_path: WritablePathLike | None = None,
    tokenizer: Tokenizer | Any | None = None,
    clients: int | None = None,
    n_requests: int | None = None,
    run_duration: int | float | timedelta | None = None,
    payload: dict | list[dict] | ReadablePathLike | None = None,
    run_name: str | None = None,
    run_description: str | None = None,
    timeout: int | float | None = None,
    callbacks: list[Callback] | None = None,
    low_memory: bool | None = None,
    progress_bar_stats: dict[str, str | tuple[str, str]] | None = None,
    disable_per_client_progress_bar: bool | None = None,
    disable_clients_progress_bar: bool | None = None,
) -> Result:
    """
    Run a test against an LLM endpoint

    This method tests the performance of the endpoint by sending multiple concurrent requests
    with the given payload(s). It measures the total time taken to complete the test, generates
    invocations for the given payload(s), and optionally saves the results and metrics.

    For arguments that are not specified, the Runner's attributes will be used by default.

    Args:
        endpoint (Endpoint | dict | None): The LLM endpoint to be tested. **Must be set** at
            either the Runner or specific Run level.
        output_path (WritablePathLike | None): The (cloud or local) base folder under which
            run outputs and configurations should be stored. By default, a new `run_name`
            sub-folder will be created under the Runner's `output_path` if set - otherwise
            outputs will not be saved to file.
        tokenizer (Tokenizer | Any | None): Optional tokenizer used to estimate input and
            output token counts for endpoints that don't report exact information.
        clients (int): The number of concurrent clients to use for sending requests.
        n_requests (int | None): The number of LLM invocations to generate *per client*.
            Mutually exclusive with ``run_duration``.
        run_duration (int | float | timedelta | None): Run each client for this many seconds
            instead of a fixed request count.  Clients send requests continuously
            until the duration expires.  Mutually exclusive with ``n_requests``.
            Accepts a number of seconds or a ``timedelta``.

            Example::

                # Run for 60 seconds with 5 concurrent clients:
                result = await runner.run(run_duration=60, clients=5)
                result.total_requests  # actual count completed

        payload (dict | list[dict] | ReadablePathLike | None): The request data to send to the
            endpoint under test. You can provide a single JSON payload (dict), a list of
            payloads (list[dict]), or a path to one or more JSON/JSON-Lines files to be loaded
            by `llmeter.prompt_utils.load_payloads()`. **Must be set** at either the Runner or
            specific Run level.
        run_name (str | None): Name to use for a specific test Run. By default, runs are named
            with the date and time they're requested in format: `%Y%m%d-%H%M`
        run_description (str | None): A natural-language description for the test Run.
        timeout (int | float): The maximum time (in seconds) to wait for each response from the
            endpoint.
        callbacks (list[Callback] | None): Optional callbacks to enable during the test Run. See
            `llmeter.callbacks` for more information.
        low_memory (bool): When ``True``, responses are written to disk but not
            kept in memory.  Stats are computed incrementally via
            :class:`~llmeter.utils.RunningStats`.  Requires ``output_path``.
            Use ``result.load_responses()`` to access responses after the run.

            Example::

                result = await runner.run(
                    output_path="/tmp/my_run",
                    low_memory=True,
                )
                result.stats          # works (computed incrementally)
                result.responses      # [] (empty)
                result.load_responses()  # loads from disk

        progress_bar_stats (dict): Controls which live stats appear on the
            progress bar.  Maps short display labels to canonical stat keys — see
            :data:`~llmeter.live_display.DEFAULT_DISPLAY_STATS` for the format and
            defaults.  Pass ``{}`` to disable live stats entirely.

            Example::

                # Show only p99 latency and tokens per second:
                result = await runner.run(
                    progress_bar_stats={
                        "p99_ttlt": "time_to_last_token-p99",
                        "tps": ("time_per_output_token-p50", "inv"),
                        "fail": "failed_requests",
                    },
                )
        disable_per_client_progress_bar (bool): Set `True` to disable per-client progress bars
            from showing during the run.
        disable_clients_progress_bar (bool): Set `True` to disable overall progress bar from
            showing during the run.

    Returns:
        Result: An object containing the test results, including the generated
        response texts, total test time, total requests, number of clients,
        number of requests per client, and other relevant metrics.

    Raises:
        Exception: If there's an error during the test execution or if the
            endpoint cannot be reached.

    Note:
        - This method uses asyncio for concurrent processing.
        - Progress is displayed using tqdm if not disabled.
        - Responses are collected and processed asynchronously.
        - If an output_path is provided, results are saved to files.
    """

    run = self._prepare_run(
        endpoint=endpoint,
        output_path=output_path,
        tokenizer=tokenizer,
        clients=clients,
        n_requests=n_requests,
        run_duration=run_duration,
        payload=payload,
        run_name=run_name,
        run_description=run_description,
        timeout=timeout,
        callbacks=callbacks,
        low_memory=low_memory,
        progress_bar_stats=progress_bar_stats,
        disable_per_client_progress_bar=disable_per_client_progress_bar,
        disable_clients_progress_bar=disable_clients_progress_bar,
    )
    assert isinstance(run.payload, list)
    assert isinstance(run.run_name, str)
    return await run._run()