Runner

def add_callback(self, callback: Callback):
    """
    Add a callback to the runner's list of callbacks.

    Args:
        callback (Callback): The callback to be added.
    """
    if self.callbacks is None:
        self.callbacks = [callback]
    else:
        self.callbacks.append(callback)

async def run(
    self,
    *,  # Prevent mistakes with this long arg list by allowing only keyword-arg based passing
    # Explicitly name and re-document the args for ease of use of this important public method
    endpoint: Endpoint | dict | None = None,
    output_path: Path | None = None,
    tokenizer: Tokenizer | Any | None = None,
    clients: int | None = None,
    n_requests: int | None = None,
    payload: dict | list[dict] | os.PathLike | str | None = None,
    run_name: str | None = None,
    run_description: str | None = None,
    timeout: int | float | None = None,
    callbacks: list[Callback] | None = None,
    disable_per_client_progress_bar: bool | None = None,
    disable_clients_progress_bar: bool | None = None,
) -> Result:
    """
    Run a test against an LLM endpoint

    This method tests the performance of the endpoint by sending multiple concurrent requests
    with the given payload(s). It measures the total time taken to complete the test, generates
    invocations for the given payload(s), and optionally saves the results and metrics.

    For arguments that are not specified, the Runner's attributes will be used by default.

    Args:
        endpoint (Endpoint | dict | None): The LLM endpoint to be tested. **Must be set** at
            either the Runner or specific Run level.
        output_path (os.PathLike | str | None): The (cloud or local) base folder under which
            run outputs and configurations should be stored. By default, a new `run_name`
            sub-folder will be created under the Runner's `output_path` if set - otherwise
            outputs will not be saved to file.
        tokenizer (Tokenizer | Any | None): Optional tokenizer used to estimate input and
            output token counts for endpoints that don't report exact information.
        clients (int): The number of concurrent clients to use for sending requests.
        n_requests (int | None): The number of LLM invocations to generate *per client*.
        payload (dict | list[dict] | os.PathLike | str | None): The request data to send to the
            endpoint under test. You can provide a single JSON payload (dict), a list of
            payloads (list[dict]), or a path to one or more JSON/JSON-Lines files to be loaded
            by `llmeter.prompt_utils.load_payloads()`. **Must be set** at either the Runner or
            specific Run level.
        run_name (str | None): Name to use for a specific test Run. By default, runs are named
            with the date and time they're requested in format: `%Y%m%d-%H%M`
        run_description (str | None): A natural-language description for the test Run.
        timeout (int | float): The maximum time (in seconds) to wait for each response from the
            endpoint.
        callbacks (list[Callback] | None): Optional callbacks to enable during the test Run. See
            `llmeter.callbacks` for more information.
        disable_per_client_progress_bar (bool): Set `True` to disable per-client progress bars
            from showing during the run.
        disable_clients_progress_bar (bool): Set `True` to disable overall progress bar from
            showing during the run.

    Returns:
        Result: An object containing the test results, including the generated
        response texts, total test time, total requests, number of clients,
        number of requests per client, and other relevant metrics.

    Raises:
        Exception: If there's an error during the test execution or if the
            endpoint cannot be reached.

    Note:
        - This method uses asyncio for concurrent processing.
        - Progress is displayed using tqdm if not disabled.
        - Responses are collected and processed asynchronously.
        - If an output_path is provided, results are saved to files.
    """

    run = self._prepare_run(
        endpoint=endpoint,
        output_path=output_path,
        tokenizer=tokenizer,
        clients=clients,
        n_requests=n_requests,
        payload=payload,
        run_name=run_name,
        run_description=run_description,
        timeout=timeout,
        callbacks=callbacks,
        disable_per_client_progress_bar=disable_per_client_progress_bar,
        disable_clients_progress_bar=disable_clients_progress_bar,
    )
    assert isinstance(run.payload, list)
    assert isinstance(run.run_name, str)
    return await run._run()