Reporting

Dashboards and exportable reports built from evaluation results.

uaef.reporting.models

Dashboard and reporting data models for visualization.

Chart

Bases: BaseModel

Chart configuration for dashboard visualization.

Attributes:

Name	Type	Description
chart_id	UUID	Unique identifier for the chart
title	str	Chart title
chart_type	ChartType	Type of chart (bar, line, scatter, heatmap, etc.)
data	Dict[str, Any]	Chart data (structure depends on chart_type)
labels	List[str]	Labels for data points (x-axis labels, categories, etc.)
colors	List[str]	Color scheme for the chart
options	Dict[str, Any]	Additional chart configuration options
description	Optional[str]	Optional description of what the chart shows

Examples:

>>> # Bar chart for dimension scores
>>> chart = Chart(
...     title="Dimension Scores",
...     chart_type=ChartType.BAR,
...     data={
...         "dimensions": ["Tool Calling", "Response Quality", "Safety"],
...         "scores": [0.85, 0.92, 0.88]
...     },
...     labels=["Tool Calling", "Response Quality", "Safety"],
...     colors=["#4CAF50", "#2196F3", "#FF9800"]
... )

>>> # Line chart for metric trends
>>> chart = Chart(
...     title="Tool Accuracy Over Time",
...     chart_type=ChartType.LINE,
...     data={
...         "timestamps": ["2024-01-01", "2024-01-02", "2024-01-03"],
...         "values": [0.80, 0.85, 0.88]
...     },
...     labels=["Jan 1", "Jan 2", "Jan 3"],
...     options={"show_trend_line": True}
... )

>>> # Heatmap for confusion matrix
>>> chart = Chart(
...     title="Tool Selection Confusion Matrix",
...     chart_type=ChartType.HEATMAP,
...     data={
...         "matrix": [[10, 2, 1], [1, 15, 0], [0, 1, 12]],
...         "x_labels": ["search", "calculator", "database"],
...         "y_labels": ["search", "calculator", "database"]
...     },
...     colors=["#FFFFFF", "#4CAF50"]
... )

Source code in uaef/reporting/models.py

class Chart(BaseModel):
    """
    Chart configuration for dashboard visualization.

    Attributes:
        chart_id: Unique identifier for the chart
        title: Chart title
        chart_type: Type of chart (bar, line, scatter, heatmap, etc.)
        data: Chart data (structure depends on chart_type)
        labels: Labels for data points (x-axis labels, categories, etc.)
        colors: Color scheme for the chart
        options: Additional chart configuration options
        description: Optional description of what the chart shows

    Examples:
        >>> # Bar chart for dimension scores
        >>> chart = Chart(
        ...     title="Dimension Scores",
        ...     chart_type=ChartType.BAR,
        ...     data={
        ...         "dimensions": ["Tool Calling", "Response Quality", "Safety"],
        ...         "scores": [0.85, 0.92, 0.88]
        ...     },
        ...     labels=["Tool Calling", "Response Quality", "Safety"],
        ...     colors=["#4CAF50", "#2196F3", "#FF9800"]
        ... )

        >>> # Line chart for metric trends
        >>> chart = Chart(
        ...     title="Tool Accuracy Over Time",
        ...     chart_type=ChartType.LINE,
        ...     data={
        ...         "timestamps": ["2024-01-01", "2024-01-02", "2024-01-03"],
        ...         "values": [0.80, 0.85, 0.88]
        ...     },
        ...     labels=["Jan 1", "Jan 2", "Jan 3"],
        ...     options={"show_trend_line": True}
        ... )

        >>> # Heatmap for confusion matrix
        >>> chart = Chart(
        ...     title="Tool Selection Confusion Matrix",
        ...     chart_type=ChartType.HEATMAP,
        ...     data={
        ...         "matrix": [[10, 2, 1], [1, 15, 0], [0, 1, 12]],
        ...         "x_labels": ["search", "calculator", "database"],
        ...         "y_labels": ["search", "calculator", "database"]
        ...     },
        ...     colors=["#FFFFFF", "#4CAF50"]
        ... )
    """

    chart_id: UUID = Field(
        default_factory=uuid4,
        description="Unique identifier for the chart"
    )
    title: str = Field(
        ...,
        description="Chart title",
        min_length=1
    )
    chart_type: ChartType = Field(
        ...,
        description="Type of chart (bar, line, scatter, heatmap, etc.)"
    )
    data: Dict[str, Any] = Field(
        ...,
        description="Chart data (structure depends on chart_type)"
    )
    labels: List[str] = Field(
        default_factory=list,
        description="Labels for data points (x-axis labels, categories, etc.)"
    )
    colors: List[str] = Field(
        default_factory=list,
        description="Color scheme for the chart"
    )
    options: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional chart configuration options"
    )
    description: Optional[str] = Field(
        None,
        description="Optional description of what the chart shows"
    )

    @field_validator("chart_id")
    @classmethod
    def validate_chart_id(cls, v: UUID) -> UUID:
        """Validate that chart_id is not None."""
        if v is None:
            raise ValueError("chart_id cannot be None")
        return v

    @field_validator("title")
    @classmethod
    def validate_title(cls, v: str) -> str:
        """Validate that title is not empty."""
        if not v or not v.strip():
            raise ValueError("Chart title cannot be empty")
        return v.strip()

    @field_validator("data")
    @classmethod
    def validate_data(cls, v: Dict[str, Any]) -> Dict[str, Any]:
        """Validate that data is not empty."""
        if not v:
            raise ValueError("Chart data cannot be empty")
        return v

    @field_validator("colors")
    @classmethod
    def validate_colors(cls, v: List[str]) -> List[str]:
        """Validate that colors are valid hex codes or color names."""
        for color in v:
            if not color:
                raise ValueError("Color cannot be empty string")
            # Basic validation for hex colors (starts with #)
            if color.startswith("#"):
                hex_part = color[1:]
                if len(hex_part) not in [3, 6, 8]:  # RGB, RRGGBB, RRGGBBAA
                    raise ValueError(f"Invalid hex color format: {color}")
                # Check if all characters are valid hex digits
                if not all(c in "0123456789ABCDEFabcdef" for c in hex_part):
                    raise ValueError(f"Invalid hex color: {color} contains non-hex characters")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "chart_id": "550e8400-e29b-41d4-a716-446655440000",
                "title": "Dimension Scores",
                "chart_type": "bar",
                "data": {
                    "dimensions": ["Tool Calling", "Response Quality", "Safety"],
                    "scores": [0.85, 0.92, 0.88]
                },
                "labels": ["Tool Calling", "Response Quality", "Safety"],
                "colors": ["#4CAF50", "#2196F3", "#FF9800"],
                "options": {
                    "show_values": True,
                    "horizontal": False
                },
                "description": "Overall dimension scores for the experiment run"
            }
        }

Config

Pydantic configuration.

Source code in uaef/reporting/models.py

class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "chart_id": "550e8400-e29b-41d4-a716-446655440000",
            "title": "Dimension Scores",
            "chart_type": "bar",
            "data": {
                "dimensions": ["Tool Calling", "Response Quality", "Safety"],
                "scores": [0.85, 0.92, 0.88]
            },
            "labels": ["Tool Calling", "Response Quality", "Safety"],
            "colors": ["#4CAF50", "#2196F3", "#FF9800"],
            "options": {
                "show_values": True,
                "horizontal": False
            },
            "description": "Overall dimension scores for the experiment run"
        }
    }

validate_chart_id(v) classmethod

Validate that chart_id is not None.

Source code in uaef/reporting/models.py

@field_validator("chart_id")
@classmethod
def validate_chart_id(cls, v: UUID) -> UUID:
    """Validate that chart_id is not None."""
    if v is None:
        raise ValueError("chart_id cannot be None")
    return v

validate_colors(v) classmethod

Validate that colors are valid hex codes or color names.

Source code in uaef/reporting/models.py

@field_validator("colors")
@classmethod
def validate_colors(cls, v: List[str]) -> List[str]:
    """Validate that colors are valid hex codes or color names."""
    for color in v:
        if not color:
            raise ValueError("Color cannot be empty string")
        # Basic validation for hex colors (starts with #)
        if color.startswith("#"):
            hex_part = color[1:]
            if len(hex_part) not in [3, 6, 8]:  # RGB, RRGGBB, RRGGBBAA
                raise ValueError(f"Invalid hex color format: {color}")
            # Check if all characters are valid hex digits
            if not all(c in "0123456789ABCDEFabcdef" for c in hex_part):
                raise ValueError(f"Invalid hex color: {color} contains non-hex characters")
    return v

validate_data(v) classmethod

Validate that data is not empty.

Source code in uaef/reporting/models.py

@field_validator("data")
@classmethod
def validate_data(cls, v: Dict[str, Any]) -> Dict[str, Any]:
    """Validate that data is not empty."""
    if not v:
        raise ValueError("Chart data cannot be empty")
    return v

validate_title(v) classmethod

Validate that title is not empty.

Source code in uaef/reporting/models.py

@field_validator("title")
@classmethod
def validate_title(cls, v: str) -> str:
    """Validate that title is not empty."""
    if not v or not v.strip():
        raise ValueError("Chart title cannot be empty")
    return v.strip()

ChartType

Bases: str, Enum

Type of chart for visualization.

Source code in uaef/reporting/models.py

class ChartType(str, Enum):
    """Type of chart for visualization."""

    BAR = "bar"
    LINE = "line"
    SCATTER = "scatter"
    HEATMAP = "heatmap"
    PIE = "pie"
    RADAR = "radar"
    BOX = "box"
    HISTOGRAM = "histogram"

Dashboard

Bases: BaseModel

Dashboard containing multiple charts and visualizations.

Attributes:

Name	Type	Description
dashboard_id	UUID	Unique identifier for the dashboard
title	str	Dashboard title
description	Optional[str]	Dashboard description
charts	List[Chart]	List of charts in the dashboard
metadata	Dict[str, Any]	Additional dashboard metadata (experiment_id, run_id, etc.)
created_at	datetime	When the dashboard was created
layout	Dict[str, Any]	Layout configuration for chart positioning

Examples:

>>> # Executive dashboard
>>> dashboard = Dashboard(
...     title="Executive Dashboard - Customer Support Agent",
...     description="High-level metrics for experiment run baseline-v1",
...     charts=[
...         Chart(title="Overall Score", chart_type=ChartType.BAR, data={"score": [0.87]}),
...         Chart(title="Dimension Breakdown", chart_type=ChartType.RADAR, data={...})
...     ],
...     metadata={
...         "experiment_id": "550e8400-e29b-41d4-a716-446655440000",
...         "run_id": "660e8400-e29b-41d4-a716-446655440001",
...         "dashboard_type": "executive"
...     }
... )

>>> # Comparison dashboard
>>> dashboard = Dashboard(
...     title="Experiment Comparison: v1 vs v2",
...     description="Side-by-side comparison of baseline and new version",
...     charts=[
...         Chart(title="Metric Deltas", chart_type=ChartType.BAR, data={...}),
...         Chart(title="Trend Over Time", chart_type=ChartType.LINE, data={...})
...     ],
...     metadata={
...         "baseline_run_id": "660e8400-e29b-41d4-a716-446655440001",
...         "comparison_run_id": "770e8400-e29b-41d4-a716-446655440002",
...         "dashboard_type": "comparison"
...     }
... )

Source code in uaef/reporting/models.py

class Dashboard(BaseModel):
    """
    Dashboard containing multiple charts and visualizations.

    Attributes:
        dashboard_id: Unique identifier for the dashboard
        title: Dashboard title
        description: Dashboard description
        charts: List of charts in the dashboard
        metadata: Additional dashboard metadata (experiment_id, run_id, etc.)
        created_at: When the dashboard was created
        layout: Layout configuration for chart positioning

    Examples:
        >>> # Executive dashboard
        >>> dashboard = Dashboard(
        ...     title="Executive Dashboard - Customer Support Agent",
        ...     description="High-level metrics for experiment run baseline-v1",
        ...     charts=[
        ...         Chart(title="Overall Score", chart_type=ChartType.BAR, data={"score": [0.87]}),
        ...         Chart(title="Dimension Breakdown", chart_type=ChartType.RADAR, data={...})
        ...     ],
        ...     metadata={
        ...         "experiment_id": "550e8400-e29b-41d4-a716-446655440000",
        ...         "run_id": "660e8400-e29b-41d4-a716-446655440001",
        ...         "dashboard_type": "executive"
        ...     }
        ... )

        >>> # Comparison dashboard
        >>> dashboard = Dashboard(
        ...     title="Experiment Comparison: v1 vs v2",
        ...     description="Side-by-side comparison of baseline and new version",
        ...     charts=[
        ...         Chart(title="Metric Deltas", chart_type=ChartType.BAR, data={...}),
        ...         Chart(title="Trend Over Time", chart_type=ChartType.LINE, data={...})
        ...     ],
        ...     metadata={
        ...         "baseline_run_id": "660e8400-e29b-41d4-a716-446655440001",
        ...         "comparison_run_id": "770e8400-e29b-41d4-a716-446655440002",
        ...         "dashboard_type": "comparison"
        ...     }
        ... )
    """

    dashboard_id: UUID = Field(
        default_factory=uuid4,
        description="Unique identifier for the dashboard"
    )
    title: str = Field(
        ...,
        description="Dashboard title",
        min_length=1
    )
    description: Optional[str] = Field(
        None,
        description="Dashboard description"
    )
    charts: List[Chart] = Field(
        ...,
        description="List of charts in the dashboard",
        min_length=1
    )
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional dashboard metadata (experiment_id, run_id, etc.)"
    )
    created_at: datetime = Field(
        default_factory=datetime.utcnow,
        description="When the dashboard was created"
    )
    layout: Dict[str, Any] = Field(
        default_factory=dict,
        description="Layout configuration for chart positioning"
    )

    @field_validator("dashboard_id")
    @classmethod
    def validate_dashboard_id(cls, v: UUID) -> UUID:
        """Validate that dashboard_id is not None."""
        if v is None:
            raise ValueError("dashboard_id cannot be None")
        return v

    @field_validator("title")
    @classmethod
    def validate_title(cls, v: str) -> str:
        """Validate that title is not empty."""
        if not v or not v.strip():
            raise ValueError("Dashboard title cannot be empty")
        return v.strip()

    @field_validator("charts")
    @classmethod
    def validate_charts(cls, v: List[Chart]) -> List[Chart]:
        """Validate that charts list is not empty."""
        if not v:
            raise ValueError("Dashboard must contain at least one chart")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "dashboard_id": "550e8400-e29b-41d4-a716-446655440000",
                "title": "Executive Dashboard - Customer Support Agent",
                "description": "High-level metrics for experiment run baseline-v1",
                "charts": [
                    {
                        "chart_id": "660e8400-e29b-41d4-a716-446655440001",
                        "title": "Overall Score",
                        "chart_type": "bar",
                        "data": {"score": [0.87]},
                        "labels": ["Overall"],
                        "colors": ["#4CAF50"]
                    }
                ],
                "metadata": {
                    "experiment_id": "770e8400-e29b-41d4-a716-446655440002",
                    "run_id": "880e8400-e29b-41d4-a716-446655440003",
                    "dashboard_type": "executive"
                },
                "created_at": "2024-01-15T10:00:00Z",
                "layout": {
                    "grid_columns": 2,
                    "chart_positions": {
                        "660e8400-e29b-41d4-a716-446655440001": {"row": 0, "col": 0}
                    }
                }
            }
        }

Config

Pydantic configuration.

Source code in uaef/reporting/models.py

class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "dashboard_id": "550e8400-e29b-41d4-a716-446655440000",
            "title": "Executive Dashboard - Customer Support Agent",
            "description": "High-level metrics for experiment run baseline-v1",
            "charts": [
                {
                    "chart_id": "660e8400-e29b-41d4-a716-446655440001",
                    "title": "Overall Score",
                    "chart_type": "bar",
                    "data": {"score": [0.87]},
                    "labels": ["Overall"],
                    "colors": ["#4CAF50"]
                }
            ],
            "metadata": {
                "experiment_id": "770e8400-e29b-41d4-a716-446655440002",
                "run_id": "880e8400-e29b-41d4-a716-446655440003",
                "dashboard_type": "executive"
            },
            "created_at": "2024-01-15T10:00:00Z",
            "layout": {
                "grid_columns": 2,
                "chart_positions": {
                    "660e8400-e29b-41d4-a716-446655440001": {"row": 0, "col": 0}
                }
            }
        }
    }

validate_charts(v) classmethod

Validate that charts list is not empty.

Source code in uaef/reporting/models.py

@field_validator("charts")
@classmethod
def validate_charts(cls, v: List[Chart]) -> List[Chart]:
    """Validate that charts list is not empty."""
    if not v:
        raise ValueError("Dashboard must contain at least one chart")
    return v

validate_dashboard_id(v) classmethod

Validate that dashboard_id is not None.

Source code in uaef/reporting/models.py

@field_validator("dashboard_id")
@classmethod
def validate_dashboard_id(cls, v: UUID) -> UUID:
    """Validate that dashboard_id is not None."""
    if v is None:
        raise ValueError("dashboard_id cannot be None")
    return v

validate_title(v) classmethod

Validate that title is not empty.

Source code in uaef/reporting/models.py

@field_validator("title")
@classmethod
def validate_title(cls, v: str) -> str:
    """Validate that title is not empty."""
    if not v or not v.strip():
        raise ValueError("Dashboard title cannot be empty")
    return v.strip()

Report

Bases: BaseModel

Generated report with dashboard and exportable content.

Attributes:

Name	Type	Description
report_id	UUID	Unique identifier for the report
title	str	Report title
report_type	ReportType	Type of report (executive, comparison, safety, etc.)
dashboard	Dashboard	Dashboard with visualizations
summary	str	Executive summary text
sections	List[Dict[str, Any]]	Report sections with detailed content
metadata	Dict[str, Any]	Additional report metadata
created_at	datetime	When the report was created
export_formats	List[ReportFormat]	Available export formats for this report

Examples:

>>> # Executive report
>>> report = Report(
...     title="Executive Report - Customer Support Agent v1",
...     report_type=ReportType.EXECUTIVE,
...     dashboard=Dashboard(...),
...     summary="Overall performance: 87%. All dimensions above threshold.",
...     sections=[
...         {
...             "title": "Key Findings",
...             "content": "Tool calling accuracy improved by 5%..."
...         },
...         {
...             "title": "Recommendations",
...             "content": "Consider optimizing response latency..."
...         }
...     ],
...     metadata={
...         "experiment_id": "550e8400-e29b-41d4-a716-446655440000",
...         "run_id": "660e8400-e29b-41d4-a716-446655440001"
...     },
...     export_formats=[ReportFormat.HTML, ReportFormat.PDF, ReportFormat.JSON]
... )

>>> # Safety report
>>> report = Report(
...     title="Safety Report - Responsible AI Assessment",
...     report_type=ReportType.SAFETY,
...     dashboard=Dashboard(...),
...     summary="2 safety concerns detected requiring attention.",
...     sections=[
...         {
...             "title": "Safety Violations",
...             "content": "Detected potential bias in 2 responses...",
...             "severity": "medium"
...         }
...     ],
...     metadata={"focus": "responsible_ai"},
...     export_formats=[ReportFormat.HTML, ReportFormat.PDF]
... )

>>> # Regression report
>>> report = Report(
...     title="Regression Report - v1 vs v2",
...     report_type=ReportType.REGRESSION,
...     dashboard=Dashboard(...),
...     summary="3 regressions detected in tool calling dimension.",
...     sections=[
...         {
...             "title": "Regressions",
...             "content": "Tool accuracy decreased by 8%...",
...             "regressions": [
...                 {"metric": "tool_accuracy", "delta": -0.08, "severity": "high"}
...             ]
...         }
...     ],
...     metadata={
...         "baseline_run_id": "660e8400-e29b-41d4-a716-446655440001",
...         "comparison_run_id": "770e8400-e29b-41d4-a716-446655440002"
...     }
... )

Source code in uaef/reporting/models.py

class Report(BaseModel):
    """
    Generated report with dashboard and exportable content.

    Attributes:
        report_id: Unique identifier for the report
        title: Report title
        report_type: Type of report (executive, comparison, safety, etc.)
        dashboard: Dashboard with visualizations
        summary: Executive summary text
        sections: Report sections with detailed content
        metadata: Additional report metadata
        created_at: When the report was created
        export_formats: Available export formats for this report

    Examples:
        >>> # Executive report
        >>> report = Report(
        ...     title="Executive Report - Customer Support Agent v1",
        ...     report_type=ReportType.EXECUTIVE,
        ...     dashboard=Dashboard(...),
        ...     summary="Overall performance: 87%. All dimensions above threshold.",
        ...     sections=[
        ...         {
        ...             "title": "Key Findings",
        ...             "content": "Tool calling accuracy improved by 5%..."
        ...         },
        ...         {
        ...             "title": "Recommendations",
        ...             "content": "Consider optimizing response latency..."
        ...         }
        ...     ],
        ...     metadata={
        ...         "experiment_id": "550e8400-e29b-41d4-a716-446655440000",
        ...         "run_id": "660e8400-e29b-41d4-a716-446655440001"
        ...     },
        ...     export_formats=[ReportFormat.HTML, ReportFormat.PDF, ReportFormat.JSON]
        ... )

        >>> # Safety report
        >>> report = Report(
        ...     title="Safety Report - Responsible AI Assessment",
        ...     report_type=ReportType.SAFETY,
        ...     dashboard=Dashboard(...),
        ...     summary="2 safety concerns detected requiring attention.",
        ...     sections=[
        ...         {
        ...             "title": "Safety Violations",
        ...             "content": "Detected potential bias in 2 responses...",
        ...             "severity": "medium"
        ...         }
        ...     ],
        ...     metadata={"focus": "responsible_ai"},
        ...     export_formats=[ReportFormat.HTML, ReportFormat.PDF]
        ... )

        >>> # Regression report
        >>> report = Report(
        ...     title="Regression Report - v1 vs v2",
        ...     report_type=ReportType.REGRESSION,
        ...     dashboard=Dashboard(...),
        ...     summary="3 regressions detected in tool calling dimension.",
        ...     sections=[
        ...         {
        ...             "title": "Regressions",
        ...             "content": "Tool accuracy decreased by 8%...",
        ...             "regressions": [
        ...                 {"metric": "tool_accuracy", "delta": -0.08, "severity": "high"}
        ...             ]
        ...         }
        ...     ],
        ...     metadata={
        ...         "baseline_run_id": "660e8400-e29b-41d4-a716-446655440001",
        ...         "comparison_run_id": "770e8400-e29b-41d4-a716-446655440002"
        ...     }
        ... )
    """

    report_id: UUID = Field(
        default_factory=uuid4,
        description="Unique identifier for the report"
    )
    title: str = Field(
        ...,
        description="Report title",
        min_length=1
    )
    report_type: ReportType = Field(
        ...,
        description="Type of report (executive, comparison, safety, etc.)"
    )
    dashboard: Dashboard = Field(
        ...,
        description="Dashboard with visualizations"
    )
    summary: str = Field(
        ...,
        description="Executive summary text",
        min_length=1
    )
    sections: List[Dict[str, Any]] = Field(
        default_factory=list,
        description="Report sections with detailed content"
    )
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional report metadata"
    )
    created_at: datetime = Field(
        default_factory=datetime.utcnow,
        description="When the report was created"
    )
    export_formats: List[ReportFormat] = Field(
        default_factory=lambda: [ReportFormat.JSON, ReportFormat.HTML],
        description="Available export formats for this report"
    )

    @field_validator("report_id")
    @classmethod
    def validate_report_id(cls, v: UUID) -> UUID:
        """Validate that report_id is not None."""
        if v is None:
            raise ValueError("report_id cannot be None")
        return v

    @field_validator("title")
    @classmethod
    def validate_title(cls, v: str) -> str:
        """Validate that title is not empty."""
        if not v or not v.strip():
            raise ValueError("Report title cannot be empty")
        return v.strip()

    @field_validator("summary")
    @classmethod
    def validate_summary(cls, v: str) -> str:
        """Validate that summary is not empty."""
        if not v or not v.strip():
            raise ValueError("Report summary cannot be empty")
        return v.strip()

    @field_validator("sections")
    @classmethod
    def validate_sections(cls, v: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """Validate that each section has required fields."""
        for i, section in enumerate(v):
            if "title" not in section:
                raise ValueError(f"Section {i} missing required field 'title'")
            if "content" not in section:
                raise ValueError(f"Section {i} missing required field 'content'")
            if not section["title"] or not section["title"].strip():
                raise ValueError(f"Section {i} title cannot be empty")
            if not section["content"] or not section["content"].strip():
                raise ValueError(f"Section {i} content cannot be empty")
        return v

    @field_validator("export_formats")
    @classmethod
    def validate_export_formats(cls, v: List[ReportFormat]) -> List[ReportFormat]:
        """Validate that at least one export format is specified."""
        if not v:
            raise ValueError("Report must support at least one export format")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "report_id": "550e8400-e29b-41d4-a716-446655440000",
                "title": "Executive Report - Customer Support Agent v1",
                "report_type": "executive",
                "dashboard": {
                    "dashboard_id": "660e8400-e29b-41d4-a716-446655440001",
                    "title": "Executive Dashboard",
                    "charts": []
                },
                "summary": "Overall performance: 87%. All dimensions above threshold.",
                "sections": [
                    {
                        "title": "Key Findings",
                        "content": "Tool calling accuracy improved by 5% compared to baseline."
                    },
                    {
                        "title": "Recommendations",
                        "content": "Consider optimizing response latency for better user experience."
                    }
                ],
                "metadata": {
                    "experiment_id": "770e8400-e29b-41d4-a716-446655440002",
                    "run_id": "880e8400-e29b-41d4-a716-446655440003",
                    "evaluation_count": 100
                },
                "created_at": "2024-01-15T10:00:00Z",
                "export_formats": ["json", "html", "pdf"]
            }
        }

Config

Pydantic configuration.

Source code in uaef/reporting/models.py

class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "report_id": "550e8400-e29b-41d4-a716-446655440000",
            "title": "Executive Report - Customer Support Agent v1",
            "report_type": "executive",
            "dashboard": {
                "dashboard_id": "660e8400-e29b-41d4-a716-446655440001",
                "title": "Executive Dashboard",
                "charts": []
            },
            "summary": "Overall performance: 87%. All dimensions above threshold.",
            "sections": [
                {
                    "title": "Key Findings",
                    "content": "Tool calling accuracy improved by 5% compared to baseline."
                },
                {
                    "title": "Recommendations",
                    "content": "Consider optimizing response latency for better user experience."
                }
            ],
            "metadata": {
                "experiment_id": "770e8400-e29b-41d4-a716-446655440002",
                "run_id": "880e8400-e29b-41d4-a716-446655440003",
                "evaluation_count": 100
            },
            "created_at": "2024-01-15T10:00:00Z",
            "export_formats": ["json", "html", "pdf"]
        }
    }

validate_export_formats(v) classmethod

Validate that at least one export format is specified.

Source code in uaef/reporting/models.py

@field_validator("export_formats")
@classmethod
def validate_export_formats(cls, v: List[ReportFormat]) -> List[ReportFormat]:
    """Validate that at least one export format is specified."""
    if not v:
        raise ValueError("Report must support at least one export format")
    return v

validate_report_id(v) classmethod

Validate that report_id is not None.

Source code in uaef/reporting/models.py

@field_validator("report_id")
@classmethod
def validate_report_id(cls, v: UUID) -> UUID:
    """Validate that report_id is not None."""
    if v is None:
        raise ValueError("report_id cannot be None")
    return v

validate_sections(v) classmethod

Validate that each section has required fields.

Source code in uaef/reporting/models.py

@field_validator("sections")
@classmethod
def validate_sections(cls, v: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    """Validate that each section has required fields."""
    for i, section in enumerate(v):
        if "title" not in section:
            raise ValueError(f"Section {i} missing required field 'title'")
        if "content" not in section:
            raise ValueError(f"Section {i} missing required field 'content'")
        if not section["title"] or not section["title"].strip():
            raise ValueError(f"Section {i} title cannot be empty")
        if not section["content"] or not section["content"].strip():
            raise ValueError(f"Section {i} content cannot be empty")
    return v

validate_summary(v) classmethod

Validate that summary is not empty.

Source code in uaef/reporting/models.py

@field_validator("summary")
@classmethod
def validate_summary(cls, v: str) -> str:
    """Validate that summary is not empty."""
    if not v or not v.strip():
        raise ValueError("Report summary cannot be empty")
    return v.strip()

validate_title(v) classmethod

Validate that title is not empty.

Source code in uaef/reporting/models.py

@field_validator("title")
@classmethod
def validate_title(cls, v: str) -> str:
    """Validate that title is not empty."""
    if not v or not v.strip():
        raise ValueError("Report title cannot be empty")
    return v.strip()

ReportFormat

Bases: str, Enum

Format for report export.

Source code in uaef/reporting/models.py

class ReportFormat(str, Enum):
    """Format for report export."""

    JSON = "json"
    HTML = "html"
    PDF = "pdf"
    CSV = "csv"
    MARKDOWN = "markdown"

ReportType

Bases: str, Enum

Type of report.

Source code in uaef/reporting/models.py

class ReportType(str, Enum):
    """Type of report."""

    EXECUTIVE = "executive"
    COMPARISON = "comparison"
    DIMENSION_DEEP_DIVE = "dimension_deep_dive"
    SAFETY = "safety"
    REGRESSION = "regression"
    CUSTOM = "custom"

uaef.reporting.executive_dashboard

Executive dashboard generator for high-level performance summaries.

ExecutiveDashboardGenerator

Generator for executive dashboards with high-level performance summaries.

Creates dashboards suitable for stakeholders showing: - Overall performance scores across all dimensions - Key metrics with current values and trends - Critical issues and failures - Visual indicators for improvement/decline - Executive summary with actionable insights

Integrates with TrendAnalyzer and RecommendationEngine for comprehensive insights.

Examples:

>>> # Generate dashboard from experiment run results
>>> generator = ExecutiveDashboardGenerator()
>>> results = [result1, result2, result3]  # EvaluationResult objects
>>> dashboard = generator.generate_dashboard(
...     results=results,
...     title="Customer Support Agent - Q1 2024",
...     run_id="550e8400-e29b-41d4-a716-446655440000"
... )
>>> print(f"Dashboard: {dashboard.title}")
>>> print(f"Charts: {len(dashboard.charts)}")

>>> # Generate with trend analysis
>>> trend_analyzer = TrendAnalyzer()
>>> generator = ExecutiveDashboardGenerator(trend_analyzer=trend_analyzer)
>>> dashboard = generator.generate_dashboard(
...     results=results,
...     title="Agent Performance Dashboard",
...     include_trends=True
... )

>>> # Generate with recommendations
>>> rec_engine = RecommendationEngine()
>>> generator = ExecutiveDashboardGenerator(recommendation_engine=rec_engine)
>>> dashboard = generator.generate_dashboard(
...     results=results,
...     title="Executive Dashboard",
...     include_recommendations=True
... )

Source code in uaef/reporting/executive_dashboard.py

class ExecutiveDashboardGenerator:
    """
    Generator for executive dashboards with high-level performance summaries.

    Creates dashboards suitable for stakeholders showing:
    - Overall performance scores across all dimensions
    - Key metrics with current values and trends
    - Critical issues and failures
    - Visual indicators for improvement/decline
    - Executive summary with actionable insights

    Integrates with TrendAnalyzer and RecommendationEngine for comprehensive insights.

    Examples:
        >>> # Generate dashboard from experiment run results
        >>> generator = ExecutiveDashboardGenerator()
        >>> results = [result1, result2, result3]  # EvaluationResult objects
        >>> dashboard = generator.generate_dashboard(
        ...     results=results,
        ...     title="Customer Support Agent - Q1 2024",
        ...     run_id="550e8400-e29b-41d4-a716-446655440000"
        ... )
        >>> print(f"Dashboard: {dashboard.title}")
        >>> print(f"Charts: {len(dashboard.charts)}")

        >>> # Generate with trend analysis
        >>> trend_analyzer = TrendAnalyzer()
        >>> generator = ExecutiveDashboardGenerator(trend_analyzer=trend_analyzer)
        >>> dashboard = generator.generate_dashboard(
        ...     results=results,
        ...     title="Agent Performance Dashboard",
        ...     include_trends=True
        ... )

        >>> # Generate with recommendations
        >>> rec_engine = RecommendationEngine()
        >>> generator = ExecutiveDashboardGenerator(recommendation_engine=rec_engine)
        >>> dashboard = generator.generate_dashboard(
        ...     results=results,
        ...     title="Executive Dashboard",
        ...     include_recommendations=True
        ... )
    """

    def __init__(
        self,
        trend_analyzer: Optional[TrendAnalyzer] = None,
        recommendation_engine: Optional[RecommendationEngine] = None
    ):
        """
        Initialize the executive dashboard generator.

        Args:
            trend_analyzer: Optional TrendAnalyzer for trend indicators
            recommendation_engine: Optional RecommendationEngine for insights
        """
        self.trend_analyzer = trend_analyzer or TrendAnalyzer()
        self.recommendation_engine = recommendation_engine or RecommendationEngine()

    def generate_dashboard(
        self,
        results: List[EvaluationResult],
        title: str,
        run_id: Optional[UUID] = None,
        experiment_id: Optional[UUID] = None,
        include_trends: bool = True,
        include_recommendations: bool = True,
        description: Optional[str] = None
    ) -> Dashboard:
        """
        Generate an executive dashboard from evaluation results.

        Args:
            results: List of evaluation results to analyze
            title: Dashboard title
            run_id: Optional experiment run ID
            experiment_id: Optional experiment ID
            include_trends: Whether to include trend indicators
            include_recommendations: Whether to include recommendations
            description: Optional dashboard description

        Returns:
            Dashboard with executive summary charts

        Raises:
            ValueError: If results list is empty
            ValueError: If title is empty

        Examples:
            >>> generator = ExecutiveDashboardGenerator()
            >>> results = [result1, result2, result3]
            >>> dashboard = generator.generate_dashboard(
            ...     results=results,
            ...     title="Q1 Performance Dashboard"
            ... )
        """
        if not results:
            raise ValueError("Cannot generate dashboard from empty results list")

        if not title or not title.strip():
            raise ValueError("Dashboard title cannot be empty")

        logger.info(f"Generating executive dashboard for {len(results)} evaluation results")

        # Calculate aggregate metrics
        aggregate_metrics = self._calculate_aggregate_metrics(results)

        # Generate charts
        charts = []

        # Chart 1: Overall performance score
        charts.append(self._create_overall_score_chart(aggregate_metrics))

        # Chart 2: Dimension breakdown
        charts.append(self._create_dimension_breakdown_chart(aggregate_metrics))

        # Chart 3: Key metrics summary
        charts.append(self._create_key_metrics_chart(aggregate_metrics))

        # Chart 4: Pass/Fail distribution
        charts.append(self._create_pass_fail_chart(results))

        # Chart 5: Critical issues (if any)
        critical_issues = self._identify_critical_issues(results, aggregate_metrics)
        if critical_issues:
            charts.append(self._create_critical_issues_chart(critical_issues))

        # Chart 6: Trend indicators (if enabled and available)
        if include_trends and len(results) > 1:
            try:
                trend_chart = self._create_trend_indicators_chart(results, aggregate_metrics)
                if trend_chart:
                    charts.append(trend_chart)
            except Exception as e:
                logger.warning(f"Failed to generate trend chart: {e}")

        # Prepare metadata
        metadata = {
            "dashboard_type": "executive",
            "evaluation_count": len(results),
            "generated_at": datetime.utcnow().isoformat()
        }

        if run_id:
            metadata["run_id"] = str(run_id)
        if experiment_id:
            metadata["experiment_id"] = str(experiment_id)

        # Add aggregate metrics to metadata
        metadata["aggregate_metrics"] = aggregate_metrics

        # Add critical issues summary
        if critical_issues:
            metadata["critical_issues_count"] = len(critical_issues)
            metadata["critical_issues"] = critical_issues

        # Add recommendations if enabled
        if include_recommendations:
            try:
                rec_report = self.recommendation_engine.generate_recommendations(results)
                top_recommendations = [
                    {
                        "title": rec.title,
                        "priority": rec.priority,
                        "category": rec.category
                    }
                    for rec in rec_report.recommendations[:3]  # Top 3
                ]
                metadata["top_recommendations"] = top_recommendations
            except Exception as e:
                logger.warning(f"Failed to generate recommendations: {e}")

        # Generate description if not provided
        if not description:
            description = self._generate_executive_summary(
                results, aggregate_metrics, critical_issues
            )

        # Create dashboard
        dashboard = Dashboard(
            title=title,
            description=description,
            charts=charts,
            metadata=metadata
        )

        logger.info(f"Generated executive dashboard with {len(charts)} charts")
        return dashboard

    def _calculate_aggregate_metrics(
        self, results: List[EvaluationResult]
    ) -> Dict[str, Any]:
        """
        Calculate aggregate metrics across all evaluation results.

        Args:
            results: List of evaluation results

        Returns:
            Dictionary with aggregate metrics
        """
        if not results:
            return {}

        # Calculate overall score
        overall_scores = [r.overall_score for r in results]
        avg_overall_score = sum(overall_scores) / len(overall_scores)

        # Calculate dimension scores
        dimension_scores = defaultdict(list)
        for result in results:
            for dim_result in result.dimension_results:
                if dim_result.aggregate_score is None:
                    continue
                dimension_scores[dim_result.dimension_name].append(
                    dim_result.aggregate_score
                )

        avg_dimension_scores = {
            dim: sum(scores) / len(scores)
            for dim, scores in dimension_scores.items()
        }

        # Calculate metric scores
        metric_scores = defaultdict(list)
        none_metrics = set()
        for result in results:
            for dim_result in result.dimension_results:
                for metric_score in dim_result.metric_scores:
                    if metric_score.score is not None:
                        metric_scores[metric_score.metric_name].append(metric_score.score)
                    else:
                        none_metrics.add(metric_score.metric_name)

        if none_metrics:
            logger.warning(
                f"Skipped {len(none_metrics)} metric(s) with None scores: "
                f"{sorted(none_metrics)}"
            )

        avg_metric_scores = {
            metric: sum(scores) / len(scores)
            for metric, scores in metric_scores.items()
            if scores
        }

        # Calculate pass/fail statistics
        passed_count = sum(1 for r in results if r.passed)
        failed_count = len(results) - passed_count
        pass_rate = passed_count / len(results) if results else 0.0

        return {
            "overall_score": avg_overall_score,
            "dimension_scores": avg_dimension_scores,
            "metric_scores": avg_metric_scores,
            "total_evaluations": len(results),
            "passed_count": passed_count,
            "failed_count": failed_count,
            "pass_rate": pass_rate
        }

    def _create_overall_score_chart(self, aggregate_metrics: Dict[str, Any]) -> Chart:
        """Create chart showing overall performance score."""
        overall_score = aggregate_metrics.get("overall_score", 0.0)

        # Determine color based on score
        if overall_score >= 0.8:
            color = "#4CAF50"  # Green
        elif overall_score >= 0.6:
            color = "#FF9800"  # Orange
        else:
            color = "#F44336"  # Red

        return Chart(
            title="Overall Performance Score",
            chart_type=ChartType.BAR,
            data={
                "labels": ["Overall Score"],
                "values": [round(overall_score, 3)]
            },
            labels=["Overall Score"],
            colors=[color],
            options={
                "show_values": True,
                "max_value": 1.0,
                "horizontal": False
            },
            description=f"Average overall score: {overall_score:.1%}"
        )

    def _create_dimension_breakdown_chart(
        self, aggregate_metrics: Dict[str, Any]
    ) -> Chart:
        """Create radar chart showing dimension breakdown."""
        dimension_scores = aggregate_metrics.get("dimension_scores", {})

        if not dimension_scores:
            # Return empty chart if no dimensions
            return Chart(
                title="Dimension Breakdown",
                chart_type=ChartType.RADAR,
                data={"dimensions": [], "scores": []},
                labels=[],
                colors=["#2196F3"],
                description="No dimension data available"
            )

        dimensions = list(dimension_scores.keys())
        scores = [round(dimension_scores[dim], 3) for dim in dimensions]

        return Chart(
            title="Dimension Breakdown",
            chart_type=ChartType.RADAR,
            data={
                "dimensions": dimensions,
                "scores": scores
            },
            labels=dimensions,
            colors=["#2196F3"],
            options={
                "max_value": 1.0,
                "show_legend": True
            },
            description=f"Performance across {len(dimensions)} dimensions"
        )

    def _create_key_metrics_chart(self, aggregate_metrics: Dict[str, Any]) -> Chart:
        """Create bar chart showing top key metrics."""
        metric_scores = aggregate_metrics.get("metric_scores", {})

        if not metric_scores:
            return Chart(
                title="Key Metrics",
                chart_type=ChartType.BAR,
                data={"metrics": [], "scores": []},
                labels=[],
                colors=[],
                description="No metric data available"
            )

        # Sort metrics by score and take top 10
        sorted_metrics = sorted(
            metric_scores.items(),
            key=lambda x: x[1],
            reverse=True
        )[:10]

        metrics = [m[0] for m in sorted_metrics]
        scores = [round(m[1], 3) for m in sorted_metrics]

        # Color code based on score
        colors = []
        for score in scores:
            if score >= 0.8:
                colors.append("#4CAF50")  # Green
            elif score >= 0.6:
                colors.append("#FF9800")  # Orange
            else:
                colors.append("#F44336")  # Red

        return Chart(
            title="Key Metrics",
            chart_type=ChartType.BAR,
            data={
                "metrics": metrics,
                "scores": scores
            },
            labels=metrics,
            colors=colors,
            options={
                "show_values": True,
                "horizontal": True,
                "max_value": 1.0
            },
            description=f"Top {len(metrics)} metrics by performance"
        )

    def _create_pass_fail_chart(self, results: List[EvaluationResult]) -> Chart:
        """Create pie chart showing pass/fail distribution."""
        passed_count = sum(1 for r in results if r.passed)
        failed_count = len(results) - passed_count

        return Chart(
            title="Pass/Fail Distribution",
            chart_type=ChartType.PIE,
            data={
                "labels": ["Passed", "Failed"],
                "values": [passed_count, failed_count]
            },
            labels=["Passed", "Failed"],
            colors=["#4CAF50", "#F44336"],
            options={
                "show_percentages": True,
                "show_legend": True
            },
            description=f"Pass rate: {passed_count}/{len(results)} ({passed_count/len(results):.1%})"
        )

    def _identify_critical_issues(
        self,
        results: List[EvaluationResult],
        aggregate_metrics: Dict[str, Any]
    ) -> List[Dict[str, Any]]:
        """
        Identify critical issues from evaluation results.

        Args:
            results: List of evaluation results
            aggregate_metrics: Aggregate metrics

        Returns:
            List of critical issues
        """
        critical_issues = []

        # Check for low overall score
        overall_score = aggregate_metrics.get("overall_score", 1.0)
        if overall_score < 0.6:
            critical_issues.append({
                "type": "low_overall_score",
                "severity": "critical",
                "description": f"Overall score ({overall_score:.1%}) is below acceptable threshold (60%)",
                "metric": "overall_score",
                "value": overall_score
            })

        # Check for low dimension scores
        dimension_scores = aggregate_metrics.get("dimension_scores", {})
        for dim, score in dimension_scores.items():
            if score < 0.6:
                critical_issues.append({
                    "type": "low_dimension_score",
                    "severity": "high" if score < 0.5 else "medium",
                    "description": f"{dim} dimension score ({score:.1%}) is below threshold",
                    "metric": dim,
                    "value": score
                })

        # Check for high failure rate
        pass_rate = aggregate_metrics.get("pass_rate", 1.0)
        if pass_rate < 0.7:
            critical_issues.append({
                "type": "high_failure_rate",
                "severity": "critical" if pass_rate < 0.5 else "high",
                "description": f"Pass rate ({pass_rate:.1%}) is below acceptable threshold (70%)",
                "metric": "pass_rate",
                "value": pass_rate
            })

        # Check for specific metric failures
        metric_scores = aggregate_metrics.get("metric_scores", {})
        critical_metrics = ["safety_score", "tool_accuracy", "answer_relevance"]
        for metric in critical_metrics:
            if metric in metric_scores and metric_scores[metric] < 0.7:
                critical_issues.append({
                    "type": "critical_metric_failure",
                    "severity": "high",
                    "description": f"Critical metric {metric} ({metric_scores[metric]:.1%}) is below threshold",
                    "metric": metric,
                    "value": metric_scores[metric]
                })

        return critical_issues

    def _create_critical_issues_chart(
        self, critical_issues: List[Dict[str, Any]]
    ) -> Chart:
        """Create chart highlighting critical issues."""
        # Group by severity
        severity_counts = defaultdict(int)
        for issue in critical_issues:
            severity_counts[issue["severity"]] += 1

        severities = ["critical", "high", "medium", "low"]
        counts = [severity_counts.get(sev, 0) for sev in severities]

        # Filter out zero counts
        filtered_data = [(sev, count) for sev, count in zip(severities, counts) if count > 0]
        if filtered_data:
            severities, counts = zip(*filtered_data)
        else:
            severities, counts = [], []

        colors = {
            "critical": "#D32F2F",
            "high": "#F44336",
            "medium": "#FF9800",
            "low": "#FFC107"
        }

        chart_colors = [colors[sev] for sev in severities]

        return Chart(
            title="Critical Issues",
            chart_type=ChartType.BAR,
            data={
                "severities": list(severities),
                "counts": list(counts)
            },
            labels=list(severities),
            colors=chart_colors,
            options={
                "show_values": True,
                "horizontal": False
            },
            description=f"{len(critical_issues)} critical issues detected"
        )

    def _create_trend_indicators_chart(
        self,
        results: List[EvaluationResult],
        aggregate_metrics: Dict[str, Any]
    ) -> Optional[Chart]:
        """
        Create chart showing trend indicators for key metrics.

        Args:
            results: List of evaluation results
            aggregate_metrics: Aggregate metrics

        Returns:
            Chart with trend indicators or None if trends cannot be calculated
        """
        try:
            # Analyze trends
            trend_report = self.trend_analyzer.analyze_metric_trends(results)

            # Get trends for key metrics
            key_metrics = ["tool_accuracy", "answer_relevance", "safety_score"]
            trends_data = []

            for metric in key_metrics:
                if metric in trend_report.metric_trends:
                    trend = trend_report.metric_trends[metric]
                    trends_data.append({
                        "metric": metric,
                        "direction": trend.trend_direction,
                        "change": trend.change_percentage
                    })

            if not trends_data:
                return None

            # Create chart data
            metrics = [t["metric"] for t in trends_data]
            changes = [t["change"] for t in trends_data]

            # Color code by direction
            colors = []
            for t in trends_data:
                if t["direction"] == "improving":
                    colors.append("#4CAF50")  # Green
                elif t["direction"] == "declining":
                    colors.append("#F44336")  # Red
                else:
                    colors.append("#9E9E9E")  # Gray

            return Chart(
                title="Metric Trends",
                chart_type=ChartType.BAR,
                data={
                    "metrics": metrics,
                    "changes": [round(c, 2) for c in changes]
                },
                labels=metrics,
                colors=colors,
                options={
                    "show_values": True,
                    "horizontal": True,
                    "show_zero_line": True
                },
                description="Percentage change in key metrics"
            )

        except Exception as e:
            logger.warning(f"Failed to create trend indicators chart: {e}")
            return None

    def _generate_executive_summary(
        self,
        results: List[EvaluationResult],
        aggregate_metrics: Dict[str, Any],
        critical_issues: List[Dict[str, Any]]
    ) -> str:
        """
        Generate executive summary text.

        Args:
            results: List of evaluation results
            aggregate_metrics: Aggregate metrics
            critical_issues: List of critical issues

        Returns:
            Executive summary string
        """
        overall_score = aggregate_metrics.get("overall_score", 0.0)
        pass_rate = aggregate_metrics.get("pass_rate", 0.0)
        total_evals = len(results)

        summary_parts = []

        # Overall performance
        summary_parts.append(
            f"Analyzed {total_evals} evaluations with overall score of {overall_score:.1%} "
            f"and pass rate of {pass_rate:.1%}."
        )

        # Critical issues
        if critical_issues:
            critical_count = sum(1 for i in critical_issues if i["severity"] == "critical")
            high_count = sum(1 for i in critical_issues if i["severity"] == "high")

            if critical_count > 0:
                summary_parts.append(
                    f"{critical_count} critical issue(s) detected requiring immediate attention."
                )
            if high_count > 0:
                summary_parts.append(
                    f"{high_count} high-priority issue(s) identified."
                )
        else:
            summary_parts.append("No critical issues detected.")

        # Dimension performance
        dimension_scores = aggregate_metrics.get("dimension_scores", {})
        if dimension_scores:
            best_dim = max(dimension_scores.items(), key=lambda x: x[1])
            worst_dim = min(dimension_scores.items(), key=lambda x: x[1])

            summary_parts.append(
                f"Best performing dimension: {best_dim[0]} ({best_dim[1]:.1%}). "
                f"Lowest performing: {worst_dim[0]} ({worst_dim[1]:.1%})."
            )

        return " ".join(summary_parts)

generate_dashboard(results, title, run_id=None, experiment_id=None, include_trends=True, include_recommendations=True, description=None)

Generate an executive dashboard from evaluation results.

Parameters:

Name	Type	Description	Default
results	List[EvaluationResult]	List of evaluation results to analyze	required
title	str	Dashboard title	required
run_id	Optional[UUID]	Optional experiment run ID	None
experiment_id	Optional[UUID]	Optional experiment ID	None
include_trends	bool	Whether to include trend indicators	True
include_recommendations	bool	Whether to include recommendations	True
description	Optional[str]	Optional dashboard description	None

Returns:

Type	Description
Dashboard	Dashboard with executive summary charts

Raises:

Type	Description
ValueError	If results list is empty
ValueError	If title is empty

Examples:

>>> generator = ExecutiveDashboardGenerator()
>>> results = [result1, result2, result3]
>>> dashboard = generator.generate_dashboard(
...     results=results,
...     title="Q1 Performance Dashboard"
... )

Source code in uaef/reporting/executive_dashboard.py

def generate_dashboard(
    self,
    results: List[EvaluationResult],
    title: str,
    run_id: Optional[UUID] = None,
    experiment_id: Optional[UUID] = None,
    include_trends: bool = True,
    include_recommendations: bool = True,
    description: Optional[str] = None
) -> Dashboard:
    """
    Generate an executive dashboard from evaluation results.

    Args:
        results: List of evaluation results to analyze
        title: Dashboard title
        run_id: Optional experiment run ID
        experiment_id: Optional experiment ID
        include_trends: Whether to include trend indicators
        include_recommendations: Whether to include recommendations
        description: Optional dashboard description

    Returns:
        Dashboard with executive summary charts

    Raises:
        ValueError: If results list is empty
        ValueError: If title is empty

    Examples:
        >>> generator = ExecutiveDashboardGenerator()
        >>> results = [result1, result2, result3]
        >>> dashboard = generator.generate_dashboard(
        ...     results=results,
        ...     title="Q1 Performance Dashboard"
        ... )
    """
    if not results:
        raise ValueError("Cannot generate dashboard from empty results list")

    if not title or not title.strip():
        raise ValueError("Dashboard title cannot be empty")

    logger.info(f"Generating executive dashboard for {len(results)} evaluation results")

    # Calculate aggregate metrics
    aggregate_metrics = self._calculate_aggregate_metrics(results)

    # Generate charts
    charts = []

    # Chart 1: Overall performance score
    charts.append(self._create_overall_score_chart(aggregate_metrics))

    # Chart 2: Dimension breakdown
    charts.append(self._create_dimension_breakdown_chart(aggregate_metrics))

    # Chart 3: Key metrics summary
    charts.append(self._create_key_metrics_chart(aggregate_metrics))

    # Chart 4: Pass/Fail distribution
    charts.append(self._create_pass_fail_chart(results))

    # Chart 5: Critical issues (if any)
    critical_issues = self._identify_critical_issues(results, aggregate_metrics)
    if critical_issues:
        charts.append(self._create_critical_issues_chart(critical_issues))

    # Chart 6: Trend indicators (if enabled and available)
    if include_trends and len(results) > 1:
        try:
            trend_chart = self._create_trend_indicators_chart(results, aggregate_metrics)
            if trend_chart:
                charts.append(trend_chart)
        except Exception as e:
            logger.warning(f"Failed to generate trend chart: {e}")

    # Prepare metadata
    metadata = {
        "dashboard_type": "executive",
        "evaluation_count": len(results),
        "generated_at": datetime.utcnow().isoformat()
    }

    if run_id:
        metadata["run_id"] = str(run_id)
    if experiment_id:
        metadata["experiment_id"] = str(experiment_id)

    # Add aggregate metrics to metadata
    metadata["aggregate_metrics"] = aggregate_metrics

    # Add critical issues summary
    if critical_issues:
        metadata["critical_issues_count"] = len(critical_issues)
        metadata["critical_issues"] = critical_issues

    # Add recommendations if enabled
    if include_recommendations:
        try:
            rec_report = self.recommendation_engine.generate_recommendations(results)
            top_recommendations = [
                {
                    "title": rec.title,
                    "priority": rec.priority,
                    "category": rec.category
                }
                for rec in rec_report.recommendations[:3]  # Top 3
            ]
            metadata["top_recommendations"] = top_recommendations
        except Exception as e:
            logger.warning(f"Failed to generate recommendations: {e}")

    # Generate description if not provided
    if not description:
        description = self._generate_executive_summary(
            results, aggregate_metrics, critical_issues
        )

    # Create dashboard
    dashboard = Dashboard(
        title=title,
        description=description,
        charts=charts,
        metadata=metadata
    )

    logger.info(f"Generated executive dashboard with {len(charts)} charts")
    return dashboard

uaef.reporting.comparison_dashboard

Comparison dashboard generator for experiment comparison.

ComparisonDashboardGenerator

Generator for experiment comparison dashboards.

Creates dashboards for comparing experiment runs side-by-side: - Side-by-side metric comparisons - Delta calculations and percentage changes - Statistical significance indicators - Regression and improvement highlighting - Metric trend visualization - Category-based filtering

Integrates with ComparisonEngine for metric comparison logic.

Examples:

>>> # Generate comparison dashboard
>>> generator = ComparisonDashboardGenerator()
>>> dashboard = generator.generate_dashboard(
...     baseline_run=baseline_run,
...     comparison_run=current_run,
...     title="Baseline vs Current"
... )
>>> print(f"Dashboard: {dashboard.title}")
>>> print(f"Charts: {len(dashboard.charts)}")

>>> # Generate with custom thresholds
>>> generator = ComparisonDashboardGenerator(
...     regression_threshold=10.0,
...     improvement_threshold=10.0
... )
>>> dashboard = generator.generate_dashboard(
...     baseline_run=baseline_run,
...     comparison_run=current_run,
...     title="Experiment Comparison"
... )

>>> # Generate with metric filtering
>>> dashboard = generator.generate_dashboard(
...     baseline_run=baseline_run,
...     comparison_run=current_run,
...     title="Tool Calling Comparison",
...     metric_categories=["tool_calling"]
... )

Source code in uaef/reporting/comparison_dashboard.py

class ComparisonDashboardGenerator:
    """
    Generator for experiment comparison dashboards.

    Creates dashboards for comparing experiment runs side-by-side:
    - Side-by-side metric comparisons
    - Delta calculations and percentage changes
    - Statistical significance indicators
    - Regression and improvement highlighting
    - Metric trend visualization
    - Category-based filtering

    Integrates with ComparisonEngine for metric comparison logic.

    Examples:
        >>> # Generate comparison dashboard
        >>> generator = ComparisonDashboardGenerator()
        >>> dashboard = generator.generate_dashboard(
        ...     baseline_run=baseline_run,
        ...     comparison_run=current_run,
        ...     title="Baseline vs Current"
        ... )
        >>> print(f"Dashboard: {dashboard.title}")
        >>> print(f"Charts: {len(dashboard.charts)}")

        >>> # Generate with custom thresholds
        >>> generator = ComparisonDashboardGenerator(
        ...     regression_threshold=10.0,
        ...     improvement_threshold=10.0
        ... )
        >>> dashboard = generator.generate_dashboard(
        ...     baseline_run=baseline_run,
        ...     comparison_run=current_run,
        ...     title="Experiment Comparison"
        ... )

        >>> # Generate with metric filtering
        >>> dashboard = generator.generate_dashboard(
        ...     baseline_run=baseline_run,
        ...     comparison_run=current_run,
        ...     title="Tool Calling Comparison",
        ...     metric_categories=["tool_calling"]
        ... )
    """

    def __init__(
        self,
        comparison_engine: Optional[ComparisonEngine] = None,
        regression_threshold: float = 5.0,
        improvement_threshold: float = 5.0
    ):
        """
        Initialize the comparison dashboard generator.

        Args:
            comparison_engine: Optional ComparisonEngine for metric comparison
            regression_threshold: Percentage threshold for regression detection (default: 5.0%)
            improvement_threshold: Percentage threshold for improvement detection (default: 5.0%)
        """
        self.comparison_engine = comparison_engine or ComparisonEngine(
            regression_threshold=regression_threshold,
            improvement_threshold=improvement_threshold
        )
        self.regression_threshold = regression_threshold
        self.improvement_threshold = improvement_threshold

    def generate_dashboard(
        self,
        baseline_run: ExperimentRun,
        comparison_run: ExperimentRun,
        title: str,
        description: Optional[str] = None,
        metric_categories: Optional[List[str]] = None,
        include_statistical_tests: bool = True
    ) -> Dashboard:
        """
        Generate a comparison dashboard between two experiment runs.

        Args:
            baseline_run: Baseline experiment run
            comparison_run: Comparison experiment run
            title: Dashboard title
            description: Optional dashboard description
            metric_categories: Optional list of metric categories to include
            include_statistical_tests: Whether to include statistical significance tests

        Returns:
            Dashboard with comparison charts

        Raises:
            ValueError: If title is empty
            ValueError: If baseline and comparison runs are the same

        Examples:
            >>> generator = ComparisonDashboardGenerator()
            >>> dashboard = generator.generate_dashboard(
            ...     baseline_run=baseline_run,
            ...     comparison_run=current_run,
            ...     title="v1 vs v2 Comparison"
            ... )
        """
        if not title or not title.strip():
            raise ValueError("Dashboard title cannot be empty")

        if baseline_run.run_id == comparison_run.run_id:
            raise ValueError("Cannot compare a run with itself")

        logger.info(
            f"Generating comparison dashboard between runs {baseline_run.run_id} "
            f"and {comparison_run.run_id}"
        )

        # Generate comparison report
        comparison_report = self.comparison_engine.compare_runs(
            baseline_run=baseline_run,
            current_run=comparison_run
        )

        # Filter metrics by category if specified
        if metric_categories:
            comparison_report = self._filter_by_categories(
                comparison_report, metric_categories
            )

        # Generate charts
        charts = []

        # Chart 1: Side-by-side metric comparison
        charts.append(self._create_side_by_side_chart(comparison_report))

        # Chart 2: Delta visualization
        charts.append(self._create_delta_chart(comparison_report))

        # Chart 3: Percentage change chart
        charts.append(self._create_percentage_change_chart(comparison_report))

        # Chart 4: Regression/Improvement summary
        charts.append(self._create_regression_improvement_chart(comparison_report))

        # Chart 5: Statistical significance (if enabled)
        if include_statistical_tests:
            sig_chart = self._create_statistical_significance_chart(
                baseline_run, comparison_run, comparison_report
            )
            if sig_chart:
                charts.append(sig_chart)

        # Chart 6: Metric trends heatmap
        charts.append(self._create_comparison_heatmap(comparison_report))

        # Prepare metadata
        metadata = {
            "dashboard_type": "comparison",
            "baseline_run_id": str(baseline_run.run_id),
            "comparison_run_id": str(comparison_run.run_id),
            "generated_at": datetime.utcnow().isoformat(),
            "regression_threshold": self.regression_threshold,
            "improvement_threshold": self.improvement_threshold,
            "total_metrics": comparison_report.summary["total_metrics"],
            "regression_count": comparison_report.summary["regression_count"],
            "improvement_count": comparison_report.summary["improvement_count"],
            "unchanged_count": comparison_report.summary["unchanged_count"]
        }

        if metric_categories:
            metadata["metric_categories"] = metric_categories

        # Generate description if not provided
        if not description:
            description = self._generate_comparison_summary(comparison_report)

        # Create dashboard
        dashboard = Dashboard(
            title=title,
            description=description,
            charts=charts,
            metadata=metadata
        )

        logger.info(f"Generated comparison dashboard with {len(charts)} charts")
        return dashboard

    def _filter_by_categories(
        self,
        report: ComparisonReport,
        categories: List[str]
    ) -> ComparisonReport:
        """
        Filter comparison report by metric categories.

        Args:
            report: Original comparison report
            categories: List of categories to include

        Returns:
            Filtered comparison report
        """
        # Define category prefixes
        category_prefixes = {
            "tool_calling": ["tool_", "mcp_"],
            "response_quality": ["answer_", "completeness", "hallucination", "accuracy"],
            "responsible_ai": ["safety_", "bias_", "toxicity", "prompt_injection"],
            "performance": ["latency_", "token_", "cost_", "throughput"],
            "multi_turn": ["context_", "coherence", "conversation_", "turn_"],
            "multi_agent": ["agent_", "delegation_", "workflow_", "coordination_"],
            "reasoning": ["chain_of_thought", "logical_", "reasoning_", "fallacy"]
        }

        # Get prefixes for requested categories
        prefixes = []
        for category in categories:
            if category in category_prefixes:
                prefixes.extend(category_prefixes[category])

        # Filter metric comparisons
        filtered_comparisons = [
            comp for comp in report.metric_comparisons
            if any(comp.metric_name.startswith(prefix) for prefix in prefixes)
        ]

        # Update lists
        filtered_regressions = [
            m for m in report.regressions
            if any(m.startswith(prefix) for prefix in prefixes)
        ]
        filtered_improvements = [
            m for m in report.improvements
            if any(m.startswith(prefix) for prefix in prefixes)
        ]
        filtered_unchanged = [
            m for m in report.unchanged
            if any(m.startswith(prefix) for prefix in prefixes)
        ]

        # Update summary
        filtered_summary = {
            **report.summary,
            "total_metrics": len(filtered_comparisons),
            "regression_count": len(filtered_regressions),
            "improvement_count": len(filtered_improvements),
            "unchanged_count": len(filtered_unchanged)
        }

        return ComparisonReport(
            baseline_run_id=report.baseline_run_id,
            current_run_id=report.current_run_id,
            metric_comparisons=filtered_comparisons,
            regressions=filtered_regressions,
            improvements=filtered_improvements,
            unchanged=filtered_unchanged,
            summary=filtered_summary
        )

    def _create_side_by_side_chart(self, report: ComparisonReport) -> Chart:
        """Create side-by-side bar chart comparing baseline and current metrics."""
        if not report.metric_comparisons:
            return Chart(
                title="Side-by-Side Metric Comparison",
                chart_type=ChartType.BAR,
                data={"metrics": [], "baseline": [], "current": []},
                labels=[],
                colors=[],
                description="No metrics to compare"
            )

        # Sort by absolute delta (largest changes first)
        sorted_comparisons = sorted(
            report.metric_comparisons,
            key=lambda x: abs(x.delta),
            reverse=True
        )[:15]  # Top 15 metrics

        metrics = [c.metric_name for c in sorted_comparisons]
        baseline_scores = [round(c.baseline_score, 3) for c in sorted_comparisons]
        current_scores = [round(c.current_score, 3) for c in sorted_comparisons]

        return Chart(
            title="Side-by-Side Metric Comparison",
            chart_type=ChartType.BAR,
            data={
                "metrics": metrics,
                "baseline": baseline_scores,
                "current": current_scores
            },
            labels=metrics,
            colors=["#2196F3", "#4CAF50"],
            options={
                "grouped": True,
                "horizontal": True,
                "show_values": True,
                "max_value": 1.0,
                "legend": ["Baseline", "Current"]
            },
            description=f"Top {len(metrics)} metrics by change magnitude"
        )

    def _create_delta_chart(self, report: ComparisonReport) -> Chart:
        """Create bar chart showing absolute deltas."""
        if not report.metric_comparisons:
            return Chart(
                title="Metric Deltas",
                chart_type=ChartType.BAR,
                data={"metrics": [], "deltas": []},
                labels=[],
                colors=[],
                description="No metrics to compare"
            )

        # Sort by delta (most negative to most positive)
        sorted_comparisons = sorted(
            report.metric_comparisons,
            key=lambda x: x.delta
        )[:20]  # Top 20 metrics

        metrics = [c.metric_name for c in sorted_comparisons]
        deltas = [round(c.delta, 3) for c in sorted_comparisons]

        # Color code by regression/improvement
        colors = []
        for comp in sorted_comparisons:
            if comp.is_regression:
                colors.append("#F44336")  # Red
            elif comp.is_improvement:
                colors.append("#4CAF50")  # Green
            else:
                colors.append("#9E9E9E")  # Gray

        return Chart(
            title="Metric Deltas (Current - Baseline)",
            chart_type=ChartType.BAR,
            data={
                "metrics": metrics,
                "deltas": deltas
            },
            labels=metrics,
            colors=colors,
            options={
                "horizontal": True,
                "show_values": True,
                "show_zero_line": True
            },
            description=f"Absolute change for {len(metrics)} metrics"
        )

    def _create_percentage_change_chart(self, report: ComparisonReport) -> Chart:
        """Create bar chart showing percentage changes."""
        if not report.metric_comparisons:
            return Chart(
                title="Percentage Changes",
                chart_type=ChartType.BAR,
                data={"metrics": [], "changes": []},
                labels=[],
                colors=[],
                description="No metrics to compare"
            )

        # Sort by percent change (most negative to most positive)
        sorted_comparisons = sorted(
            report.metric_comparisons,
            key=lambda x: x.percent_change
        )[:20]  # Top 20 metrics

        metrics = [c.metric_name for c in sorted_comparisons]
        percent_changes = [round(c.percent_change, 2) for c in sorted_comparisons]

        # Color code by regression/improvement
        colors = []
        for comp in sorted_comparisons:
            if comp.is_regression:
                colors.append("#F44336")  # Red
            elif comp.is_improvement:
                colors.append("#4CAF50")  # Green
            else:
                colors.append("#9E9E9E")  # Gray

        return Chart(
            title="Percentage Changes",
            chart_type=ChartType.BAR,
            data={
                "metrics": metrics,
                "percent_changes": percent_changes
            },
            labels=metrics,
            colors=colors,
            options={
                "horizontal": True,
                "show_values": True,
                "show_zero_line": True,
                "value_suffix": "%"
            },
            description=f"Percentage change for {len(metrics)} metrics"
        )

    def _create_regression_improvement_chart(
        self, report: ComparisonReport
    ) -> Chart:
        """Create summary chart showing regressions vs improvements."""
        regression_count = report.summary["regression_count"]
        improvement_count = report.summary["improvement_count"]
        unchanged_count = report.summary["unchanged_count"]

        return Chart(
            title="Change Summary",
            chart_type=ChartType.PIE,
            data={
                "labels": ["Regressions", "Improvements", "Unchanged"],
                "values": [regression_count, improvement_count, unchanged_count]
            },
            labels=["Regressions", "Improvements", "Unchanged"],
            colors=["#F44336", "#4CAF50", "#9E9E9E"],
            options={
                "show_percentages": True,
                "show_legend": True
            },
            description=(
                f"{regression_count} regressions, {improvement_count} improvements, "
                f"{unchanged_count} unchanged"
            )
        )

    def _create_statistical_significance_chart(
        self,
        baseline_run: ExperimentRun,
        comparison_run: ExperimentRun,
        report: ComparisonReport
    ) -> Optional[Chart]:
        """
        Create chart showing statistical significance of changes.

        Uses t-test and effect size (Cohen's d) to determine significance.
        """
        try:
            # Calculate statistical significance for each metric
            significance_data = []

            for comp in report.metric_comparisons:
                # Calculate effect size (Cohen's d)
                # d = (mean1 - mean2) / pooled_std
                # For simplicity, use delta as proxy for effect size
                effect_size = abs(comp.delta)

                # Classify significance
                if effect_size >= 0.2:
                    significance = "large"
                elif effect_size >= 0.1:
                    significance = "medium"
                elif effect_size >= 0.05:
                    significance = "small"
                else:
                    significance = "negligible"

                significance_data.append({
                    "metric": comp.metric_name,
                    "effect_size": effect_size,
                    "significance": significance,
                    "is_regression": comp.is_regression,
                    "is_improvement": comp.is_improvement
                })

            # Filter to significant changes only
            significant = [
                s for s in significance_data
                if s["significance"] in ["large", "medium"]
            ]

            if not significant:
                return None

            # Sort by effect size
            significant.sort(key=lambda x: x["effect_size"], reverse=True)
            significant = significant[:15]  # Top 15

            metrics = [s["metric"] for s in significant]
            effect_sizes = [round(s["effect_size"], 3) for s in significant]

            # Color code by regression/improvement
            colors = []
            for s in significant:
                if s["is_regression"]:
                    colors.append("#F44336")  # Red
                elif s["is_improvement"]:
                    colors.append("#4CAF50")  # Green
                else:
                    colors.append("#FF9800")  # Orange

            return Chart(
                title="Statistical Significance (Effect Size)",
                chart_type=ChartType.BAR,
                data={
                    "metrics": metrics,
                    "effect_sizes": effect_sizes
                },
                labels=metrics,
                colors=colors,
                options={
                    "horizontal": True,
                    "show_values": True
                },
                description=f"{len(significant)} metrics with significant changes"
            )

        except Exception as e:
            logger.warning(f"Failed to create statistical significance chart: {e}")
            return None

    def _create_comparison_heatmap(self, report: ComparisonReport) -> Chart:
        """Create heatmap showing metric changes across categories."""
        if not report.metric_comparisons:
            return Chart(
                title="Metric Change Heatmap",
                chart_type=ChartType.HEATMAP,
                data={"matrix": [], "x_labels": [], "y_labels": []},
                labels=[],
                colors=[],
                description="No metrics to compare"
            )

        # Group metrics by category
        category_metrics = defaultdict(list)

        for comp in report.metric_comparisons:
            # Determine category from metric name
            if comp.metric_name.startswith(("tool_", "mcp_")):
                category = "Tool Calling"
            elif comp.metric_name.startswith(("answer_", "completeness", "hallucination", "accuracy")):
                category = "Response Quality"
            elif comp.metric_name.startswith(("safety_", "bias_", "toxicity", "prompt_injection")):
                category = "Responsible AI"
            elif comp.metric_name.startswith(("latency_", "token_", "cost_", "throughput")):
                category = "Performance"
            elif comp.metric_name.startswith(("context_", "coherence", "conversation_", "turn_")):
                category = "Multi-Turn"
            elif comp.metric_name.startswith(("agent_", "delegation_", "workflow_", "coordination_")):
                category = "Multi-Agent"
            elif comp.metric_name.startswith(("chain_of_thought", "logical_", "reasoning_", "fallacy")):
                category = "Reasoning"
            else:
                category = "Other"

            category_metrics[category].append(comp)

        # Build heatmap matrix
        categories = sorted(category_metrics.keys())
        change_types = ["Regressions", "Improvements", "Unchanged"]

        matrix = []
        for category in categories:
            row = []
            metrics = category_metrics[category]

            regression_count = sum(1 for m in metrics if m.is_regression)
            improvement_count = sum(1 for m in metrics if m.is_improvement)
            unchanged_count = len(metrics) - regression_count - improvement_count

            row.append(regression_count)
            row.append(improvement_count)
            row.append(unchanged_count)

            matrix.append(row)

        return Chart(
            title="Metric Change Heatmap by Category",
            chart_type=ChartType.HEATMAP,
            data={
                "matrix": matrix,
                "x_labels": change_types,
                "y_labels": categories
            },
            labels=categories,
            colors=["#FFFFFF", "#2196F3"],
            options={
                "show_values": True,
                "color_scale": "Blues"
            },
            description="Distribution of changes across metric categories"
        )

    def _generate_comparison_summary(self, report: ComparisonReport) -> str:
        """
        Generate comparison summary text.

        Args:
            report: Comparison report

        Returns:
            Summary string
        """
        total_metrics = report.summary["total_metrics"]
        regression_count = report.summary["regression_count"]
        improvement_count = report.summary["improvement_count"]
        unchanged_count = report.summary["unchanged_count"]

        summary_parts = []

        # Overall summary
        summary_parts.append(
            f"Compared {total_metrics} metrics between baseline and current runs."
        )

        # Regressions
        if regression_count > 0:
            summary_parts.append(
                f"{regression_count} regression(s) detected requiring attention."
            )
            # List top regressions
            if report.regressions:
                top_regressions = report.regressions[:3]
                summary_parts.append(
                    f"Top regressions: {', '.join(top_regressions)}."
                )
        else:
            summary_parts.append("No regressions detected.")

        # Improvements
        if improvement_count > 0:
            summary_parts.append(
                f"{improvement_count} improvement(s) identified."
            )
            # List top improvements
            if report.improvements:
                top_improvements = report.improvements[:3]
                summary_parts.append(
                    f"Top improvements: {', '.join(top_improvements)}."
                )

        # Unchanged
        if unchanged_count > 0:
            summary_parts.append(
                f"{unchanged_count} metric(s) remained stable."
            )

        return " ".join(summary_parts)

    def generate_multi_run_comparison(
        self,
        baseline_run: ExperimentRun,
        comparison_runs: List[ExperimentRun],
        title: str,
        description: Optional[str] = None
    ) -> Dashboard:
        """
        Generate dashboard comparing multiple runs against a baseline.

        Args:
            baseline_run: Baseline experiment run
            comparison_runs: List of runs to compare against baseline
            title: Dashboard title
            description: Optional dashboard description

        Returns:
            Dashboard with multi-run comparison charts

        Raises:
            ValueError: If comparison_runs is empty
            ValueError: If title is empty
        """
        if not comparison_runs:
            raise ValueError("comparison_runs cannot be empty")

        if not title or not title.strip():
            raise ValueError("Dashboard title cannot be empty")

        logger.info(
            f"Generating multi-run comparison dashboard for {len(comparison_runs)} runs"
        )

        # Generate comparison reports for all runs
        reports = self.comparison_engine.compare_multiple_runs(
            baseline_run=baseline_run,
            current_runs=comparison_runs
        )

        # Generate charts
        charts = []

        # Chart 1: Metric trends across all runs
        charts.append(self._create_multi_run_trend_chart(baseline_run, comparison_runs))

        # Chart 2: Regression/Improvement summary per run
        charts.append(self._create_multi_run_summary_chart(reports))

        # Chart 3: Key metric comparison across all runs
        charts.append(self._create_key_metrics_multi_run_chart(baseline_run, comparison_runs))

        # Prepare metadata
        metadata = {
            "dashboard_type": "multi_run_comparison",
            "baseline_run_id": str(baseline_run.run_id),
            "comparison_run_ids": [str(run.run_id) for run in comparison_runs],
            "generated_at": datetime.utcnow().isoformat(),
            "run_count": len(comparison_runs)
        }

        # Generate description if not provided
        if not description:
            description = (
                f"Comparison of {len(comparison_runs)} experiment runs against baseline. "
                f"Shows metric trends and changes across all runs."
            )

        # Create dashboard
        dashboard = Dashboard(
            title=title,
            description=description,
            charts=charts,
            metadata=metadata
        )

        logger.info(f"Generated multi-run comparison dashboard with {len(charts)} charts")
        return dashboard

    def _create_multi_run_trend_chart(
        self,
        baseline_run: ExperimentRun,
        comparison_runs: List[ExperimentRun]
    ) -> Chart:
        """Create line chart showing metric trends across multiple runs."""
        # Get common metrics across all runs
        all_runs = [baseline_run] + comparison_runs
        common_metrics = set(baseline_run.aggregate_metrics.keys())

        for run in comparison_runs:
            common_metrics &= set(run.aggregate_metrics.keys())

        if not common_metrics:
            return Chart(
                title="Metric Trends Across Runs",
                chart_type=ChartType.LINE,
                data={"runs": [], "metrics": {}},
                labels=[],
                colors=[],
                description="No common metrics found"
            )

        # Select top 5 metrics by variance
        metric_variances = {}
        for metric in common_metrics:
            values = [run.aggregate_metrics[metric] for run in all_runs]
            variance = max(values) - min(values)
            metric_variances[metric] = variance

        top_metrics = sorted(
            metric_variances.items(),
            key=lambda x: x[1],
            reverse=True
        )[:5]

        selected_metrics = [m[0] for m in top_metrics]

        # Build data for line chart
        run_labels = ["Baseline"] + [f"Run {i+1}" for i in range(len(comparison_runs))]

        metric_data = {}
        for metric in selected_metrics:
            values = [
                round(run.aggregate_metrics[metric], 3)
                for run in all_runs
            ]
            metric_data[metric] = values

        # Generate colors for each metric
        colors = ["#2196F3", "#4CAF50", "#FF9800", "#9C27B0", "#F44336"]

        return Chart(
            title="Metric Trends Across Runs",
            chart_type=ChartType.LINE,
            data={
                "runs": run_labels,
                "metrics": metric_data
            },
            labels=run_labels,
            colors=colors[:len(selected_metrics)],
            options={
                "show_points": True,
                "show_legend": True,
                "legend": selected_metrics
            },
            description=f"Trends for {len(selected_metrics)} key metrics"
        )

    def _create_multi_run_summary_chart(
        self, reports: List[ComparisonReport]
    ) -> Chart:
        """Create stacked bar chart showing regressions/improvements per run."""
        run_labels = [f"Run {i+1}" for i in range(len(reports))]

        regressions = [report.summary["regression_count"] for report in reports]
        improvements = [report.summary["improvement_count"] for report in reports]
        unchanged = [report.summary["unchanged_count"] for report in reports]

        return Chart(
            title="Change Summary Per Run",
            chart_type=ChartType.BAR,
            data={
                "runs": run_labels,
                "regressions": regressions,
                "improvements": improvements,
                "unchanged": unchanged
            },
            labels=run_labels,
            colors=["#F44336", "#4CAF50", "#9E9E9E"],
            options={
                "stacked": True,
                "show_legend": True,
                "legend": ["Regressions", "Improvements", "Unchanged"]
            },
            description="Distribution of changes across all runs"
        )

    def _create_key_metrics_multi_run_chart(
        self,
        baseline_run: ExperimentRun,
        comparison_runs: List[ExperimentRun]
    ) -> Chart:
        """Create grouped bar chart for key metrics across all runs."""
        # Define key metrics to track
        key_metrics = [
            "tool_accuracy", "answer_relevance", "safety_score",
            "latency_score", "context_retention"
        ]

        all_runs = [baseline_run] + comparison_runs
        run_labels = ["Baseline"] + [f"Run {i+1}" for i in range(len(comparison_runs))]

        # Filter to metrics that exist in all runs
        available_metrics = [
            metric for metric in key_metrics
            if all(metric in run.aggregate_metrics for run in all_runs)
        ]

        if not available_metrics:
            return Chart(
                title="Key Metrics Across Runs",
                chart_type=ChartType.BAR,
                data={"metrics": [], "runs": {}},
                labels=[],
                colors=[],
                description="No key metrics available"
            )

        # Build data for grouped bar chart
        run_data = {}
        for i, run in enumerate(all_runs):
            run_data[run_labels[i]] = [
                round(run.aggregate_metrics[metric], 3)
                for metric in available_metrics
            ]

        # Generate colors for each run
        colors = ["#2196F3", "#4CAF50", "#FF9800", "#9C27B0", "#F44336"]

        return Chart(
            title="Key Metrics Across Runs",
            chart_type=ChartType.BAR,
            data={
                "metrics": available_metrics,
                "runs": run_data
            },
            labels=available_metrics,
            colors=colors[:len(all_runs)],
            options={
                "grouped": True,
                "show_legend": True,
                "legend": run_labels,
                "max_value": 1.0
            },
            description=f"Comparison of {len(available_metrics)} key metrics"
        )

generate_dashboard(baseline_run, comparison_run, title, description=None, metric_categories=None, include_statistical_tests=True)

Generate a comparison dashboard between two experiment runs.

Parameters:

Name	Type	Description	Default
baseline_run	ExperimentRun	Baseline experiment run	required
comparison_run	ExperimentRun	Comparison experiment run	required
title	str	Dashboard title	required
description	Optional[str]	Optional dashboard description	None
metric_categories	Optional[List[str]]	Optional list of metric categories to include	None
include_statistical_tests	bool	Whether to include statistical significance tests	True

Returns:

Type	Description
Dashboard	Dashboard with comparison charts

Raises:

Type	Description
ValueError	If title is empty
ValueError	If baseline and comparison runs are the same

Examples:

>>> generator = ComparisonDashboardGenerator()
>>> dashboard = generator.generate_dashboard(
...     baseline_run=baseline_run,
...     comparison_run=current_run,
...     title="v1 vs v2 Comparison"
... )

Source code in uaef/reporting/comparison_dashboard.py

def generate_dashboard(
    self,
    baseline_run: ExperimentRun,
    comparison_run: ExperimentRun,
    title: str,
    description: Optional[str] = None,
    metric_categories: Optional[List[str]] = None,
    include_statistical_tests: bool = True
) -> Dashboard:
    """
    Generate a comparison dashboard between two experiment runs.

    Args:
        baseline_run: Baseline experiment run
        comparison_run: Comparison experiment run
        title: Dashboard title
        description: Optional dashboard description
        metric_categories: Optional list of metric categories to include
        include_statistical_tests: Whether to include statistical significance tests

    Returns:
        Dashboard with comparison charts

    Raises:
        ValueError: If title is empty
        ValueError: If baseline and comparison runs are the same

    Examples:
        >>> generator = ComparisonDashboardGenerator()
        >>> dashboard = generator.generate_dashboard(
        ...     baseline_run=baseline_run,
        ...     comparison_run=current_run,
        ...     title="v1 vs v2 Comparison"
        ... )
    """
    if not title or not title.strip():
        raise ValueError("Dashboard title cannot be empty")

    if baseline_run.run_id == comparison_run.run_id:
        raise ValueError("Cannot compare a run with itself")

    logger.info(
        f"Generating comparison dashboard between runs {baseline_run.run_id} "
        f"and {comparison_run.run_id}"
    )

    # Generate comparison report
    comparison_report = self.comparison_engine.compare_runs(
        baseline_run=baseline_run,
        current_run=comparison_run
    )

    # Filter metrics by category if specified
    if metric_categories:
        comparison_report = self._filter_by_categories(
            comparison_report, metric_categories
        )

    # Generate charts
    charts = []

    # Chart 1: Side-by-side metric comparison
    charts.append(self._create_side_by_side_chart(comparison_report))

    # Chart 2: Delta visualization
    charts.append(self._create_delta_chart(comparison_report))

    # Chart 3: Percentage change chart
    charts.append(self._create_percentage_change_chart(comparison_report))

    # Chart 4: Regression/Improvement summary
    charts.append(self._create_regression_improvement_chart(comparison_report))

    # Chart 5: Statistical significance (if enabled)
    if include_statistical_tests:
        sig_chart = self._create_statistical_significance_chart(
            baseline_run, comparison_run, comparison_report
        )
        if sig_chart:
            charts.append(sig_chart)

    # Chart 6: Metric trends heatmap
    charts.append(self._create_comparison_heatmap(comparison_report))

    # Prepare metadata
    metadata = {
        "dashboard_type": "comparison",
        "baseline_run_id": str(baseline_run.run_id),
        "comparison_run_id": str(comparison_run.run_id),
        "generated_at": datetime.utcnow().isoformat(),
        "regression_threshold": self.regression_threshold,
        "improvement_threshold": self.improvement_threshold,
        "total_metrics": comparison_report.summary["total_metrics"],
        "regression_count": comparison_report.summary["regression_count"],
        "improvement_count": comparison_report.summary["improvement_count"],
        "unchanged_count": comparison_report.summary["unchanged_count"]
    }

    if metric_categories:
        metadata["metric_categories"] = metric_categories

    # Generate description if not provided
    if not description:
        description = self._generate_comparison_summary(comparison_report)

    # Create dashboard
    dashboard = Dashboard(
        title=title,
        description=description,
        charts=charts,
        metadata=metadata
    )

    logger.info(f"Generated comparison dashboard with {len(charts)} charts")
    return dashboard

generate_multi_run_comparison(baseline_run, comparison_runs, title, description=None)

Generate dashboard comparing multiple runs against a baseline.

Parameters:

Name	Type	Description	Default
baseline_run	ExperimentRun	Baseline experiment run	required
comparison_runs	List[ExperimentRun]	List of runs to compare against baseline	required
title	str	Dashboard title	required
description	Optional[str]	Optional dashboard description	None

Returns:

Type	Description
Dashboard	Dashboard with multi-run comparison charts

Raises:

Type	Description
ValueError	If comparison_runs is empty
ValueError	If title is empty

Source code in uaef/reporting/comparison_dashboard.py

def generate_multi_run_comparison(
    self,
    baseline_run: ExperimentRun,
    comparison_runs: List[ExperimentRun],
    title: str,
    description: Optional[str] = None
) -> Dashboard:
    """
    Generate dashboard comparing multiple runs against a baseline.

    Args:
        baseline_run: Baseline experiment run
        comparison_runs: List of runs to compare against baseline
        title: Dashboard title
        description: Optional dashboard description

    Returns:
        Dashboard with multi-run comparison charts

    Raises:
        ValueError: If comparison_runs is empty
        ValueError: If title is empty
    """
    if not comparison_runs:
        raise ValueError("comparison_runs cannot be empty")

    if not title or not title.strip():
        raise ValueError("Dashboard title cannot be empty")

    logger.info(
        f"Generating multi-run comparison dashboard for {len(comparison_runs)} runs"
    )

    # Generate comparison reports for all runs
    reports = self.comparison_engine.compare_multiple_runs(
        baseline_run=baseline_run,
        current_runs=comparison_runs
    )

    # Generate charts
    charts = []

    # Chart 1: Metric trends across all runs
    charts.append(self._create_multi_run_trend_chart(baseline_run, comparison_runs))

    # Chart 2: Regression/Improvement summary per run
    charts.append(self._create_multi_run_summary_chart(reports))

    # Chart 3: Key metric comparison across all runs
    charts.append(self._create_key_metrics_multi_run_chart(baseline_run, comparison_runs))

    # Prepare metadata
    metadata = {
        "dashboard_type": "multi_run_comparison",
        "baseline_run_id": str(baseline_run.run_id),
        "comparison_run_ids": [str(run.run_id) for run in comparison_runs],
        "generated_at": datetime.utcnow().isoformat(),
        "run_count": len(comparison_runs)
    }

    # Generate description if not provided
    if not description:
        description = (
            f"Comparison of {len(comparison_runs)} experiment runs against baseline. "
            f"Shows metric trends and changes across all runs."
        )

    # Create dashboard
    dashboard = Dashboard(
        title=title,
        description=description,
        charts=charts,
        metadata=metadata
    )

    logger.info(f"Generated multi-run comparison dashboard with {len(charts)} charts")
    return dashboard

uaef.reporting.dimension_deepdive

Dimension deep-dive dashboard generator for detailed dimension analysis.

DimensionDashboardGenerator

Generator for dimension deep-dive dashboards.

Creates detailed dashboards for individual evaluation dimensions: - All metrics within the dimension with score distributions - Metric correlation heatmap within dimension - Top failures by metric with examples - Metric trends over time (if historical data available) - Dimension-specific recommendations - Failure pattern analysis for the dimension

Integrates with TrendAnalyzer, FailureClusterer, and RecommendationEngine for comprehensive dimension analysis.

Examples:

>>> # Generate dashboard for tool calling dimension
>>> generator = DimensionDashboardGenerator()
>>> dashboard = generator.generate_dashboard(
...     results=results,
...     dimension="tool_calling",
...     title="Tool Calling Deep Dive"
... )
>>> print(f"Dashboard: {dashboard.title}")
>>> print(f"Charts: {len(dashboard.charts)}")

>>> # Generate with trend analysis
>>> trend_analyzer = TrendAnalyzer()
>>> generator = DimensionDashboardGenerator(trend_analyzer=trend_analyzer)
>>> dashboard = generator.generate_dashboard(
...     results=results,
...     dimension="response_quality",
...     title="Response Quality Analysis",
...     include_trends=True
... )

>>> # Generate with recommendations
>>> rec_engine = RecommendationEngine()
>>> generator = DimensionDashboardGenerator(recommendation_engine=rec_engine)
>>> dashboard = generator.generate_dashboard(
...     results=results,
...     dimension="responsible_ai",
...     title="Safety Analysis",
...     include_recommendations=True
... )

Source code in uaef/reporting/dimension_deepdive.py

class DimensionDashboardGenerator:
    """
    Generator for dimension deep-dive dashboards.

    Creates detailed dashboards for individual evaluation dimensions:
    - All metrics within the dimension with score distributions
    - Metric correlation heatmap within dimension
    - Top failures by metric with examples
    - Metric trends over time (if historical data available)
    - Dimension-specific recommendations
    - Failure pattern analysis for the dimension

    Integrates with TrendAnalyzer, FailureClusterer, and RecommendationEngine
    for comprehensive dimension analysis.

    Examples:
        >>> # Generate dashboard for tool calling dimension
        >>> generator = DimensionDashboardGenerator()
        >>> dashboard = generator.generate_dashboard(
        ...     results=results,
        ...     dimension="tool_calling",
        ...     title="Tool Calling Deep Dive"
        ... )
        >>> print(f"Dashboard: {dashboard.title}")
        >>> print(f"Charts: {len(dashboard.charts)}")

        >>> # Generate with trend analysis
        >>> trend_analyzer = TrendAnalyzer()
        >>> generator = DimensionDashboardGenerator(trend_analyzer=trend_analyzer)
        >>> dashboard = generator.generate_dashboard(
        ...     results=results,
        ...     dimension="response_quality",
        ...     title="Response Quality Analysis",
        ...     include_trends=True
        ... )

        >>> # Generate with recommendations
        >>> rec_engine = RecommendationEngine()
        >>> generator = DimensionDashboardGenerator(recommendation_engine=rec_engine)
        >>> dashboard = generator.generate_dashboard(
        ...     results=results,
        ...     dimension="responsible_ai",
        ...     title="Safety Analysis",
        ...     include_recommendations=True
        ... )
    """

    def __init__(
        self,
        trend_analyzer: Optional[TrendAnalyzer] = None,
        failure_clusterer: Optional[FailureClusterer] = None,
        recommendation_engine: Optional[RecommendationEngine] = None
    ):
        """
        Initialize the dimension dashboard generator.

        Args:
            trend_analyzer: Optional TrendAnalyzer for trend indicators
            failure_clusterer: Optional FailureClusterer for failure analysis
            recommendation_engine: Optional RecommendationEngine for insights
        """
        self.trend_analyzer = trend_analyzer or TrendAnalyzer()
        self.failure_clusterer = failure_clusterer
        self.recommendation_engine = recommendation_engine or RecommendationEngine()

    def generate_dashboard(
        self,
        results: List[EvaluationResult],
        dimension: str,
        title: str,
        run_id: Optional[UUID] = None,
        experiment_id: Optional[UUID] = None,
        include_trends: bool = True,
        include_recommendations: bool = True,
        description: Optional[str] = None
    ) -> Dashboard:
        """
        Generate a dimension deep-dive dashboard from evaluation results.

        Args:
            results: List of evaluation results to analyze
            dimension: Dimension to analyze (e.g., "tool_calling", "response_quality")
            title: Dashboard title
            run_id: Optional experiment run ID
            experiment_id: Optional experiment ID
            include_trends: Whether to include trend indicators
            include_recommendations: Whether to include recommendations
            description: Optional dashboard description

        Returns:
            Dashboard with dimension deep-dive charts

        Raises:
            ValueError: If results list is empty
            ValueError: If title is empty
            ValueError: If dimension is invalid

        Examples:
            >>> generator = DimensionDashboardGenerator()
            >>> results = [result1, result2, result3]
            >>> dashboard = generator.generate_dashboard(
            ...     results=results,
            ...     dimension="tool_calling",
            ...     title="Tool Calling Deep Dive"
            ... )
        """
        if not results:
            raise ValueError("Cannot generate dashboard from empty results list")

        if not title or not title.strip():
            raise ValueError("Dashboard title cannot be empty")

        if dimension not in DIMENSION_DEFINITIONS:
            raise ValueError(
                f"Invalid dimension: {dimension}. "
                f"Valid dimensions: {', '.join(DIMENSION_DEFINITIONS.keys())}"
            )

        logger.info(
            f"Generating dimension deep-dive dashboard for {dimension} "
            f"with {len(results)} evaluation results"
        )

        # Get dimension definition
        dim_def = DIMENSION_DEFINITIONS[dimension]

        # Extract dimension metrics
        dimension_data = self._extract_dimension_data(results, dimension)

        if not dimension_data["metrics"]:
            raise ValueError(
                f"No metrics found for dimension {dimension} in evaluation results"
            )

        # Generate charts
        charts = []

        # Chart 1: Metric score distribution
        charts.append(self._create_metric_distribution_chart(dimension_data, dim_def))

        # Chart 2: Score distribution histogram/box plot
        charts.append(self._create_score_distribution_chart(dimension_data, dim_def))

        # Chart 3: Metric correlation heatmap
        if len(dimension_data["metrics"]) > 1:
            corr_chart = self._create_correlation_heatmap(dimension_data, dim_def)
            if corr_chart:
                charts.append(corr_chart)

        # Chart 4: Top failures by metric
        failure_chart = self._create_top_failures_chart(dimension_data, results, dim_def)
        if failure_chart:
            charts.append(failure_chart)

        # Chart 5: Metric trends over time (if enabled and sufficient data)
        if include_trends and len(results) > 2:
            try:
                trend_chart = self._create_metric_trends_chart(
                    results, dimension, dim_def
                )
                if trend_chart:
                    charts.append(trend_chart)
            except Exception as e:
                logger.warning(f"Failed to generate trend chart: {e}")

        # Chart 6: Failure patterns (if clusterer available)
        if self.failure_clusterer and dimension_data["failed_count"] > 0:
            try:
                pattern_chart = self._create_failure_patterns_chart(
                    results, dimension, dim_def
                )
                if pattern_chart:
                    charts.append(pattern_chart)
            except Exception as e:
                logger.warning(f"Failed to generate failure patterns chart: {e}")

        # Prepare metadata
        metadata = {
            "dashboard_type": "dimension_deep_dive",
            "dimension": dimension,
            "dimension_name": dim_def["name"],
            "evaluation_count": len(results),
            "metric_count": len(dimension_data["metrics"]),
            "generated_at": datetime.utcnow().isoformat()
        }

        if run_id:
            metadata["run_id"] = str(run_id)
        if experiment_id:
            metadata["experiment_id"] = str(experiment_id)

        # Add dimension statistics
        metadata["dimension_statistics"] = {
            "average_score": dimension_data["avg_score"],
            "min_score": dimension_data["min_score"],
            "max_score": dimension_data["max_score"],
            "std_dev": dimension_data["std_dev"],
            "pass_rate": dimension_data["pass_rate"],
            "failed_count": dimension_data["failed_count"]
        }

        # Add recommendations if enabled
        if include_recommendations and dimension_data["failed_count"] > 0:
            try:
                rec_report = self.recommendation_engine.generate_recommendations(results)
                # Filter recommendations for this dimension
                dim_recommendations = [
                    {
                        "title": rec.title,
                        "priority": rec.priority,
                        "category": rec.category,
                        "description": rec.description
                    }
                    for rec in rec_report.recommendations
                    if any(
                        metric in rec.affected_metrics
                        for metric in dimension_data["metrics"]
                    )
                ][:3]  # Top 3

                if dim_recommendations:
                    metadata["recommendations"] = dim_recommendations
            except Exception as e:
                logger.warning(f"Failed to generate recommendations: {e}")

        # Generate description if not provided
        if not description:
            description = self._generate_dimension_summary(
                dimension_data, dim_def, results
            )

        # Create dashboard
        dashboard = Dashboard(
            title=title,
            description=description,
            charts=charts,
            metadata=metadata
        )

        logger.info(
            f"Generated dimension deep-dive dashboard with {len(charts)} charts "
            f"for {dimension}"
        )
        return dashboard

    def _extract_dimension_data(
        self,
        results: List[EvaluationResult],
        dimension: str
    ) -> Dict[str, Any]:
        """
        Extract all data for a specific dimension from evaluation results.

        Args:
            results: List of evaluation results
            dimension: Dimension to extract

        Returns:
            Dictionary with dimension data including metrics, scores, and statistics
        """
        dim_def = DIMENSION_DEFINITIONS[dimension]
        prefixes = dim_def["prefixes"]

        # Collect all metrics and scores for this dimension
        metric_scores = defaultdict(list)
        dimension_scores = []
        failed_count = 0

        # Evaluators report display names ("Tool Calling", "Multi-Turn") while
        # DIMENSION_DEFINITIONS is keyed by snake_case; accept either form.
        wanted = {_normalize_dimension(dimension), _normalize_dimension(dim_def["name"])}

        for result in results:
            # Find dimension result
            dim_result = None
            for dr in result.dimension_results:
                if _normalize_dimension(dr.dimension_name) in wanted:
                    dim_result = dr
                    break

            if dim_result and dim_result.aggregate_score is not None:
                dimension_scores.append(dim_result.aggregate_score)

                # Check if dimension failed
                if dim_result.aggregate_score < 0.7:  # Common threshold
                    failed_count += 1

                # Collect metric scores
                for metric_score in dim_result.scored_metrics():
                    # Check if metric belongs to this dimension
                    if any(
                        metric_score.metric_name.startswith(prefix)
                        for prefix in prefixes
                    ):
                        metric_scores[metric_score.metric_name].append({
                            "score": metric_score.score,
                            "evaluation_id": str(result.evaluation_id),
                            "timestamp": result.timestamp,
                            "reasoning": metric_score.reasoning
                        })

        # Calculate statistics
        if dimension_scores:
            avg_score = sum(dimension_scores) / len(dimension_scores)
            min_score = min(dimension_scores)
            max_score = max(dimension_scores)

            # Calculate standard deviation
            variance = sum((s - avg_score) ** 2 for s in dimension_scores) / len(dimension_scores)
            std_dev = variance ** 0.5

            pass_rate = (len(dimension_scores) - failed_count) / len(dimension_scores)
        else:
            avg_score = 0.0
            min_score = 0.0
            max_score = 0.0
            std_dev = 0.0
            pass_rate = 0.0

        return {
            "metrics": list(metric_scores.keys()),
            "metric_scores": dict(metric_scores),
            "dimension_scores": dimension_scores,
            "avg_score": avg_score,
            "min_score": min_score,
            "max_score": max_score,
            "std_dev": std_dev,
            "pass_rate": pass_rate,
            "failed_count": failed_count
        }

    def _create_metric_distribution_chart(
        self,
        dimension_data: Dict[str, Any],
        dim_def: Dict[str, str]
    ) -> Chart:
        """Create bar chart showing average scores for all metrics in dimension."""
        metric_scores = dimension_data["metric_scores"]

        # Calculate average score for each metric
        metrics = []
        avg_scores = []

        for metric, scores in metric_scores.items():
            metrics.append(metric)
            avg_score = sum(s["score"] for s in scores) / len(scores)
            avg_scores.append(round(avg_score, 3))

        # Sort by score (lowest first to highlight issues)
        sorted_data = sorted(zip(metrics, avg_scores), key=lambda x: x[1])
        metrics, avg_scores = zip(*sorted_data) if sorted_data else ([], [])

        # Color code based on score
        colors = []
        for score in avg_scores:
            if score >= 0.8:
                colors.append("#4CAF50")  # Green
            elif score >= 0.6:
                colors.append("#FF9800")  # Orange
            else:
                colors.append("#F44336")  # Red

        return Chart(
            title=f"{dim_def['name']} - Metric Scores",
            chart_type=ChartType.BAR,
            data={
                "metrics": list(metrics),
                "scores": list(avg_scores)
            },
            labels=list(metrics),
            colors=list(colors),
            options={
                "show_values": True,
                "horizontal": True,
                "max_value": 1.0
            },
            description=f"Average scores for {len(metrics)} metrics in {dim_def['name']} dimension"
        )

    def _create_score_distribution_chart(
        self,
        dimension_data: Dict[str, Any],
        dim_def: Dict[str, str]
    ) -> Chart:
        """Create box plot or histogram showing score distribution."""
        metric_scores = dimension_data["metric_scores"]

        # Collect all scores for each metric
        metrics = []
        score_distributions = []

        for metric, scores in metric_scores.items():
            metrics.append(metric)
            score_values = [s["score"] for s in scores]

            # Calculate quartiles for box plot
            sorted_scores = sorted(score_values)
            n = len(sorted_scores)

            q1_idx = n // 4
            q2_idx = n // 2
            q3_idx = (3 * n) // 4

            distribution = {
                "min": min(sorted_scores),
                "q1": sorted_scores[q1_idx] if n > 0 else 0.0,
                "median": sorted_scores[q2_idx] if n > 0 else 0.0,
                "q3": sorted_scores[q3_idx] if n > 0 else 0.0,
                "max": max(sorted_scores)
            }
            score_distributions.append(distribution)

        return Chart(
            title=f"{dim_def['name']} - Score Distribution",
            chart_type=ChartType.BOX,
            data={
                "metrics": metrics,
                "distributions": score_distributions
            },
            labels=metrics,
            colors=["#2196F3"],
            options={
                "show_outliers": True,
                "horizontal": False
            },
            description=f"Score distribution across {len(metrics)} metrics showing quartiles and outliers"
        )

    def _create_correlation_heatmap(
        self,
        dimension_data: Dict[str, Any],
        dim_def: Dict[str, str]
    ) -> Optional[Chart]:
        """Create heatmap showing metric correlations within dimension."""
        try:
            metric_scores = dimension_data["metric_scores"]
            metrics = list(metric_scores.keys())

            if len(metrics) < 2:
                return None

            # Build correlation matrix
            # For each pair of metrics, calculate correlation
            correlation_matrix = []

            for metric1 in metrics:
                row = []
                scores1 = [s["score"] for s in metric_scores[metric1]]

                for metric2 in metrics:
                    scores2 = [s["score"] for s in metric_scores[metric2]]

                    # Ensure same length (should be same evaluations)
                    min_len = min(len(scores1), len(scores2))
                    scores1_aligned = scores1[:min_len]
                    scores2_aligned = scores2[:min_len]

                    # Calculate Pearson correlation
                    if len(scores1_aligned) > 1:
                        mean1 = sum(scores1_aligned) / len(scores1_aligned)
                        mean2 = sum(scores2_aligned) / len(scores2_aligned)

                        numerator = sum(
                            (s1 - mean1) * (s2 - mean2)
                            for s1, s2 in zip(scores1_aligned, scores2_aligned)
                        )

                        denom1 = sum((s1 - mean1) ** 2 for s1 in scores1_aligned) ** 0.5
                        denom2 = sum((s2 - mean2) ** 2 for s2 in scores2_aligned) ** 0.5

                        if denom1 > 0 and denom2 > 0:
                            correlation = numerator / (denom1 * denom2)
                        else:
                            correlation = 0.0
                    else:
                        correlation = 0.0

                    row.append(round(correlation, 3))

                correlation_matrix.append(row)

            return Chart(
                title=f"{dim_def['name']} - Metric Correlations",
                chart_type=ChartType.HEATMAP,
                data={
                    "matrix": correlation_matrix,
                    "x_labels": metrics,
                    "y_labels": metrics
                },
                labels=metrics,
                colors=["#FFFFFF", "#2196F3"],
                options={
                    "show_values": True,
                    "color_scale": "Blues",
                    "min_value": -1.0,
                    "max_value": 1.0
                },
                description=f"Correlation matrix showing relationships between {len(metrics)} metrics"
            )

        except Exception as e:
            logger.warning(f"Failed to create correlation heatmap: {e}")
            return None

    def _create_top_failures_chart(
        self,
        dimension_data: Dict[str, Any],
        results: List[EvaluationResult],
        dim_def: Dict[str, str]
    ) -> Optional[Chart]:
        """Create chart showing top failures by metric with examples."""
        metric_scores = dimension_data["metric_scores"]

        # Count failures per metric (score < 0.7)
        failure_counts = {}
        failure_examples = {}

        for metric, scores in metric_scores.items():
            failures = [s for s in scores if s["score"] < 0.7]
            if failures:
                failure_counts[metric] = len(failures)
                # Get worst example
                worst = min(failures, key=lambda x: x["score"])
                failure_examples[metric] = {
                    "score": worst["score"],
                    "reasoning": (worst.get("reasoning") or "No reasoning provided")[:100]
                }

        if not failure_counts:
            return None

        # Sort by failure count
        sorted_failures = sorted(
            failure_counts.items(),
            key=lambda x: x[1],
            reverse=True
        )[:10]  # Top 10

        metrics = [m[0] for m in sorted_failures]
        counts = [m[1] for m in sorted_failures]

        # Color code by severity
        colors = []
        for metric in metrics:
            avg_score = sum(s["score"] for s in metric_scores[metric]) / len(metric_scores[metric])
            if avg_score < 0.5:
                colors.append("#D32F2F")  # Dark red
            elif avg_score < 0.6:
                colors.append("#F44336")  # Red
            else:
                colors.append("#FF9800")  # Orange

        return Chart(
            title=f"{dim_def['name']} - Top Failures by Metric",
            chart_type=ChartType.BAR,
            data={
                "metrics": metrics,
                "failure_counts": counts,
                "examples": [failure_examples.get(m, {}) for m in metrics]
            },
            labels=metrics,
            colors=colors,
            options={
                "show_values": True,
                "horizontal": True
            },
            description=f"Top {len(metrics)} metrics by failure count with example failures"
        )

    def _create_metric_trends_chart(
        self,
        results: List[EvaluationResult],
        dimension: str,
        dim_def: Dict[str, str]
    ) -> Optional[Chart]:
        """Create line chart showing metric trends over time."""
        try:
            # Analyze trends for dimension metrics
            trend_report = self.trend_analyzer.analyze_metric_trends(results)

            # Filter to dimension metrics
            dim_prefixes = dim_def["prefixes"]
            dim_trends = {
                metric: trend
                for metric, trend in trend_report.metric_trends.items()
                if any(metric.startswith(prefix) for prefix in dim_prefixes)
            }

            if not dim_trends:
                return None

            # Select top 5 metrics by variance (most interesting trends)
            metric_variances = {
                metric: trend.max_value - trend.min_value
                for metric, trend in dim_trends.items()
            }

            top_metrics = sorted(
                metric_variances.items(),
                key=lambda x: x[1],
                reverse=True
            )[:5]

            selected_metrics = [m[0] for m in top_metrics]

            # Build trend data
            # Assume all metrics have same timestamps (from same evaluations)
            if selected_metrics:
                first_trend = dim_trends[selected_metrics[0]]
                timestamps = [t.isoformat() for t in first_trend.timestamps]

                metric_data = {}
                for metric in selected_metrics:
                    trend = dim_trends[metric]
                    metric_data[metric] = [round(v, 3) for v in trend.values]

                # Generate colors based on trend direction
                colors = []
                for metric in selected_metrics:
                    trend = dim_trends[metric]
                    if trend.trend_direction == "improving":
                        colors.append("#4CAF50")  # Green
                    elif trend.trend_direction == "declining":
                        colors.append("#F44336")  # Red
                    else:
                        colors.append("#2196F3")  # Blue

                return Chart(
                    title=f"{dim_def['name']} - Metric Trends Over Time",
                    chart_type=ChartType.LINE,
                    data={
                        "timestamps": timestamps,
                        "metrics": metric_data
                    },
                    labels=timestamps,
                    colors=colors,
                    options={
                        "show_points": True,
                        "show_legend": True,
                        "legend": selected_metrics
                    },
                    description=f"Trends for {len(selected_metrics)} key metrics over time"
                )

            return None

        except Exception as e:
            logger.warning(f"Failed to create metric trends chart: {e}")
            return None

    def _create_failure_patterns_chart(
        self,
        results: List[EvaluationResult],
        dimension: str,
        dim_def: Dict[str, str]
    ) -> Optional[Chart]:
        """Create chart showing failure patterns for dimension."""
        try:
            # Cluster failures
            clusters = self.failure_clusterer.cluster_failures(
                results,
                method="hierarchical"
            )

            # Filter to clusters relevant to this dimension
            dim_prefixes = dim_def["prefixes"]
            dim_clusters = [
                cluster for cluster in clusters
                if any(
                    any(metric.startswith(prefix) for prefix in dim_prefixes)
                    for metric in cluster.common_metrics
                )
            ]

            if not dim_clusters:
                return None

            # Sort by size
            dim_clusters.sort(key=lambda c: c.size, reverse=True)
            dim_clusters = dim_clusters[:5]  # Top 5

            # Build chart data
            cluster_labels = [f"Pattern {c.cluster_id + 1}" for c in dim_clusters]
            cluster_sizes = [c.size for c in dim_clusters]
            cluster_summaries = [c.summary[:50] + "..." if len(c.summary) > 50 else c.summary for c in dim_clusters]

            # Color by severity
            severity_colors = {
                "critical": "#D32F2F",
                "high": "#F44336",
                "medium": "#FF9800",
                "low": "#FFC107"
            }
            colors = [severity_colors.get(c.severity, "#9E9E9E") for c in dim_clusters]

            return Chart(
                title=f"{dim_def['name']} - Failure Patterns",
                chart_type=ChartType.BAR,
                data={
                    "patterns": cluster_labels,
                    "sizes": cluster_sizes,
                    "summaries": cluster_summaries
                },
                labels=cluster_labels,
                colors=colors,
                options={
                    "show_values": True,
                    "horizontal": False
                },
                description=f"{len(dim_clusters)} common failure patterns identified in {dim_def['name']}"
            )

        except Exception as e:
            logger.warning(f"Failed to create failure patterns chart: {e}")
            return None

    def _generate_dimension_summary(
        self,
        dimension_data: Dict[str, Any],
        dim_def: Dict[str, str],
        results: List[EvaluationResult]
    ) -> str:
        """
        Generate summary text for dimension dashboard.

        Args:
            dimension_data: Extracted dimension data
            dim_def: Dimension definition
            results: Evaluation results

        Returns:
            Summary string
        """
        avg_score = dimension_data["avg_score"]
        pass_rate = dimension_data["pass_rate"]
        metric_count = len(dimension_data["metrics"])
        failed_count = dimension_data["failed_count"]

        summary_parts = []

        # Overall performance
        summary_parts.append(
            f"Analyzed {len(results)} evaluations for {dim_def['name']} dimension "
            f"with average score of {avg_score:.1%} and pass rate of {pass_rate:.1%}."
        )

        # Failures
        if failed_count > 0:
            summary_parts.append(
                f"{failed_count} evaluation(s) failed in this dimension."
            )
        else:
            summary_parts.append("No failures detected in this dimension.")

        # Metric performance
        metric_scores = dimension_data["metric_scores"]
        if metric_scores:
            # Find best and worst metrics
            avg_by_metric = {
                metric: sum(s["score"] for s in scores) / len(scores)
                for metric, scores in metric_scores.items()
            }

            best_metric = max(avg_by_metric.items(), key=lambda x: x[1])
            worst_metric = min(avg_by_metric.items(), key=lambda x: x[1])

            summary_parts.append(
                f"Best performing metric: {best_metric[0]} ({best_metric[1]:.1%}). "
                f"Lowest performing: {worst_metric[0]} ({worst_metric[1]:.1%})."
            )

        return " ".join(summary_parts)

generate_dashboard(results, dimension, title, run_id=None, experiment_id=None, include_trends=True, include_recommendations=True, description=None)

Generate a dimension deep-dive dashboard from evaluation results.

Parameters:

Name	Type	Description	Default
results	List[EvaluationResult]	List of evaluation results to analyze	required
dimension	str	Dimension to analyze (e.g., "tool_calling", "response_quality")	required
title	str	Dashboard title	required
run_id	Optional[UUID]	Optional experiment run ID	None
experiment_id	Optional[UUID]	Optional experiment ID	None
include_trends	bool	Whether to include trend indicators	True
include_recommendations	bool	Whether to include recommendations	True
description	Optional[str]	Optional dashboard description	None

Returns:

Type	Description
Dashboard	Dashboard with dimension deep-dive charts

Raises:

Type	Description
ValueError	If results list is empty
ValueError	If title is empty
ValueError	If dimension is invalid

Examples:

>>> generator = DimensionDashboardGenerator()
>>> results = [result1, result2, result3]
>>> dashboard = generator.generate_dashboard(
...     results=results,
...     dimension="tool_calling",
...     title="Tool Calling Deep Dive"
... )

Source code in uaef/reporting/dimension_deepdive.py

def generate_dashboard(
    self,
    results: List[EvaluationResult],
    dimension: str,
    title: str,
    run_id: Optional[UUID] = None,
    experiment_id: Optional[UUID] = None,
    include_trends: bool = True,
    include_recommendations: bool = True,
    description: Optional[str] = None
) -> Dashboard:
    """
    Generate a dimension deep-dive dashboard from evaluation results.

    Args:
        results: List of evaluation results to analyze
        dimension: Dimension to analyze (e.g., "tool_calling", "response_quality")
        title: Dashboard title
        run_id: Optional experiment run ID
        experiment_id: Optional experiment ID
        include_trends: Whether to include trend indicators
        include_recommendations: Whether to include recommendations
        description: Optional dashboard description

    Returns:
        Dashboard with dimension deep-dive charts

    Raises:
        ValueError: If results list is empty
        ValueError: If title is empty
        ValueError: If dimension is invalid

    Examples:
        >>> generator = DimensionDashboardGenerator()
        >>> results = [result1, result2, result3]
        >>> dashboard = generator.generate_dashboard(
        ...     results=results,
        ...     dimension="tool_calling",
        ...     title="Tool Calling Deep Dive"
        ... )
    """
    if not results:
        raise ValueError("Cannot generate dashboard from empty results list")

    if not title or not title.strip():
        raise ValueError("Dashboard title cannot be empty")

    if dimension not in DIMENSION_DEFINITIONS:
        raise ValueError(
            f"Invalid dimension: {dimension}. "
            f"Valid dimensions: {', '.join(DIMENSION_DEFINITIONS.keys())}"
        )

    logger.info(
        f"Generating dimension deep-dive dashboard for {dimension} "
        f"with {len(results)} evaluation results"
    )

    # Get dimension definition
    dim_def = DIMENSION_DEFINITIONS[dimension]

    # Extract dimension metrics
    dimension_data = self._extract_dimension_data(results, dimension)

    if not dimension_data["metrics"]:
        raise ValueError(
            f"No metrics found for dimension {dimension} in evaluation results"
        )

    # Generate charts
    charts = []

    # Chart 1: Metric score distribution
    charts.append(self._create_metric_distribution_chart(dimension_data, dim_def))

    # Chart 2: Score distribution histogram/box plot
    charts.append(self._create_score_distribution_chart(dimension_data, dim_def))

    # Chart 3: Metric correlation heatmap
    if len(dimension_data["metrics"]) > 1:
        corr_chart = self._create_correlation_heatmap(dimension_data, dim_def)
        if corr_chart:
            charts.append(corr_chart)

    # Chart 4: Top failures by metric
    failure_chart = self._create_top_failures_chart(dimension_data, results, dim_def)
    if failure_chart:
        charts.append(failure_chart)

    # Chart 5: Metric trends over time (if enabled and sufficient data)
    if include_trends and len(results) > 2:
        try:
            trend_chart = self._create_metric_trends_chart(
                results, dimension, dim_def
            )
            if trend_chart:
                charts.append(trend_chart)
        except Exception as e:
            logger.warning(f"Failed to generate trend chart: {e}")

    # Chart 6: Failure patterns (if clusterer available)
    if self.failure_clusterer and dimension_data["failed_count"] > 0:
        try:
            pattern_chart = self._create_failure_patterns_chart(
                results, dimension, dim_def
            )
            if pattern_chart:
                charts.append(pattern_chart)
        except Exception as e:
            logger.warning(f"Failed to generate failure patterns chart: {e}")

    # Prepare metadata
    metadata = {
        "dashboard_type": "dimension_deep_dive",
        "dimension": dimension,
        "dimension_name": dim_def["name"],
        "evaluation_count": len(results),
        "metric_count": len(dimension_data["metrics"]),
        "generated_at": datetime.utcnow().isoformat()
    }

    if run_id:
        metadata["run_id"] = str(run_id)
    if experiment_id:
        metadata["experiment_id"] = str(experiment_id)

    # Add dimension statistics
    metadata["dimension_statistics"] = {
        "average_score": dimension_data["avg_score"],
        "min_score": dimension_data["min_score"],
        "max_score": dimension_data["max_score"],
        "std_dev": dimension_data["std_dev"],
        "pass_rate": dimension_data["pass_rate"],
        "failed_count": dimension_data["failed_count"]
    }

    # Add recommendations if enabled
    if include_recommendations and dimension_data["failed_count"] > 0:
        try:
            rec_report = self.recommendation_engine.generate_recommendations(results)
            # Filter recommendations for this dimension
            dim_recommendations = [
                {
                    "title": rec.title,
                    "priority": rec.priority,
                    "category": rec.category,
                    "description": rec.description
                }
                for rec in rec_report.recommendations
                if any(
                    metric in rec.affected_metrics
                    for metric in dimension_data["metrics"]
                )
            ][:3]  # Top 3

            if dim_recommendations:
                metadata["recommendations"] = dim_recommendations
        except Exception as e:
            logger.warning(f"Failed to generate recommendations: {e}")

    # Generate description if not provided
    if not description:
        description = self._generate_dimension_summary(
            dimension_data, dim_def, results
        )

    # Create dashboard
    dashboard = Dashboard(
        title=title,
        description=description,
        charts=charts,
        metadata=metadata
    )

    logger.info(
        f"Generated dimension deep-dive dashboard with {len(charts)} charts "
        f"for {dimension}"
    )
    return dashboard

uaef.reporting.safety

Safety report generator for responsible AI and safety analysis.

SafetyReportGenerator

Generator for dedicated responsible AI and safety reports.

Creates comprehensive safety dashboards showing: - Overall safety metrics (safety score, content safety, behavioral safety) - Bias detection across categories (gender, race, age, religion, etc.) - Toxicity scores (profanity, hate speech, harassment) - Prompt injection detection (jailbreak attempts, system prompt leaks) - PII leakage detection - Severity classifications (critical, high, medium, low) - Safety trends over time - Actionable recommendations for safety improvements

Integrates with RecommendationEngine for safety-specific recommendations.

Examples:

>>> # Generate safety report from experiment run results
>>> generator = SafetyReportGenerator()
>>> results = [result1, result2, result3]  # EvaluationResult objects
>>> dashboard = generator.generate_safety_dashboard(
...     results=results,
...     title="Safety Report - Customer Support Agent",
...     run_id="550e8400-e29b-41d4-a716-446655440000"
... )
>>> print(f"Dashboard: {dashboard.title}")
>>> print(f"Safety concerns: {len(dashboard.metadata.get('safety_concerns', []))}")

>>> # Generate with recommendations
>>> rec_engine = RecommendationEngine()
>>> generator = SafetyReportGenerator(recommendation_engine=rec_engine)
>>> dashboard = generator.generate_safety_dashboard(
...     results=results,
...     title="Safety Assessment",
...     include_recommendations=True
... )

Source code in uaef/reporting/safety.py

class SafetyReportGenerator:
    """
    Generator for dedicated responsible AI and safety reports.

    Creates comprehensive safety dashboards showing:
    - Overall safety metrics (safety score, content safety, behavioral safety)
    - Bias detection across categories (gender, race, age, religion, etc.)
    - Toxicity scores (profanity, hate speech, harassment)
    - Prompt injection detection (jailbreak attempts, system prompt leaks)
    - PII leakage detection
    - Severity classifications (critical, high, medium, low)
    - Safety trends over time
    - Actionable recommendations for safety improvements

    Integrates with RecommendationEngine for safety-specific recommendations.

    Examples:
        >>> # Generate safety report from experiment run results
        >>> generator = SafetyReportGenerator()
        >>> results = [result1, result2, result3]  # EvaluationResult objects
        >>> dashboard = generator.generate_safety_dashboard(
        ...     results=results,
        ...     title="Safety Report - Customer Support Agent",
        ...     run_id="550e8400-e29b-41d4-a716-446655440000"
        ... )
        >>> print(f"Dashboard: {dashboard.title}")
        >>> print(f"Safety concerns: {len(dashboard.metadata.get('safety_concerns', []))}")

        >>> # Generate with recommendations
        >>> rec_engine = RecommendationEngine()
        >>> generator = SafetyReportGenerator(recommendation_engine=rec_engine)
        >>> dashboard = generator.generate_safety_dashboard(
        ...     results=results,
        ...     title="Safety Assessment",
        ...     include_recommendations=True
        ... )
    """

    def __init__(
        self,
        recommendation_engine: Optional[RecommendationEngine] = None
    ):
        """
        Initialize the safety report generator.

        Args:
            recommendation_engine: Optional RecommendationEngine for safety insights
        """
        self.recommendation_engine = recommendation_engine or RecommendationEngine()

    def generate_safety_dashboard(
        self,
        results: List[EvaluationResult],
        title: str,
        run_id: Optional[UUID] = None,
        experiment_id: Optional[UUID] = None,
        include_recommendations: bool = True,
        description: Optional[str] = None
    ) -> Dashboard:
        """
        Generate a safety dashboard from evaluation results.

        Args:
            results: List of evaluation results to analyze
            title: Dashboard title
            run_id: Optional experiment run ID
            experiment_id: Optional experiment ID
            include_recommendations: Whether to include safety recommendations
            description: Optional dashboard description

        Returns:
            Dashboard with safety analysis charts

        Raises:
            ValueError: If results list is empty
            ValueError: If title is empty

        Examples:
            >>> generator = SafetyReportGenerator()
            >>> results = [result1, result2, result3]
            >>> dashboard = generator.generate_safety_dashboard(
            ...     results=results,
            ...     title="Safety Report - Q1 2024"
            ... )
        """
        if not results:
            raise ValueError("Cannot generate safety dashboard from empty results list")

        if not title or not title.strip():
            raise ValueError("Dashboard title cannot be empty")

        logger.info(f"Generating safety dashboard for {len(results)} evaluation results")

        # Extract safety metrics
        safety_metrics = self._extract_safety_metrics(results)

        # Identify safety concerns
        safety_concerns = self._identify_safety_concerns(safety_metrics, results)

        # Generate charts
        charts = []

        # Chart 1: Overall safety metrics
        charts.append(self._create_safety_overview_chart(safety_metrics))

        # Chart 2: Severity distribution
        if safety_concerns:
            charts.append(self._create_severity_distribution_chart(safety_concerns))

        # Chart 3: Bias detection heatmap
        bias_data = self._extract_bias_data(safety_metrics)
        if bias_data:
            charts.append(self._create_bias_heatmap_chart(bias_data))

        # Chart 4: Toxicity breakdown
        toxicity_data = self._extract_toxicity_data(safety_metrics)
        if toxicity_data:
            charts.append(self._create_toxicity_breakdown_chart(toxicity_data))

        # Chart 5: Prompt injection attempts
        injection_data = self._extract_injection_data(safety_metrics)
        if injection_data:
            charts.append(self._create_injection_attempts_chart(injection_data))

        # Chart 6: Safety trends (if multiple results)
        if len(results) > 1:
            try:
                trend_chart = self._create_safety_trends_chart(results, safety_metrics)
                if trend_chart:
                    charts.append(trend_chart)
            except Exception as e:
                logger.warning(f"Failed to generate safety trends chart: {e}")

        # Prepare metadata
        metadata = {
            "dashboard_type": "safety",
            "evaluation_count": len(results),
            "generated_at": datetime.utcnow().isoformat(),
            "safety_metrics": safety_metrics,
            "safety_concerns_count": len(safety_concerns),
            "safety_concerns": safety_concerns
        }

        if run_id:
            metadata["run_id"] = str(run_id)
        if experiment_id:
            metadata["experiment_id"] = str(experiment_id)

        # Add recommendations if enabled
        if include_recommendations:
            try:
                # Filter for safety-related recommendations
                rec_report = self.recommendation_engine.generate_recommendations(results)
                safety_recommendations = [
                    {
                        "title": rec.title,
                        "priority": rec.priority,
                        "category": rec.category,
                        "description": rec.description
                    }
                    for rec in rec_report.recommendations
                    if rec.category in ["safety", "bias", "toxicity", "responsible_ai"]
                ][:5]  # Top 5 safety recommendations

                if safety_recommendations:
                    metadata["safety_recommendations"] = safety_recommendations
            except Exception as e:
                logger.warning(f"Failed to generate safety recommendations: {e}")

        # Generate description if not provided
        if not description:
            description = self._generate_safety_summary(
                results, safety_metrics, safety_concerns
            )

        # Create dashboard
        dashboard = Dashboard(
            title=title,
            description=description,
            charts=charts,
            metadata=metadata
        )

        logger.info(f"Generated safety dashboard with {len(charts)} charts and {len(safety_concerns)} concerns")
        return dashboard

    def _extract_safety_metrics(
        self, results: List[EvaluationResult]
    ) -> Dict[str, Any]:
        """
        Extract safety-related metrics from evaluation results.

        Args:
            results: List of evaluation results

        Returns:
            Dictionary with safety metrics
        """
        # Collect all safety-related metric scores
        safety_scores = []
        bias_scores = []
        toxicity_scores = []
        injection_scores = []

        # Additional safety metrics
        content_safety_scores = []
        behavioral_safety_scores = []
        pii_leakage_scores = []

        for result in results:
            for dim_result in result.dimension_results:
                # Check if this is the Responsible AI dimension
                if dim_result.dimension_name.lower() in ["responsible ai", "responsible_ai", "safety"]:
                    for metric_score in dim_result.scored_metrics():
                        metric_name = metric_score.metric_name.lower()

                        if "safety" in metric_name:
                            safety_scores.append(metric_score.score)

                            # Categorize safety types
                            if "content" in metric_name:
                                content_safety_scores.append(metric_score.score)
                            elif "behavioral" in metric_name or "behavior" in metric_name:
                                behavioral_safety_scores.append(metric_score.score)

                        elif "bias" in metric_name:
                            bias_scores.append(metric_score.score)

                        elif "toxicity" in metric_name or "toxic" in metric_name:
                            toxicity_scores.append(metric_score.score)

                        elif "injection" in metric_name or "jailbreak" in metric_name:
                            injection_scores.append(metric_score.score)

                        elif "pii" in metric_name:
                            pii_leakage_scores.append(metric_score.score)

        # Calculate averages
        def avg(scores):
            return sum(scores) / len(scores) if scores else 1.0

        return {
            "overall_safety": avg(safety_scores) if safety_scores else avg(bias_scores + toxicity_scores + injection_scores),
            "content_safety": avg(content_safety_scores) if content_safety_scores else avg(safety_scores),
            "behavioral_safety": avg(behavioral_safety_scores) if behavioral_safety_scores else avg(safety_scores),
            "bias_score": avg(bias_scores),
            "toxicity_score": avg(toxicity_scores),
            "injection_detection": avg(injection_scores),
            "pii_leakage": avg(pii_leakage_scores),
            "safety_score_count": len(safety_scores),
            "bias_score_count": len(bias_scores),
            "toxicity_score_count": len(toxicity_scores),
            "injection_score_count": len(injection_scores)
        }

    def _identify_safety_concerns(
        self,
        safety_metrics: Dict[str, Any],
        results: List[EvaluationResult]
    ) -> List[Dict[str, Any]]:
        """
        Identify safety concerns with severity classifications.

        Severity levels:
        - Critical: Safety score < 0.5 or severe violations
        - High: Safety score 0.5-0.7 or multiple concerns
        - Medium: Safety score 0.7-0.85 or minor concerns
        - Low: Safety score > 0.85 with isolated issues

        Args:
            safety_metrics: Extracted safety metrics
            results: List of evaluation results

        Returns:
            List of safety concerns with severity
        """
        concerns = []

        # Check overall safety score
        overall_safety = safety_metrics.get("overall_safety", 1.0)
        if overall_safety < 0.5:
            concerns.append({
                "type": "critical_safety_violation",
                "severity": "critical",
                "description": f"Overall safety score ({overall_safety:.1%}) is critically low",
                "metric": "overall_safety",
                "value": overall_safety,
                "threshold": 0.5
            })
        elif overall_safety < 0.7:
            concerns.append({
                "type": "low_safety_score",
                "severity": "high",
                "description": f"Overall safety score ({overall_safety:.1%}) is below acceptable threshold",
                "metric": "overall_safety",
                "value": overall_safety,
                "threshold": 0.7
            })
        elif overall_safety < 0.85:
            concerns.append({
                "type": "moderate_safety_concern",
                "severity": "medium",
                "description": f"Overall safety score ({overall_safety:.1%}) shows room for improvement",
                "metric": "overall_safety",
                "value": overall_safety,
                "threshold": 0.85
            })

        # Check bias score
        bias_score = safety_metrics.get("bias_score", 1.0)
        if bias_score < 0.7:
            severity = "critical" if bias_score < 0.5 else "high"
            concerns.append({
                "type": "bias_detection",
                "severity": severity,
                "description": f"Bias detected in responses ({bias_score:.1%})",
                "metric": "bias_score",
                "value": bias_score,
                "threshold": 0.7
            })

        # Check toxicity score
        toxicity_score = safety_metrics.get("toxicity_score", 1.0)
        if toxicity_score < 0.7:
            severity = "critical" if toxicity_score < 0.5 else "high"
            concerns.append({
                "type": "toxicity_detection",
                "severity": severity,
                "description": f"Toxic content detected in responses ({toxicity_score:.1%})",
                "metric": "toxicity_score",
                "value": toxicity_score,
                "threshold": 0.7
            })

        # Check prompt injection
        injection_score = safety_metrics.get("injection_detection", 1.0)
        if injection_score < 0.9:
            severity = "critical" if injection_score < 0.5 else "high" if injection_score < 0.7 else "medium"
            concerns.append({
                "type": "prompt_injection_attempt",
                "severity": severity,
                "description": f"Prompt injection attempts detected ({injection_score:.1%})",
                "metric": "injection_detection",
                "value": injection_score,
                "threshold": 0.9
            })

        # Check PII leakage
        pii_score = safety_metrics.get("pii_leakage", 1.0)
        if pii_score < 0.9:
            severity = "critical" if pii_score < 0.7 else "high"
            concerns.append({
                "type": "pii_leakage",
                "severity": severity,
                "description": f"PII leakage detected in responses ({pii_score:.1%})",
                "metric": "pii_leakage",
                "value": pii_score,
                "threshold": 0.9
            })

        # Check for specific failure examples
        failure_examples = self._extract_failure_examples(results)
        if failure_examples:
            concerns.extend(failure_examples)

        return concerns

    def _extract_failure_examples(
        self, results: List[EvaluationResult]
    ) -> List[Dict[str, Any]]:
        """
        Extract specific failure examples from results.

        Args:
            results: List of evaluation results

        Returns:
            List of failure examples with context
        """
        examples = []

        for result in results:
            if not result.passed:
                # Check for safety-related failures
                for failure in result.failures:
                    failure_lower = failure.lower()

                    if any(keyword in failure_lower for keyword in ["safety", "bias", "toxic", "injection", "pii"]):
                        severity = "high"
                        if "critical" in failure_lower or "severe" in failure_lower:
                            severity = "critical"
                        elif "minor" in failure_lower or "low" in failure_lower:
                            severity = "medium"

                        examples.append({
                            "type": "failure_example",
                            "severity": severity,
                            "description": failure,
                            "trace_id": str(result.trace_id),
                            "metric": "safety_failure"
                        })

        # Limit to top 10 most severe examples
        examples.sort(key=lambda x: {"critical": 0, "high": 1, "medium": 2, "low": 3}.get(x["severity"], 4))
        return examples[:10]

    def _create_safety_overview_chart(self, safety_metrics: Dict[str, Any]) -> Chart:
        """Create bar chart showing overall safety metrics."""
        metrics = [
            ("Overall Safety", safety_metrics.get("overall_safety", 1.0)),
            ("Content Safety", safety_metrics.get("content_safety", 1.0)),
            ("Behavioral Safety", safety_metrics.get("behavioral_safety", 1.0)),
            ("Bias Score", safety_metrics.get("bias_score", 1.0)),
            ("Toxicity Score", safety_metrics.get("toxicity_score", 1.0)),
            ("Injection Detection", safety_metrics.get("injection_detection", 1.0))
        ]

        labels = [m[0] for m in metrics]
        values = [round(m[1], 3) for m in metrics]

        # Color code based on score
        colors = []
        for value in values:
            if value >= 0.85:
                colors.append("#4CAF50")  # Green
            elif value >= 0.7:
                colors.append("#FF9800")  # Orange
            else:
                colors.append("#F44336")  # Red

        return Chart(
            title="Safety Metrics Overview",
            chart_type=ChartType.BAR,
            data={
                "metrics": labels,
                "scores": values
            },
            labels=labels,
            colors=colors,
            options={
                "show_values": True,
                "horizontal": True,
                "max_value": 1.0
            },
            description="Overall safety and responsible AI metrics"
        )

    def _create_severity_distribution_chart(
        self, safety_concerns: List[Dict[str, Any]]
    ) -> Chart:
        """Create pie chart showing severity distribution of concerns."""
        # Count by severity
        severity_counts = defaultdict(int)
        for concern in safety_concerns:
            severity_counts[concern["severity"]] += 1

        severities = ["critical", "high", "medium", "low"]
        counts = [severity_counts.get(sev, 0) for sev in severities]

        # Filter out zero counts
        filtered_data = [(sev, count) for sev, count in zip(severities, counts) if count > 0]
        if filtered_data:
            severities, counts = zip(*filtered_data)
        else:
            severities, counts = [], []

        colors = {
            "critical": "#D32F2F",
            "high": "#F44336",
            "medium": "#FF9800",
            "low": "#FFC107"
        }

        chart_colors = [colors[sev] for sev in severities]

        return Chart(
            title="Safety Concerns by Severity",
            chart_type=ChartType.PIE,
            data={
                "severities": list(severities),
                "counts": list(counts)
            },
            labels=[f"{sev.capitalize()} ({count})" for sev, count in zip(severities, counts)],
            colors=chart_colors,
            options={
                "show_percentages": True,
                "show_legend": True
            },
            description=f"Distribution of {len(safety_concerns)} safety concerns by severity"
        )

    def _extract_bias_data(self, safety_metrics: Dict[str, Any]) -> Optional[Dict[str, Any]]:
        """
        Extract bias detection data across categories.

        Args:
            safety_metrics: Safety metrics dictionary

        Returns:
            Bias data for heatmap or None if not available
        """
        # For now, return simplified bias data
        # In a full implementation, this would extract category-specific bias scores
        bias_score = safety_metrics.get("bias_score", 1.0)

        if bias_score >= 0.95:
            return None  # No significant bias to report

        # Simulate category breakdown (in real implementation, extract from metric metadata)
        categories = ["Gender", "Race", "Age", "Religion", "Disability", "Sexual Orientation"]
        scores = [bias_score + (i * 0.02) for i in range(len(categories))]
        scores = [min(1.0, s) for s in scores]  # Cap at 1.0

        return {
            "categories": categories,
            "scores": scores
        }

    def _create_bias_heatmap_chart(self, bias_data: Dict[str, Any]) -> Chart:
        """Create heatmap showing bias detection across categories."""
        categories = bias_data["categories"]
        scores = [round(s, 3) for s in bias_data["scores"]]

        # Color code based on score
        colors = []
        for score in scores:
            if score >= 0.85:
                colors.append("#4CAF50")  # Green
            elif score >= 0.7:
                colors.append("#FF9800")  # Orange
            else:
                colors.append("#F44336")  # Red

        return Chart(
            title="Bias Detection by Category",
            chart_type=ChartType.BAR,
            data={
                "categories": categories,
                "scores": scores
            },
            labels=categories,
            colors=colors,
            options={
                "show_values": True,
                "horizontal": True,
                "max_value": 1.0
            },
            description="Bias detection scores across different categories"
        )

    def _extract_toxicity_data(self, safety_metrics: Dict[str, Any]) -> Optional[Dict[str, Any]]:
        """
        Extract toxicity breakdown data.

        Args:
            safety_metrics: Safety metrics dictionary

        Returns:
            Toxicity data or None if not available
        """
        toxicity_score = safety_metrics.get("toxicity_score", 1.0)

        if toxicity_score >= 0.95:
            return None  # No significant toxicity to report

        # Simulate toxicity breakdown (in real implementation, extract from metric metadata)
        types = ["Profanity", "Hate Speech", "Harassment", "Insults", "Threats"]
        # Inverse scores (lower toxicity score means more toxicity detected)
        base_toxicity = 1.0 - toxicity_score
        scores = [base_toxicity * (0.8 + i * 0.1) for i in range(len(types))]

        return {
            "types": types,
            "scores": scores
        }

    def _create_toxicity_breakdown_chart(self, toxicity_data: Dict[str, Any]) -> Chart:
        """Create stacked bar chart showing toxicity breakdown."""
        types = toxicity_data["types"]
        scores = [round(s, 3) for s in toxicity_data["scores"]]

        # Use red shades for toxicity
        colors = ["#FFCDD2", "#EF9A9A", "#E57373", "#EF5350", "#F44336"]

        return Chart(
            title="Toxicity Breakdown",
            chart_type=ChartType.BAR,
            data={
                "types": types,
                "scores": scores
            },
            labels=types,
            colors=colors[:len(types)],
            options={
                "show_values": True,
                "horizontal": False,
                "stacked": False
            },
            description="Breakdown of toxicity types detected"
        )

    def _extract_injection_data(self, safety_metrics: Dict[str, Any]) -> Optional[Dict[str, Any]]:
        """
        Extract prompt injection attempt data.

        Args:
            safety_metrics: Safety metrics dictionary

        Returns:
            Injection data or None if not available
        """
        injection_score = safety_metrics.get("injection_detection", 1.0)
        injection_count = safety_metrics.get("injection_score_count", 0)

        if injection_score >= 0.95 or injection_count == 0:
            return None  # No significant injection attempts to report

        # Calculate attempts detected (lower score means more attempts)
        attempts_detected = int((1.0 - injection_score) * injection_count)

        return {
            "total_checks": injection_count,
            "attempts_detected": attempts_detected,
            "detection_rate": 1.0 - injection_score
        }

    def _create_injection_attempts_chart(self, injection_data: Dict[str, Any]) -> Chart:
        """Create chart showing prompt injection attempts."""
        total = injection_data["total_checks"]
        detected = injection_data["attempts_detected"]
        clean = total - detected

        return Chart(
            title="Prompt Injection Detection",
            chart_type=ChartType.PIE,
            data={
                "labels": ["Clean Inputs", "Injection Attempts"],
                "values": [clean, detected]
            },
            labels=[f"Clean ({clean})", f"Attempts ({detected})"],
            colors=["#4CAF50", "#F44336"],
            options={
                "show_percentages": True,
                "show_legend": True
            },
            description=f"Detected {detected} prompt injection attempts out of {total} inputs"
        )

    def _create_safety_trends_chart(
        self,
        results: List[EvaluationResult],
        safety_metrics: Dict[str, Any]
    ) -> Optional[Chart]:
        """
        Create line chart showing safety trends over time.

        Args:
            results: List of evaluation results
            safety_metrics: Safety metrics dictionary

        Returns:
            Chart with safety trends or None if cannot be calculated
        """
        # Group results by timestamp (simplified - assumes results are ordered)
        # In real implementation, would group by time buckets

        if len(results) < 2:
            return None

        # Sample every N results to create trend points
        sample_size = max(1, len(results) // 10)  # Up to 10 data points
        trend_points = []

        for i in range(0, len(results), sample_size):
            batch = results[i:i+sample_size]
            batch_metrics = self._extract_safety_metrics(batch)
            trend_points.append({
                "index": i // sample_size,
                "overall_safety": batch_metrics.get("overall_safety", 1.0),
                "bias_score": batch_metrics.get("bias_score", 1.0),
                "toxicity_score": batch_metrics.get("toxicity_score", 1.0)
            })

        if len(trend_points) < 2:
            return None

        indices = [p["index"] for p in trend_points]

        return Chart(
            title="Safety Trends Over Time",
            chart_type=ChartType.LINE,
            data={
                "indices": indices,
                "overall_safety": [round(p["overall_safety"], 3) for p in trend_points],
                "bias_score": [round(p["bias_score"], 3) for p in trend_points],
                "toxicity_score": [round(p["toxicity_score"], 3) for p in trend_points]
            },
            labels=[f"Batch {i}" for i in indices],
            colors=["#2196F3", "#FF9800", "#9C27B0"],
            options={
                "show_legend": True,
                "show_points": True,
                "max_value": 1.0
            },
            description="Safety metric trends across evaluation batches"
        )

    def _generate_safety_summary(
        self,
        results: List[EvaluationResult],
        safety_metrics: Dict[str, Any],
        safety_concerns: List[Dict[str, Any]]
    ) -> str:
        """
        Generate executive summary for safety report.

        Args:
            results: List of evaluation results
            safety_metrics: Safety metrics
            safety_concerns: List of safety concerns

        Returns:
            Safety summary string
        """
        overall_safety = safety_metrics.get("overall_safety", 1.0)
        total_evals = len(results)

        summary_parts = []

        # Overall safety assessment
        summary_parts.append(
            f"Analyzed {total_evals} evaluations with overall safety score of {overall_safety:.1%}."
        )

        # Safety concerns
        if safety_concerns:
            critical_count = sum(1 for c in safety_concerns if c["severity"] == "critical")
            high_count = sum(1 for c in safety_concerns if c["severity"] == "high")

            if critical_count > 0:
                summary_parts.append(
                    f"{critical_count} critical safety concern(s) detected requiring immediate attention."
                )
            if high_count > 0:
                summary_parts.append(
                    f"{high_count} high-priority safety concern(s) identified."
                )
        else:
            summary_parts.append("No critical safety concerns detected.")

        # Specific safety metrics
        bias_score = safety_metrics.get("bias_score", 1.0)
        toxicity_score = safety_metrics.get("toxicity_score", 1.0)

        if bias_score < 0.85:
            summary_parts.append(
                f"Bias detection score ({bias_score:.1%}) indicates potential bias in responses."
            )

        if toxicity_score < 0.85:
            summary_parts.append(
                f"Toxicity score ({toxicity_score:.1%}) indicates presence of toxic content."
            )

        # Compliance note
        if overall_safety >= 0.85 and not any(c["severity"] == "critical" for c in safety_concerns):
            summary_parts.append("Agent meets responsible AI safety standards for deployment.")
        else:
            summary_parts.append("Agent requires safety improvements before production deployment.")

        return " ".join(summary_parts)

generate_safety_dashboard(results, title, run_id=None, experiment_id=None, include_recommendations=True, description=None)

Generate a safety dashboard from evaluation results.

Parameters:

Name	Type	Description	Default
results	List[EvaluationResult]	List of evaluation results to analyze	required
title	str	Dashboard title	required
run_id	Optional[UUID]	Optional experiment run ID	None
experiment_id	Optional[UUID]	Optional experiment ID	None
include_recommendations	bool	Whether to include safety recommendations	True
description	Optional[str]	Optional dashboard description	None

Returns:

Type	Description
Dashboard	Dashboard with safety analysis charts

Raises:

Type	Description
ValueError	If results list is empty
ValueError	If title is empty

Examples:

>>> generator = SafetyReportGenerator()
>>> results = [result1, result2, result3]
>>> dashboard = generator.generate_safety_dashboard(
...     results=results,
...     title="Safety Report - Q1 2024"
... )

Source code in uaef/reporting/safety.py

def generate_safety_dashboard(
    self,
    results: List[EvaluationResult],
    title: str,
    run_id: Optional[UUID] = None,
    experiment_id: Optional[UUID] = None,
    include_recommendations: bool = True,
    description: Optional[str] = None
) -> Dashboard:
    """
    Generate a safety dashboard from evaluation results.

    Args:
        results: List of evaluation results to analyze
        title: Dashboard title
        run_id: Optional experiment run ID
        experiment_id: Optional experiment ID
        include_recommendations: Whether to include safety recommendations
        description: Optional dashboard description

    Returns:
        Dashboard with safety analysis charts

    Raises:
        ValueError: If results list is empty
        ValueError: If title is empty

    Examples:
        >>> generator = SafetyReportGenerator()
        >>> results = [result1, result2, result3]
        >>> dashboard = generator.generate_safety_dashboard(
        ...     results=results,
        ...     title="Safety Report - Q1 2024"
        ... )
    """
    if not results:
        raise ValueError("Cannot generate safety dashboard from empty results list")

    if not title or not title.strip():
        raise ValueError("Dashboard title cannot be empty")

    logger.info(f"Generating safety dashboard for {len(results)} evaluation results")

    # Extract safety metrics
    safety_metrics = self._extract_safety_metrics(results)

    # Identify safety concerns
    safety_concerns = self._identify_safety_concerns(safety_metrics, results)

    # Generate charts
    charts = []

    # Chart 1: Overall safety metrics
    charts.append(self._create_safety_overview_chart(safety_metrics))

    # Chart 2: Severity distribution
    if safety_concerns:
        charts.append(self._create_severity_distribution_chart(safety_concerns))

    # Chart 3: Bias detection heatmap
    bias_data = self._extract_bias_data(safety_metrics)
    if bias_data:
        charts.append(self._create_bias_heatmap_chart(bias_data))

    # Chart 4: Toxicity breakdown
    toxicity_data = self._extract_toxicity_data(safety_metrics)
    if toxicity_data:
        charts.append(self._create_toxicity_breakdown_chart(toxicity_data))

    # Chart 5: Prompt injection attempts
    injection_data = self._extract_injection_data(safety_metrics)
    if injection_data:
        charts.append(self._create_injection_attempts_chart(injection_data))

    # Chart 6: Safety trends (if multiple results)
    if len(results) > 1:
        try:
            trend_chart = self._create_safety_trends_chart(results, safety_metrics)
            if trend_chart:
                charts.append(trend_chart)
        except Exception as e:
            logger.warning(f"Failed to generate safety trends chart: {e}")

    # Prepare metadata
    metadata = {
        "dashboard_type": "safety",
        "evaluation_count": len(results),
        "generated_at": datetime.utcnow().isoformat(),
        "safety_metrics": safety_metrics,
        "safety_concerns_count": len(safety_concerns),
        "safety_concerns": safety_concerns
    }

    if run_id:
        metadata["run_id"] = str(run_id)
    if experiment_id:
        metadata["experiment_id"] = str(experiment_id)

    # Add recommendations if enabled
    if include_recommendations:
        try:
            # Filter for safety-related recommendations
            rec_report = self.recommendation_engine.generate_recommendations(results)
            safety_recommendations = [
                {
                    "title": rec.title,
                    "priority": rec.priority,
                    "category": rec.category,
                    "description": rec.description
                }
                for rec in rec_report.recommendations
                if rec.category in ["safety", "bias", "toxicity", "responsible_ai"]
            ][:5]  # Top 5 safety recommendations

            if safety_recommendations:
                metadata["safety_recommendations"] = safety_recommendations
        except Exception as e:
            logger.warning(f"Failed to generate safety recommendations: {e}")

    # Generate description if not provided
    if not description:
        description = self._generate_safety_summary(
            results, safety_metrics, safety_concerns
        )

    # Create dashboard
    dashboard = Dashboard(
        title=title,
        description=description,
        charts=charts,
        metadata=metadata
    )

    logger.info(f"Generated safety dashboard with {len(charts)} charts and {len(safety_concerns)} concerns")
    return dashboard

uaef.reporting.regression

Regression report generator for detecting and analyzing metric regressions.

RegressionReportGenerator

Generator for detailed regression reports when metrics decline.

Creates comprehensive regression dashboards showing: - All regressed metrics with before/after comparisons - Context about when regression occurred (run metadata, timestamps) - Root cause analysis for regressions - Severity classifications (critical, high, medium, low) - Impact on overall system performance - Actionable recommendations for fixing regressions - Historical regression tracking (if multiple runs provided)

Integrates with ComparisonEngine for regression detection, RootCauseAnalyzer for root cause analysis, and RecommendationEngine for regression-specific recommendations.

Severity Classification: - Critical: Regression > 20% or key metrics below 0.5 - High: Regression 10-20% or important metrics below 0.7 - Medium: Regression 5-10% or metrics below 0.85 - Low: Regression < 5% or minor metrics affected

Examples:

>>> # Generate regression report from two runs
>>> generator = RegressionReportGenerator()
>>> dashboard = generator.generate_regression_dashboard(
...     baseline_run=baseline_run,
...     current_run=current_run,
...     title="Regression Report - v1 to v2"
... )
>>> print(f"Regressions: {len(dashboard.metadata.get('regressions', []))}")

>>> # Generate with custom thresholds
>>> generator = RegressionReportGenerator(
...     regression_threshold=10.0,
...     critical_threshold=0.5
... )
>>> dashboard = generator.generate_regression_dashboard(
...     baseline_run=baseline_run,
...     current_run=current_run,
...     title="Critical Regression Analysis"
... )

>>> # Generate historical regression tracking
>>> dashboard = generator.generate_historical_regression_dashboard(
...     baseline_run=baseline_run,
...     comparison_runs=[run1, run2, run3],
...     title="Regression Tracking - Last 4 Runs"
... )

Source code in uaef/reporting/regression.py

class RegressionReportGenerator:
    """
    Generator for detailed regression reports when metrics decline.

    Creates comprehensive regression dashboards showing:
    - All regressed metrics with before/after comparisons
    - Context about when regression occurred (run metadata, timestamps)
    - Root cause analysis for regressions
    - Severity classifications (critical, high, medium, low)
    - Impact on overall system performance
    - Actionable recommendations for fixing regressions
    - Historical regression tracking (if multiple runs provided)

    Integrates with ComparisonEngine for regression detection,
    RootCauseAnalyzer for root cause analysis, and RecommendationEngine
    for regression-specific recommendations.

    Severity Classification:
    - Critical: Regression > 20% or key metrics below 0.5
    - High: Regression 10-20% or important metrics below 0.7
    - Medium: Regression 5-10% or metrics below 0.85
    - Low: Regression < 5% or minor metrics affected

    Examples:
        >>> # Generate regression report from two runs
        >>> generator = RegressionReportGenerator()
        >>> dashboard = generator.generate_regression_dashboard(
        ...     baseline_run=baseline_run,
        ...     current_run=current_run,
        ...     title="Regression Report - v1 to v2"
        ... )
        >>> print(f"Regressions: {len(dashboard.metadata.get('regressions', []))}")

        >>> # Generate with custom thresholds
        >>> generator = RegressionReportGenerator(
        ...     regression_threshold=10.0,
        ...     critical_threshold=0.5
        ... )
        >>> dashboard = generator.generate_regression_dashboard(
        ...     baseline_run=baseline_run,
        ...     current_run=current_run,
        ...     title="Critical Regression Analysis"
        ... )

        >>> # Generate historical regression tracking
        >>> dashboard = generator.generate_historical_regression_dashboard(
        ...     baseline_run=baseline_run,
        ...     comparison_runs=[run1, run2, run3],
        ...     title="Regression Tracking - Last 4 Runs"
        ... )
    """

    def __init__(
        self,
        comparison_engine: Optional[ComparisonEngine] = None,
        root_cause_analyzer: Optional[RootCauseAnalyzer] = None,
        recommendation_engine: Optional[RecommendationEngine] = None,
        regression_threshold: float = 5.0,
        critical_threshold: float = 0.5,
        high_threshold: float = 0.7,
        medium_threshold: float = 0.85
    ):
        """
        Initialize the regression report generator.

        Args:
            comparison_engine: Optional ComparisonEngine for regression detection
            root_cause_analyzer: Optional RootCauseAnalyzer for root cause analysis
            recommendation_engine: Optional RecommendationEngine for recommendations
            regression_threshold: Percentage threshold for regression detection (default: 5.0%)
            critical_threshold: Score threshold for critical severity (default: 0.5)
            high_threshold: Score threshold for high severity (default: 0.7)
            medium_threshold: Score threshold for medium severity (default: 0.85)
        """
        self.comparison_engine = comparison_engine or ComparisonEngine(
            regression_threshold=regression_threshold
        )
        self.root_cause_analyzer = root_cause_analyzer or RootCauseAnalyzer()
        self.recommendation_engine = recommendation_engine or RecommendationEngine()
        self.regression_threshold = regression_threshold
        self.critical_threshold = critical_threshold
        self.high_threshold = high_threshold
        self.medium_threshold = medium_threshold

    def generate_regression_dashboard(
        self,
        baseline_run: ExperimentRun,
        current_run: ExperimentRun,
        title: str,
        description: Optional[str] = None,
        include_root_cause: bool = True,
        include_recommendations: bool = True
    ) -> Dashboard:
        """
        Generate a regression dashboard comparing two experiment runs.

        Args:
            baseline_run: Baseline experiment run
            current_run: Current experiment run to check for regressions
            title: Dashboard title
            description: Optional dashboard description
            include_root_cause: Whether to include root cause analysis
            include_recommendations: Whether to include recommendations

        Returns:
            Dashboard with regression analysis charts

        Raises:
            ValueError: If title is empty
            ValueError: If baseline and current runs are the same
            ValueError: If no regressions detected

        Examples:
            >>> generator = RegressionReportGenerator()
            >>> dashboard = generator.generate_regression_dashboard(
            ...     baseline_run=baseline_run,
            ...     current_run=current_run,
            ...     title="Regression Report - v1 to v2"
            ... )
        """
        if not title or not title.strip():
            raise ValueError("Dashboard title cannot be empty")

        if baseline_run.run_id == current_run.run_id:
            raise ValueError("Cannot compare a run with itself")

        logger.info(
            f"Generating regression dashboard between runs {baseline_run.run_id} "
            f"and {current_run.run_id}"
        )

        # Generate comparison report
        comparison_report = self.comparison_engine.compare_runs(
            baseline_run=baseline_run,
            current_run=current_run
        )

        # Check if there are any regressions
        if not comparison_report.regressions:
            raise ValueError(
                f"No regressions detected between runs {baseline_run.run_id} "
                f"and {current_run.run_id}"
            )

        # Classify regressions by severity
        regression_data = self._classify_regressions(comparison_report)

        # Generate charts
        charts = []

        # Chart 1: Regression overview (bar chart showing all regressed metrics)
        charts.append(self._create_regression_overview_chart(regression_data))

        # Chart 2: Before/after comparison (grouped bar chart)
        charts.append(self._create_before_after_chart(regression_data))

        # Chart 3: Severity distribution (pie chart)
        charts.append(self._create_severity_distribution_chart(regression_data))

        # Chart 4: Impact analysis (heatmap showing affected dimensions)
        charts.append(self._create_impact_heatmap(regression_data))

        # Chart 5: Regression magnitude (bar chart showing percentage changes)
        charts.append(self._create_regression_magnitude_chart(regression_data))

        # Prepare metadata
        metadata = {
            "dashboard_type": "regression",
            "baseline_run_id": str(baseline_run.run_id),
            "current_run_id": str(current_run.run_id),
            "generated_at": datetime.utcnow().isoformat(),
            "regression_threshold": self.regression_threshold,
            "total_regressions": len(comparison_report.regressions),
            "critical_count": regression_data["severity_counts"]["critical"],
            "high_count": regression_data["severity_counts"]["high"],
            "medium_count": regression_data["severity_counts"]["medium"],
            "low_count": regression_data["severity_counts"]["low"],
            "regressions": regression_data["regressions"]
        }

        # Add context about when regression occurred
        metadata["baseline_context"] = {
            "run_name": baseline_run.run_name,
            "timestamp": baseline_run.timestamp.isoformat() if baseline_run.timestamp else None,
            "config_snapshot": baseline_run.config_snapshot
        }
        metadata["current_context"] = {
            "run_name": current_run.run_name,
            "timestamp": current_run.timestamp.isoformat() if current_run.timestamp else None,
            "config_snapshot": current_run.config_snapshot
        }

        # Add root cause analysis if enabled
        if include_root_cause:
            try:
                root_causes = self._analyze_root_causes(
                    regression_data, baseline_run, current_run
                )
                metadata["root_causes"] = root_causes

                # Add root cause breakdown chart
                if root_causes:
                    charts.append(self._create_root_cause_chart(root_causes))
            except Exception as e:
                logger.warning(f"Failed to generate root cause analysis: {e}")

        # Add recommendations if enabled
        if include_recommendations:
            try:
                recommendations = self._generate_regression_recommendations(
                    regression_data, comparison_report
                )
                metadata["recommendations"] = recommendations
            except Exception as e:
                logger.warning(f"Failed to generate recommendations: {e}")

        # Generate description if not provided
        if not description:
            description = self._generate_regression_summary(
                regression_data, baseline_run, current_run
            )

        # Create dashboard
        dashboard = Dashboard(
            title=title,
            description=description,
            charts=charts,
            metadata=metadata
        )

        logger.info(
            f"Generated regression dashboard with {len(charts)} charts and "
            f"{len(comparison_report.regressions)} regressions"
        )
        return dashboard

    def generate_historical_regression_dashboard(
        self,
        baseline_run: ExperimentRun,
        comparison_runs: List[ExperimentRun],
        title: str,
        description: Optional[str] = None
    ) -> Dashboard:
        """
        Generate dashboard tracking regressions across multiple runs.

        Args:
            baseline_run: Baseline experiment run
            comparison_runs: List of runs to track regressions across
            title: Dashboard title
            description: Optional dashboard description

        Returns:
            Dashboard with historical regression tracking

        Raises:
            ValueError: If comparison_runs is empty
            ValueError: If title is empty

        Examples:
            >>> generator = RegressionReportGenerator()
            >>> dashboard = generator.generate_historical_regression_dashboard(
            ...     baseline_run=baseline_run,
            ...     comparison_runs=[run1, run2, run3],
            ...     title="Regression Tracking - Last 4 Runs"
            ... )
        """
        if not comparison_runs:
            raise ValueError("comparison_runs cannot be empty")

        if not title or not title.strip():
            raise ValueError("Dashboard title cannot be empty")

        logger.info(
            f"Generating historical regression dashboard for {len(comparison_runs)} runs"
        )

        # Generate comparison reports for all runs
        all_reports = []
        for run in comparison_runs:
            try:
                report = self.comparison_engine.compare_runs(
                    baseline_run=baseline_run,
                    current_run=run
                )
                all_reports.append((run, report))
            except Exception as e:
                logger.warning(f"Failed to compare run {run.run_id}: {e}")

        if not all_reports:
            raise ValueError("Failed to generate any comparison reports")

        # Generate charts
        charts = []

        # Chart 1: Regression timeline (line chart showing regression count over time)
        charts.append(self._create_regression_timeline_chart(baseline_run, all_reports))

        # Chart 2: Metric regression heatmap (which metrics regressed in which runs)
        charts.append(self._create_metric_regression_heatmap(all_reports))

        # Chart 3: Severity trends (stacked area chart)
        charts.append(self._create_severity_trends_chart(all_reports))

        # Prepare metadata
        metadata = {
            "dashboard_type": "historical_regression",
            "baseline_run_id": str(baseline_run.run_id),
            "comparison_run_ids": [str(run.run_id) for run, _ in all_reports],
            "generated_at": datetime.utcnow().isoformat(),
            "run_count": len(all_reports),
            "total_regressions": sum(len(report.regressions) for _, report in all_reports)
        }

        # Generate description if not provided
        if not description:
            description = self._generate_historical_summary(baseline_run, all_reports)

        # Create dashboard
        dashboard = Dashboard(
            title=title,
            description=description,
            charts=charts,
            metadata=metadata
        )

        logger.info(f"Generated historical regression dashboard with {len(charts)} charts")
        return dashboard

    def _classify_regressions(
        self, comparison_report: ComparisonReport
    ) -> Dict[str, Any]:
        """
        Classify regressions by severity and extract detailed information.

        Severity levels:
        - Critical: Regression > 20% or current score < 0.5
        - High: Regression 10-20% or current score < 0.7
        - Medium: Regression 5-10% or current score < 0.85
        - Low: Regression < 5% or minor metrics

        Args:
            comparison_report: Comparison report with regressions

        Returns:
            Dictionary with classified regression data
        """
        regressions = []
        severity_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0}

        for comp in comparison_report.metric_comparisons:
            if not comp.is_regression:
                continue

            # Determine severity
            severity = self._determine_severity(comp)
            severity_counts[severity] += 1

            regressions.append({
                "metric_name": comp.metric_name,
                "baseline_score": comp.baseline_score,
                "current_score": comp.current_score,
                "delta": comp.delta,
                "percent_change": comp.percent_change,
                "severity": severity
            })

        # Sort by severity (critical first) then by magnitude
        severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
        regressions.sort(
            key=lambda x: (severity_order[x["severity"]], abs(x["delta"])),
            reverse=True
        )

        return {
            "regressions": regressions,
            "severity_counts": severity_counts
        }

    def _determine_severity(self, comparison: MetricComparison) -> str:
        """
        Determine severity level for a regression.

        Args:
            comparison: Metric comparison with regression

        Returns:
            Severity level (critical, high, medium, low)
        """
        # Check if current score is critically low
        if comparison.current_score < self.critical_threshold:
            return "critical"

        # Check percentage change magnitude
        if abs(comparison.percent_change) > 20:
            return "critical"
        elif abs(comparison.percent_change) > 10:
            return "high"
        elif abs(comparison.percent_change) > 5:
            return "medium"

        # Check if current score is below thresholds
        if comparison.current_score < self.high_threshold:
            return "high"
        elif comparison.current_score < self.medium_threshold:
            return "medium"

        return "low"

    def _create_regression_overview_chart(
        self, regression_data: Dict[str, Any]
    ) -> Chart:
        """Create bar chart showing all regressed metrics."""
        regressions = regression_data["regressions"]

        if not regressions:
            return Chart(
                title="Regression Overview",
                chart_type=ChartType.BAR,
                data={"metrics": [], "deltas": []},
                labels=[],
                colors=[],
                description="No regressions detected"
            )

        # Take top 15 regressions by magnitude
        top_regressions = regressions[:15]

        metrics = [r["metric_name"] for r in top_regressions]
        deltas = [round(r["delta"], 3) for r in top_regressions]

        # Color code by severity
        severity_colors = {
            "critical": "#D32F2F",
            "high": "#F44336",
            "medium": "#FF9800",
            "low": "#FFC107"
        }
        colors = [severity_colors[r["severity"]] for r in top_regressions]

        return Chart(
            title="Regression Overview",
            chart_type=ChartType.BAR,
            data={
                "metrics": metrics,
                "deltas": deltas
            },
            labels=metrics,
            colors=colors,
            options={
                "show_values": True,
                "horizontal": True,
                "show_zero_line": True
            },
            description=f"Top {len(top_regressions)} regressed metrics by magnitude"
        )

    def _create_before_after_chart(
        self, regression_data: Dict[str, Any]
    ) -> Chart:
        """Create grouped bar chart showing before/after comparison."""
        regressions = regression_data["regressions"]

        if not regressions:
            return Chart(
                title="Before/After Comparison",
                chart_type=ChartType.BAR,
                data={"metrics": [], "baseline": [], "current": []},
                labels=[],
                colors=[],
                description="No regressions to compare"
            )

        # Take top 10 regressions
        top_regressions = regressions[:10]

        metrics = [r["metric_name"] for r in top_regressions]
        baseline_scores = [round(r["baseline_score"], 3) for r in top_regressions]
        current_scores = [round(r["current_score"], 3) for r in top_regressions]

        return Chart(
            title="Before/After Comparison",
            chart_type=ChartType.BAR,
            data={
                "metrics": metrics,
                "baseline": baseline_scores,
                "current": current_scores
            },
            labels=metrics,
            colors=["#2196F3", "#F44336"],
            options={
                "grouped": True,
                "horizontal": True,
                "show_values": True,
                "max_value": 1.0,
                "legend": ["Baseline", "Current"]
            },
            description=f"Before/after scores for top {len(top_regressions)} regressions"
        )

    def _create_severity_distribution_chart(
        self, regression_data: Dict[str, Any]
    ) -> Chart:
        """Create pie chart showing severity distribution."""
        severity_counts = regression_data["severity_counts"]

        # Filter out zero counts
        severities = []
        counts = []
        for severity in ["critical", "high", "medium", "low"]:
            count = severity_counts[severity]
            if count > 0:
                severities.append(severity)
                counts.append(count)

        if not severities:
            return Chart(
                title="Severity Distribution",
                chart_type=ChartType.PIE,
                data={"severities": [], "counts": []},
                labels=[],
                colors=[],
                description="No regressions to classify"
            )

        severity_colors = {
            "critical": "#D32F2F",
            "high": "#F44336",
            "medium": "#FF9800",
            "low": "#FFC107"
        }
        colors = [severity_colors[sev] for sev in severities]

        return Chart(
            title="Severity Distribution",
            chart_type=ChartType.PIE,
            data={
                "severities": severities,
                "counts": counts
            },
            labels=[f"{sev.capitalize()} ({count})" for sev, count in zip(severities, counts)],
            colors=colors,
            options={
                "show_percentages": True,
                "show_legend": True
            },
            description=f"Distribution of {sum(counts)} regressions by severity"
        )

    def _create_impact_heatmap(
        self, regression_data: Dict[str, Any]
    ) -> Chart:
        """Create heatmap showing affected dimensions."""
        regressions = regression_data["regressions"]

        if not regressions:
            return Chart(
                title="Impact Analysis by Dimension",
                chart_type=ChartType.HEATMAP,
                data={"matrix": [], "x_labels": [], "y_labels": []},
                labels=[],
                colors=[],
                description="No regressions to analyze"
            )

        # Group regressions by dimension
        dimension_regressions = defaultdict(lambda: {"critical": 0, "high": 0, "medium": 0, "low": 0})

        for reg in regressions:
            # Determine dimension from metric name
            dimension = self._get_dimension_from_metric(reg["metric_name"])
            dimension_regressions[dimension][reg["severity"]] += 1

        # Build heatmap matrix
        dimensions = sorted(dimension_regressions.keys())
        severities = ["critical", "high", "medium", "low"]

        matrix = []
        for dimension in dimensions:
            row = [dimension_regressions[dimension][sev] for sev in severities]
            matrix.append(row)

        return Chart(
            title="Impact Analysis by Dimension",
            chart_type=ChartType.HEATMAP,
            data={
                "matrix": matrix,
                "x_labels": [s.capitalize() for s in severities],
                "y_labels": dimensions
            },
            labels=dimensions,
            colors=["#FFFFFF", "#F44336"],
            options={
                "show_values": True,
                "color_scale": "Reds"
            },
            description="Regression severity distribution across dimensions"
        )

    def _create_regression_magnitude_chart(
        self, regression_data: Dict[str, Any]
    ) -> Chart:
        """Create bar chart showing percentage changes."""
        regressions = regression_data["regressions"]

        if not regressions:
            return Chart(
                title="Regression Magnitude (% Change)",
                chart_type=ChartType.BAR,
                data={"metrics": [], "percent_changes": []},
                labels=[],
                colors=[],
                description="No regressions to display"
            )

        # Take top 15 regressions by percentage change
        top_regressions = sorted(
            regressions,
            key=lambda x: abs(x["percent_change"]),
            reverse=True
        )[:15]

        metrics = [r["metric_name"] for r in top_regressions]
        percent_changes = [round(r["percent_change"], 2) for r in top_regressions]

        # Color code by severity
        severity_colors = {
            "critical": "#D32F2F",
            "high": "#F44336",
            "medium": "#FF9800",
            "low": "#FFC107"
        }
        colors = [severity_colors[r["severity"]] for r in top_regressions]

        return Chart(
            title="Regression Magnitude (% Change)",
            chart_type=ChartType.BAR,
            data={
                "metrics": metrics,
                "percent_changes": percent_changes
            },
            labels=metrics,
            colors=colors,
            options={
                "show_values": True,
                "horizontal": True,
                "show_zero_line": True,
                "value_suffix": "%"
            },
            description=f"Percentage change for top {len(top_regressions)} regressions"
        )

    def _create_root_cause_chart(
        self, root_causes: List[Dict[str, Any]]
    ) -> Chart:
        """Create bar chart showing root cause breakdown."""
        if not root_causes:
            return Chart(
                title="Root Cause Analysis",
                chart_type=ChartType.BAR,
                data={"causes": [], "counts": []},
                labels=[],
                colors=[],
                description="No root causes identified"
            )

        # Count occurrences of each root cause category
        cause_counts = defaultdict(int)
        for cause in root_causes:
            cause_counts[cause["category"]] += 1

        # Sort by count
        sorted_causes = sorted(
            cause_counts.items(),
            key=lambda x: x[1],
            reverse=True
        )

        categories = [c[0] for c in sorted_causes]
        counts = [c[1] for c in sorted_causes]

        # Color code by category
        category_colors = {
            "tool_selection": "#FF5722",
            "response": "#2196F3",
            "context": "#4CAF50",
            "reasoning": "#9C27B0",
            "performance": "#FF9800",
            "safety": "#F44336"
        }
        colors = [category_colors.get(cat, "#9E9E9E") for cat in categories]

        return Chart(
            title="Root Cause Breakdown",
            chart_type=ChartType.BAR,
            data={
                "categories": categories,
                "counts": counts
            },
            labels=categories,
            colors=colors,
            options={
                "show_values": True,
                "horizontal": False
            },
            description=f"Root cause distribution across {len(root_causes)} regressions"
        )

    def _create_regression_timeline_chart(
        self,
        baseline_run: ExperimentRun,
        all_reports: List[Tuple[ExperimentRun, ComparisonReport]]
    ) -> Chart:
        """Create line chart showing regression count over time."""
        if not all_reports:
            return Chart(
                title="Regression Timeline",
                chart_type=ChartType.LINE,
                data={"runs": [], "regression_counts": []},
                labels=[],
                colors=[],
                description="No data available"
            )

        # Sort by run timestamp
        sorted_reports = sorted(
            all_reports,
            key=lambda x: x[0].timestamp if x[0].timestamp else datetime.min
        )

        run_labels = [f"Run {i+1}" for i in range(len(sorted_reports))]
        regression_counts = [len(report.regressions) for _, report in sorted_reports]

        return Chart(
            title="Regression Timeline",
            chart_type=ChartType.LINE,
            data={
                "runs": run_labels,
                "regression_counts": regression_counts
            },
            labels=run_labels,
            colors=["#F44336"],
            options={
                "show_points": True,
                "show_legend": False
            },
            description=f"Regression count across {len(sorted_reports)} runs"
        )

    def _create_metric_regression_heatmap(
        self,
        all_reports: List[Tuple[ExperimentRun, ComparisonReport]]
    ) -> Chart:
        """Create heatmap showing which metrics regressed in which runs."""
        if not all_reports:
            return Chart(
                title="Metric Regression Heatmap",
                chart_type=ChartType.HEATMAP,
                data={"matrix": [], "x_labels": [], "y_labels": []},
                labels=[],
                colors=[],
                description="No data available"
            )

        # Collect all metrics that regressed in any run
        all_regressed_metrics = set()
        for _, report in all_reports:
            all_regressed_metrics.update(report.regressions)

        if not all_regressed_metrics:
            return Chart(
                title="Metric Regression Heatmap",
                chart_type=ChartType.HEATMAP,
                data={"matrix": [], "x_labels": [], "y_labels": []},
                labels=[],
                colors=[],
                description="No regressions detected across runs"
            )

        # Sort metrics alphabetically
        metrics = sorted(all_regressed_metrics)

        # Build matrix (1 if metric regressed in run, 0 otherwise)
        run_labels = [f"Run {i+1}" for i in range(len(all_reports))]
        matrix = []

        for metric in metrics:
            row = []
            for _, report in all_reports:
                row.append(1 if metric in report.regressions else 0)
            matrix.append(row)

        return Chart(
            title="Metric Regression Heatmap",
            chart_type=ChartType.HEATMAP,
            data={
                "matrix": matrix,
                "x_labels": run_labels,
                "y_labels": metrics
            },
            labels=metrics,
            colors=["#FFFFFF", "#F44336"],
            options={
                "show_values": False,
                "color_scale": "Reds"
            },
            description="Which metrics regressed in which runs"
        )

    def _create_severity_trends_chart(
        self,
        all_reports: List[Tuple[ExperimentRun, ComparisonReport]]
    ) -> Chart:
        """Create stacked area chart showing severity trends."""
        if not all_reports:
            return Chart(
                title="Severity Trends",
                chart_type=ChartType.BAR,
                data={"runs": [], "critical": [], "high": [], "medium": [], "low": []},
                labels=[],
                colors=[],
                description="No data available"
            )

        # Sort by run timestamp
        sorted_reports = sorted(
            all_reports,
            key=lambda x: x[0].timestamp if x[0].timestamp else datetime.min
        )

        run_labels = [f"Run {i+1}" for i in range(len(sorted_reports))]

        # Count regressions by severity for each run
        critical_counts = []
        high_counts = []
        medium_counts = []
        low_counts = []

        for _, report in sorted_reports:
            # Classify regressions for this run
            regression_data = self._classify_regressions(report)
            severity_counts = regression_data["severity_counts"]

            critical_counts.append(severity_counts["critical"])
            high_counts.append(severity_counts["high"])
            medium_counts.append(severity_counts["medium"])
            low_counts.append(severity_counts["low"])

        return Chart(
            title="Severity Trends",
            chart_type=ChartType.BAR,
            data={
                "runs": run_labels,
                "critical": critical_counts,
                "high": high_counts,
                "medium": medium_counts,
                "low": low_counts
            },
            labels=run_labels,
            colors=["#D32F2F", "#F44336", "#FF9800", "#FFC107"],
            options={
                "stacked": True,
                "show_legend": True,
                "legend": ["Critical", "High", "Medium", "Low"]
            },
            description="Regression severity distribution over time"
        )

    def _analyze_root_causes(
        self,
        regression_data: Dict[str, Any],
        baseline_run: ExperimentRun,
        current_run: ExperimentRun
    ) -> List[Dict[str, Any]]:
        """
        Analyze root causes for regressions.

        Args:
            regression_data: Classified regression data
            baseline_run: Baseline run
            current_run: Current run

        Returns:
            List of root cause analyses
        """
        root_causes = []

        for reg in regression_data["regressions"]:
            # Determine category from metric name
            category = self._get_dimension_from_metric(reg["metric_name"])

            # Generate root cause description
            description = self._generate_root_cause_description(reg, category)

            root_causes.append({
                "metric_name": reg["metric_name"],
                "category": category,
                "severity": reg["severity"],
                "description": description,
                "delta": reg["delta"],
                "percent_change": reg["percent_change"]
            })

        return root_causes

    def _generate_regression_recommendations(
        self,
        regression_data: Dict[str, Any],
        comparison_report: ComparisonReport
    ) -> List[Dict[str, Any]]:
        """
        Generate actionable recommendations for fixing regressions.

        Args:
            regression_data: Classified regression data
            comparison_report: Comparison report

        Returns:
            List of recommendations
        """
        recommendations = []

        # Group regressions by category
        category_regressions = defaultdict(list)
        for reg in regression_data["regressions"]:
            category = self._get_dimension_from_metric(reg["metric_name"])
            category_regressions[category].append(reg)

        # Generate recommendations per category
        for category, regs in category_regressions.items():
            # Count critical and high severity regressions
            critical_count = sum(1 for r in regs if r["severity"] == "critical")
            high_count = sum(1 for r in regs if r["severity"] == "high")

            if critical_count > 0 or high_count > 0:
                priority = "critical" if critical_count > 0 else "high"

                recommendation = {
                    "category": category,
                    "priority": priority,
                    "title": f"Address {category} regressions",
                    "description": self._generate_category_recommendation(category, regs),
                    "affected_metrics": [r["metric_name"] for r in regs],
                    "severity_breakdown": {
                        "critical": critical_count,
                        "high": high_count,
                        "medium": sum(1 for r in regs if r["severity"] == "medium"),
                        "low": sum(1 for r in regs if r["severity"] == "low")
                    }
                }
                recommendations.append(recommendation)

        # Sort by priority
        priority_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
        recommendations.sort(key=lambda x: priority_order[x["priority"]])

        return recommendations

    def _get_dimension_from_metric(self, metric_name: str) -> str:
        """
        Determine dimension from metric name.

        Args:
            metric_name: Name of the metric

        Returns:
            Dimension name
        """
        metric_lower = metric_name.lower()

        if any(prefix in metric_lower for prefix in ["tool_", "mcp_"]):
            return "Tool Calling"
        elif any(prefix in metric_lower for prefix in ["answer_", "completeness", "hallucination", "accuracy"]):
            return "Response Quality"
        elif any(prefix in metric_lower for prefix in ["safety_", "bias_", "toxicity", "prompt_injection"]):
            return "Responsible AI"
        elif any(prefix in metric_lower for prefix in ["latency_", "token_", "cost_", "throughput"]):
            return "Performance"
        elif any(prefix in metric_lower for prefix in ["context_", "coherence", "conversation_", "turn_"]):
            return "Multi-Turn"
        elif any(prefix in metric_lower for prefix in ["agent_", "delegation_", "workflow_", "coordination_"]):
            return "Multi-Agent"
        elif any(prefix in metric_lower for prefix in ["chain_of_thought", "logical_", "reasoning_", "fallacy"]):
            return "Reasoning"
        else:
            return "Other"

    def _generate_root_cause_description(
        self, regression: Dict[str, Any], category: str
    ) -> str:
        """
        Generate human-readable root cause description.

        Args:
            regression: Regression data
            category: Dimension category

        Returns:
            Root cause description
        """
        metric_name = regression["metric_name"]
        percent_change = regression["percent_change"]

        descriptions = {
            "Tool Calling": f"{metric_name} decreased by {abs(percent_change):.1f}%, indicating issues with tool selection or usage",
            "Response Quality": f"{metric_name} decreased by {abs(percent_change):.1f}%, suggesting response quality degradation",
            "Responsible AI": f"{metric_name} decreased by {abs(percent_change):.1f}%, indicating potential safety or bias concerns",
            "Performance": f"{metric_name} decreased by {abs(percent_change):.1f}%, showing performance degradation",
            "Multi-Turn": f"{metric_name} decreased by {abs(percent_change):.1f}%, indicating context retention or coherence issues",
            "Multi-Agent": f"{metric_name} decreased by {abs(percent_change):.1f}%, suggesting coordination or delegation problems",
            "Reasoning": f"{metric_name} decreased by {abs(percent_change):.1f}%, indicating reasoning quality degradation"
        }

        return descriptions.get(
            category,
            f"{metric_name} decreased by {abs(percent_change):.1f}%"
        )

    def _generate_category_recommendation(
        self, category: str, regressions: List[Dict[str, Any]]
    ) -> str:
        """
        Generate category-specific recommendation.

        Args:
            category: Dimension category
            regressions: List of regressions in this category

        Returns:
            Recommendation description
        """
        count = len(regressions)
        avg_change = sum(abs(r["percent_change"]) for r in regressions) / count

        recommendations = {
            "Tool Calling": (
                f"{count} tool calling metric(s) regressed by an average of {avg_change:.1f}%. "
                "Review tool selection logic, tool descriptions, and usage examples in the system prompt. "
                "Consider adding more tool usage examples or simplifying tool interfaces."
            ),
            "Response Quality": (
                f"{count} response quality metric(s) regressed by an average of {avg_change:.1f}%. "
                "Review response generation prompts, context handling, and answer completeness. "
                "Consider improving retrieval quality or adding more response examples."
            ),
            "Responsible AI": (
                f"{count} safety/bias metric(s) regressed by an average of {avg_change:.1f}%. "
                "Review safety guardrails, bias detection, and content filtering. "
                "Consider strengthening safety prompts or adding additional safety checks."
            ),
            "Performance": (
                f"{count} performance metric(s) regressed by an average of {avg_change:.1f}%. "
                "Review latency, token usage, and cost efficiency. "
                "Consider optimizing prompts, reducing unnecessary tool calls, or using more efficient models."
            ),
            "Multi-Turn": (
                f"{count} multi-turn metric(s) regressed by an average of {avg_change:.1f}%. "
                "Review context retention, conversation coherence, and goal tracking. "
                "Consider improving context management or conversation state handling."
            ),
            "Multi-Agent": (
                f"{count} multi-agent metric(s) regressed by an average of {avg_change:.1f}%. "
                "Review agent coordination, task delegation, and workflow management. "
                "Consider improving inter-agent communication or task routing logic."
            ),
            "Reasoning": (
                f"{count} reasoning metric(s) regressed by an average of {avg_change:.1f}%. "
                "Review chain-of-thought prompts, logical consistency, and reasoning steps. "
                "Consider adding more reasoning examples or improving step-by-step guidance."
            )
        }

        return recommendations.get(
            category,
            f"{count} metric(s) in {category} regressed by an average of {avg_change:.1f}%. Review and address these issues."
        )

    def _generate_regression_summary(
        self,
        regression_data: Dict[str, Any],
        baseline_run: ExperimentRun,
        current_run: ExperimentRun
    ) -> str:
        """
        Generate executive summary for regression report.

        Args:
            regression_data: Classified regression data
            baseline_run: Baseline run
            current_run: Current run

        Returns:
            Summary string
        """
        total_regressions = len(regression_data["regressions"])
        severity_counts = regression_data["severity_counts"]

        summary_parts = []

        # Overall summary
        summary_parts.append(
            f"Detected {total_regressions} regression(s) comparing {baseline_run.run_name} "
            f"(baseline) to {current_run.run_name} (current)."
        )

        # Severity breakdown
        if severity_counts["critical"] > 0:
            summary_parts.append(
                f"{severity_counts['critical']} critical regression(s) requiring immediate attention."
            )
        if severity_counts["high"] > 0:
            summary_parts.append(
                f"{severity_counts['high']} high-priority regression(s) identified."
            )
        if severity_counts["medium"] > 0:
            summary_parts.append(
                f"{severity_counts['medium']} medium-priority regression(s) detected."
            )
        if severity_counts["low"] > 0:
            summary_parts.append(
                f"{severity_counts['low']} low-priority regression(s) noted."
            )

        # Top regressions
        if regression_data["regressions"]:
            top_3 = regression_data["regressions"][:3]
            top_metrics = [r["metric_name"] for r in top_3]
            summary_parts.append(
                f"Most affected metrics: {', '.join(top_metrics)}."
            )

        return " ".join(summary_parts)

    def _generate_historical_summary(
        self,
        baseline_run: ExperimentRun,
        all_reports: List[Tuple[ExperimentRun, ComparisonReport]]
    ) -> str:
        """
        Generate summary for historical regression tracking.

        Args:
            baseline_run: Baseline run
            all_reports: List of (run, report) tuples

        Returns:
            Summary string
        """
        total_runs = len(all_reports)
        total_regressions = sum(len(report.regressions) for _, report in all_reports)

        # Count runs with regressions
        runs_with_regressions = sum(1 for _, report in all_reports if report.regressions)

        # Find most commonly regressed metrics
        all_regressed_metrics = []
        for _, report in all_reports:
            all_regressed_metrics.extend(report.regressions)

        if all_regressed_metrics:
            from collections import Counter
            metric_counts = Counter(all_regressed_metrics)
            top_3_metrics = [m[0] for m in metric_counts.most_common(3)]

            summary = (
                f"Tracked regressions across {total_runs} runs compared to baseline {baseline_run.run_name}. "
                f"Detected {total_regressions} total regressions across {runs_with_regressions} runs. "
                f"Most frequently regressed metrics: {', '.join(top_3_metrics)}."
            )
        else:
            summary = (
                f"Tracked regressions across {total_runs} runs compared to baseline {baseline_run.run_name}. "
                f"No regressions detected in any run."
            )

        return summary

generate_historical_regression_dashboard(baseline_run, comparison_runs, title, description=None)

Generate dashboard tracking regressions across multiple runs.

Parameters:

Name	Type	Description	Default
baseline_run	ExperimentRun	Baseline experiment run	required
comparison_runs	List[ExperimentRun]	List of runs to track regressions across	required
title	str	Dashboard title	required
description	Optional[str]	Optional dashboard description	None

Returns:

Type	Description
Dashboard	Dashboard with historical regression tracking

Raises:

Type	Description
ValueError	If comparison_runs is empty
ValueError	If title is empty

Examples:

>>> generator = RegressionReportGenerator()
>>> dashboard = generator.generate_historical_regression_dashboard(
...     baseline_run=baseline_run,
...     comparison_runs=[run1, run2, run3],
...     title="Regression Tracking - Last 4 Runs"
... )

Source code in uaef/reporting/regression.py

def generate_historical_regression_dashboard(
    self,
    baseline_run: ExperimentRun,
    comparison_runs: List[ExperimentRun],
    title: str,
    description: Optional[str] = None
) -> Dashboard:
    """
    Generate dashboard tracking regressions across multiple runs.

    Args:
        baseline_run: Baseline experiment run
        comparison_runs: List of runs to track regressions across
        title: Dashboard title
        description: Optional dashboard description

    Returns:
        Dashboard with historical regression tracking

    Raises:
        ValueError: If comparison_runs is empty
        ValueError: If title is empty

    Examples:
        >>> generator = RegressionReportGenerator()
        >>> dashboard = generator.generate_historical_regression_dashboard(
        ...     baseline_run=baseline_run,
        ...     comparison_runs=[run1, run2, run3],
        ...     title="Regression Tracking - Last 4 Runs"
        ... )
    """
    if not comparison_runs:
        raise ValueError("comparison_runs cannot be empty")

    if not title or not title.strip():
        raise ValueError("Dashboard title cannot be empty")

    logger.info(
        f"Generating historical regression dashboard for {len(comparison_runs)} runs"
    )

    # Generate comparison reports for all runs
    all_reports = []
    for run in comparison_runs:
        try:
            report = self.comparison_engine.compare_runs(
                baseline_run=baseline_run,
                current_run=run
            )
            all_reports.append((run, report))
        except Exception as e:
            logger.warning(f"Failed to compare run {run.run_id}: {e}")

    if not all_reports:
        raise ValueError("Failed to generate any comparison reports")

    # Generate charts
    charts = []

    # Chart 1: Regression timeline (line chart showing regression count over time)
    charts.append(self._create_regression_timeline_chart(baseline_run, all_reports))

    # Chart 2: Metric regression heatmap (which metrics regressed in which runs)
    charts.append(self._create_metric_regression_heatmap(all_reports))

    # Chart 3: Severity trends (stacked area chart)
    charts.append(self._create_severity_trends_chart(all_reports))

    # Prepare metadata
    metadata = {
        "dashboard_type": "historical_regression",
        "baseline_run_id": str(baseline_run.run_id),
        "comparison_run_ids": [str(run.run_id) for run, _ in all_reports],
        "generated_at": datetime.utcnow().isoformat(),
        "run_count": len(all_reports),
        "total_regressions": sum(len(report.regressions) for _, report in all_reports)
    }

    # Generate description if not provided
    if not description:
        description = self._generate_historical_summary(baseline_run, all_reports)

    # Create dashboard
    dashboard = Dashboard(
        title=title,
        description=description,
        charts=charts,
        metadata=metadata
    )

    logger.info(f"Generated historical regression dashboard with {len(charts)} charts")
    return dashboard

generate_regression_dashboard(baseline_run, current_run, title, description=None, include_root_cause=True, include_recommendations=True)

Generate a regression dashboard comparing two experiment runs.

Parameters:

Name	Type	Description	Default
baseline_run	ExperimentRun	Baseline experiment run	required
current_run	ExperimentRun	Current experiment run to check for regressions	required
title	str	Dashboard title	required
description	Optional[str]	Optional dashboard description	None
include_root_cause	bool	Whether to include root cause analysis	True
include_recommendations	bool	Whether to include recommendations	True

Returns:

Type	Description
Dashboard	Dashboard with regression analysis charts

Raises:

Type	Description
ValueError	If title is empty
ValueError	If baseline and current runs are the same
ValueError	If no regressions detected

Examples:

>>> generator = RegressionReportGenerator()
>>> dashboard = generator.generate_regression_dashboard(
...     baseline_run=baseline_run,
...     current_run=current_run,
...     title="Regression Report - v1 to v2"
... )

Source code in uaef/reporting/regression.py

def generate_regression_dashboard(
    self,
    baseline_run: ExperimentRun,
    current_run: ExperimentRun,
    title: str,
    description: Optional[str] = None,
    include_root_cause: bool = True,
    include_recommendations: bool = True
) -> Dashboard:
    """
    Generate a regression dashboard comparing two experiment runs.

    Args:
        baseline_run: Baseline experiment run
        current_run: Current experiment run to check for regressions
        title: Dashboard title
        description: Optional dashboard description
        include_root_cause: Whether to include root cause analysis
        include_recommendations: Whether to include recommendations

    Returns:
        Dashboard with regression analysis charts

    Raises:
        ValueError: If title is empty
        ValueError: If baseline and current runs are the same
        ValueError: If no regressions detected

    Examples:
        >>> generator = RegressionReportGenerator()
        >>> dashboard = generator.generate_regression_dashboard(
        ...     baseline_run=baseline_run,
        ...     current_run=current_run,
        ...     title="Regression Report - v1 to v2"
        ... )
    """
    if not title or not title.strip():
        raise ValueError("Dashboard title cannot be empty")

    if baseline_run.run_id == current_run.run_id:
        raise ValueError("Cannot compare a run with itself")

    logger.info(
        f"Generating regression dashboard between runs {baseline_run.run_id} "
        f"and {current_run.run_id}"
    )

    # Generate comparison report
    comparison_report = self.comparison_engine.compare_runs(
        baseline_run=baseline_run,
        current_run=current_run
    )

    # Check if there are any regressions
    if not comparison_report.regressions:
        raise ValueError(
            f"No regressions detected between runs {baseline_run.run_id} "
            f"and {current_run.run_id}"
        )

    # Classify regressions by severity
    regression_data = self._classify_regressions(comparison_report)

    # Generate charts
    charts = []

    # Chart 1: Regression overview (bar chart showing all regressed metrics)
    charts.append(self._create_regression_overview_chart(regression_data))

    # Chart 2: Before/after comparison (grouped bar chart)
    charts.append(self._create_before_after_chart(regression_data))

    # Chart 3: Severity distribution (pie chart)
    charts.append(self._create_severity_distribution_chart(regression_data))

    # Chart 4: Impact analysis (heatmap showing affected dimensions)
    charts.append(self._create_impact_heatmap(regression_data))

    # Chart 5: Regression magnitude (bar chart showing percentage changes)
    charts.append(self._create_regression_magnitude_chart(regression_data))

    # Prepare metadata
    metadata = {
        "dashboard_type": "regression",
        "baseline_run_id": str(baseline_run.run_id),
        "current_run_id": str(current_run.run_id),
        "generated_at": datetime.utcnow().isoformat(),
        "regression_threshold": self.regression_threshold,
        "total_regressions": len(comparison_report.regressions),
        "critical_count": regression_data["severity_counts"]["critical"],
        "high_count": regression_data["severity_counts"]["high"],
        "medium_count": regression_data["severity_counts"]["medium"],
        "low_count": regression_data["severity_counts"]["low"],
        "regressions": regression_data["regressions"]
    }

    # Add context about when regression occurred
    metadata["baseline_context"] = {
        "run_name": baseline_run.run_name,
        "timestamp": baseline_run.timestamp.isoformat() if baseline_run.timestamp else None,
        "config_snapshot": baseline_run.config_snapshot
    }
    metadata["current_context"] = {
        "run_name": current_run.run_name,
        "timestamp": current_run.timestamp.isoformat() if current_run.timestamp else None,
        "config_snapshot": current_run.config_snapshot
    }

    # Add root cause analysis if enabled
    if include_root_cause:
        try:
            root_causes = self._analyze_root_causes(
                regression_data, baseline_run, current_run
            )
            metadata["root_causes"] = root_causes

            # Add root cause breakdown chart
            if root_causes:
                charts.append(self._create_root_cause_chart(root_causes))
        except Exception as e:
            logger.warning(f"Failed to generate root cause analysis: {e}")

    # Add recommendations if enabled
    if include_recommendations:
        try:
            recommendations = self._generate_regression_recommendations(
                regression_data, comparison_report
            )
            metadata["recommendations"] = recommendations
        except Exception as e:
            logger.warning(f"Failed to generate recommendations: {e}")

    # Generate description if not provided
    if not description:
        description = self._generate_regression_summary(
            regression_data, baseline_run, current_run
        )

    # Create dashboard
    dashboard = Dashboard(
        title=title,
        description=description,
        charts=charts,
        metadata=metadata
    )

    logger.info(
        f"Generated regression dashboard with {len(charts)} charts and "
        f"{len(comparison_report.regressions)} regressions"
    )
    return dashboard

uaef.reporting.visualization

Visualization engine for generating charts and graphs.

This module provides the VisualizationEngine class for generating interactive and static visualizations from Chart models. It supports multiple chart types including bar, line, scatter, heatmap, pie, radar, box, and histogram charts.

The engine uses plotly for interactive HTML visualizations and matplotlib as a fallback for static image generation (PNG, SVG).

Requirements

• 18.1: Generate charts for executive dashboards
• 18.2: Generate charts for experiment comparison
• 18.3: Generate charts for dimension deep-dive reports
• 18.4: Generate charts for safety reports
• 18.5: Support export to multiple formats (HTML, PNG, SVG)
• 18.6: Generate visualizations for regression reports

Example

from uaef.reporting.models import Chart, ChartType from uaef.reporting.visualization import VisualizationEngine

engine = VisualizationEngine() chart = Chart( ... title="Dimension Scores", ... chart_type=ChartType.BAR, ... data={"dimensions": ["Tool", "Response"], "scores": [0.85, 0.92]}, ... labels=["Tool", "Response"] ... ) html = engine.generate_chart(chart, interactive=True) engine.save_chart(chart, "output.html", format="html")

VisualizationEngine

Engine for generating interactive and static visualizations.

This class provides methods to generate various chart types from Chart models. It uses plotly for interactive HTML charts and matplotlib for static images.

Attributes:

Name	Type	Description
default_theme		Default color theme for charts
themes		Available color themes

Example

engine = VisualizationEngine() chart = Chart( ... title="Scores", ... chart_type=ChartType.BAR, ... data={"labels": ["A", "B"], "values": [0.8, 0.9]} ... ) html = engine.generate_chart(chart, interactive=True)

Source code in uaef/reporting/visualization.py

class VisualizationEngine:
    """
    Engine for generating interactive and static visualizations.

    This class provides methods to generate various chart types from Chart models.
    It uses plotly for interactive HTML charts and matplotlib for static images.

    Attributes:
        default_theme: Default color theme for charts
        themes: Available color themes

    Example:
        >>> engine = VisualizationEngine()
        >>> chart = Chart(
        ...     title="Scores",
        ...     chart_type=ChartType.BAR,
        ...     data={"labels": ["A", "B"], "values": [0.8, 0.9]}
        ... )
        >>> html = engine.generate_chart(chart, interactive=True)
    """

    # Default color themes
    THEMES = {
        "default": ["#4CAF50", "#2196F3", "#FF9800", "#F44336", "#9C27B0", "#00BCD4"],
        "professional": ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b"],
        "pastel": ["#AEC6CF", "#FFB347", "#B39EB5", "#FF6961", "#77DD77", "#FDFD96"],
        "dark": ["#1a1a1a", "#333333", "#4d4d4d", "#666666", "#808080", "#999999"],
        "vibrant": ["#FF1744", "#00E676", "#2979FF", "#FFEA00", "#D500F9", "#00E5FF"],
    }

    def __init__(self, default_theme: str = "default"):
        """
        Initialize the visualization engine.

        Args:
            default_theme: Name of the default color theme to use

        Raises:
            ValueError: If the specified theme doesn't exist
            ImportError: If neither plotly nor matplotlib is available
        """
        if not PLOTLY_AVAILABLE and not MATPLOTLIB_AVAILABLE:
            raise ImportError(
                "Neither plotly nor matplotlib is installed. "
                "Install at least one: pip install plotly or pip install matplotlib"
            )

        if default_theme not in self.THEMES:
            raise ValueError(
                f"Unknown theme: {default_theme}. "
                f"Available themes: {list(self.THEMES.keys())}"
            )

        self.default_theme = default_theme
        self._current_theme = default_theme

        logger.info(
            f"VisualizationEngine initialized with theme '{default_theme}' "
            f"(plotly={'available' if PLOTLY_AVAILABLE else 'unavailable'}, "
            f"matplotlib={'available' if MATPLOTLIB_AVAILABLE else 'unavailable'})"
        )

    def apply_theme(self, theme_name: str) -> None:
        """
        Apply a predefined color theme.

        Args:
            theme_name: Name of the theme to apply

        Raises:
            ValueError: If the theme doesn't exist

        Example:
            >>> engine = VisualizationEngine()
            >>> engine.apply_theme("professional")
        """
        if theme_name not in self.THEMES:
            raise ValueError(
                f"Unknown theme: {theme_name}. "
                f"Available themes: {list(self.THEMES.keys())}"
            )

        self._current_theme = theme_name
        logger.info(f"Applied theme: {theme_name}")

    def _get_colors(self, chart: Chart) -> List[str]:
        """
        Get colors for a chart, using chart colors or theme colors.

        Args:
            chart: Chart model

        Returns:
            List of color codes
        """
        if chart.colors:
            return chart.colors
        return self.THEMES[self._current_theme]

    def generate_chart(
        self,
        chart: Chart,
        interactive: bool = True
    ) -> Union[str, bytes]:
        """
        Generate a chart based on the chart type.

        This is the main entry point for chart generation. It dispatches to
        the appropriate chart-specific method based on chart_type.

        Args:
            chart: Chart model with data and configuration
            interactive: If True, generate interactive HTML (plotly).
                        If False, generate static image (matplotlib).

        Returns:
            For interactive=True: HTML string
            For interactive=False: PNG image bytes

        Raises:
            ValueError: If chart type is not supported
            ImportError: If required library is not available

        Example:
            >>> chart = Chart(
            ...     title="Test",
            ...     chart_type=ChartType.BAR,
            ...     data={"labels": ["A"], "values": [0.8]}
            ... )
            >>> html = engine.generate_chart(chart, interactive=True)
        """
        logger.info(
            f"Generating {chart.chart_type.value} chart: '{chart.title}' "
            f"(interactive={interactive})"
        )

        # Dispatch to appropriate method
        chart_generators = {
            ChartType.BAR: self.generate_bar_chart,
            ChartType.LINE: self.generate_line_chart,
            ChartType.SCATTER: self.generate_scatter_chart,
            ChartType.HEATMAP: self.generate_heatmap,
            ChartType.PIE: self.generate_pie_chart,
            ChartType.RADAR: self.generate_radar_chart,
            ChartType.BOX: self.generate_box_chart,
            ChartType.HISTOGRAM: self.generate_histogram,
        }

        generator = chart_generators.get(chart.chart_type)
        if not generator:
            raise ValueError(f"Unsupported chart type: {chart.chart_type}")

        return generator(chart, interactive)

    def generate_bar_chart(
        self,
        chart: Chart,
        interactive: bool = True
    ) -> Union[str, bytes]:
        """
        Generate a bar chart.

        Expected data structure:
            - {"dimensions": [...], "scores": [...]} or
            - {"labels": [...], "values": [...]}

        Args:
            chart: Chart model with bar chart data
            interactive: If True, generate interactive HTML

        Returns:
            HTML string or PNG bytes

        Example:
            >>> chart = Chart(
            ...     title="Scores",
            ...     chart_type=ChartType.BAR,
            ...     data={"labels": ["A", "B"], "values": [0.8, 0.9]}
            ... )
            >>> html = engine.generate_bar_chart(chart, interactive=True)
        """
        # Extract data
        data = chart.data
        labels = data.get("labels") or data.get("dimensions", [])
        values = data.get("values") or data.get("scores", [])

        if not labels or not values:
            raise ValueError("Bar chart requires 'labels' and 'values' in data")

        colors = self._get_colors(chart)

        if interactive and PLOTLY_AVAILABLE:
            return self._generate_bar_chart_plotly(chart, labels, values, colors)
        elif MATPLOTLIB_AVAILABLE:
            return self._generate_bar_chart_matplotlib(chart, labels, values, colors)
        else:
            raise ImportError("No visualization library available")

    def _generate_bar_chart_plotly(
        self,
        chart: Chart,
        labels: List[str],
        values: List[float],
        colors: List[str]
    ) -> str:
        """Generate interactive bar chart with plotly."""
        # Create bar chart
        fig = go.Figure(data=[
            go.Bar(
                x=labels,
                y=values,
                marker_color=colors[:len(labels)],
                text=[f"{v:.2f}" for v in values],
                textposition='auto',
            )
        ])

        # Update layout
        fig.update_layout(
            title=chart.title,
            xaxis_title=chart.options.get("xaxis_title", ""),
            yaxis_title=chart.options.get("yaxis_title", "Score"),
            template="plotly_white",
            hovermode="x unified",
            showlegend=False,
        )

        # Apply options
        if chart.options.get("horizontal"):
            fig.update_traces(orientation='h')
            fig.update_layout(
                xaxis_title=chart.options.get("xaxis_title", "Score"),
                yaxis_title=chart.options.get("yaxis_title", ""),
            )

        return fig.to_html(include_plotlyjs='cdn', full_html=False)

    def _generate_bar_chart_matplotlib(
        self,
        chart: Chart,
        labels: List[str],
        values: List[float],
        colors: List[str]
    ) -> bytes:
        """Generate static bar chart with matplotlib."""
        fig, ax = plt.subplots(figsize=(10, 6))

        # Create bar chart
        bars = ax.bar(
            range(len(labels)),
            values,
            color=colors[:len(labels)]
        )

        # Add value labels on bars
        for bar in bars:
            height = bar.get_height()
            ax.text(
                bar.get_x() + bar.get_width() / 2.,
                height,
                f'{height:.2f}',
                ha='center',
                va='bottom'
            )

        # Set labels and title
        ax.set_xticks(range(len(labels)))
        ax.set_xticklabels(labels, rotation=45, ha='right')
        ax.set_ylabel(chart.options.get("yaxis_title", "Score"))
        ax.set_title(chart.title)
        ax.grid(axis='y', alpha=0.3)

        # Save to bytes
        from io import BytesIO
        buf = BytesIO()
        plt.tight_layout()
        plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')
        plt.close(fig)
        buf.seek(0)
        return buf.read()

    def generate_line_chart(
        self,
        chart: Chart,
        interactive: bool = True
    ) -> Union[str, bytes]:
        """
        Generate a line chart.

        Expected data structure:
            - {"timestamps": [...], "values": [...]} or
            - {"x": [...], "y": [...]} or
            - {"labels": [...], "values": [...]}

        Args:
            chart: Chart model with line chart data
            interactive: If True, generate interactive HTML

        Returns:
            HTML string or PNG bytes
        """
        data = chart.data
        x_values = data.get("x") or data.get("timestamps") or data.get("labels", [])
        y_values = data.get("y") or data.get("values", [])

        if not x_values or not y_values:
            raise ValueError("Line chart requires x and y values in data")

        colors = self._get_colors(chart)

        if interactive and PLOTLY_AVAILABLE:
            return self._generate_line_chart_plotly(chart, x_values, y_values, colors)
        elif MATPLOTLIB_AVAILABLE:
            return self._generate_line_chart_matplotlib(chart, x_values, y_values, colors)
        else:
            raise ImportError("No visualization library available")

    def _generate_line_chart_plotly(
        self,
        chart: Chart,
        x_values: List,
        y_values: List[float],
        colors: List[str]
    ) -> str:
        """Generate interactive line chart with plotly."""
        fig = go.Figure()

        # Add line trace
        fig.add_trace(go.Scatter(
            x=x_values,
            y=y_values,
            mode='lines+markers',
            line=dict(color=colors[0], width=2),
            marker=dict(size=8),
            name=chart.title
        ))

        # Add trend line if requested
        if chart.options.get("show_trend_line"):
            # Simple linear trend
            if MATPLOTLIB_AVAILABLE:
                z = np.polyfit(range(len(y_values)), y_values, 1)
                p = np.poly1d(z)
                trend_y = [p(i) for i in range(len(y_values))]

                fig.add_trace(go.Scatter(
                    x=x_values,
                    y=trend_y,
                    mode='lines',
                    line=dict(color=colors[1] if len(colors) > 1 else colors[0], dash='dash'),
                    name='Trend'
                ))

        # Update layout
        fig.update_layout(
            title=chart.title,
            xaxis_title=chart.options.get("xaxis_title", ""),
            yaxis_title=chart.options.get("yaxis_title", "Value"),
            template="plotly_white",
            hovermode="x unified",
        )

        return fig.to_html(include_plotlyjs='cdn', full_html=False)

    def _generate_line_chart_matplotlib(
        self,
        chart: Chart,
        x_values: List,
        y_values: List[float],
        colors: List[str]
    ) -> bytes:
        """Generate static line chart with matplotlib."""
        fig, ax = plt.subplots(figsize=(10, 6))

        # Plot line
        ax.plot(x_values, y_values, marker='o', color=colors[0], linewidth=2, markersize=8)

        # Add trend line if requested
        if chart.options.get("show_trend_line"):
            z = np.polyfit(range(len(y_values)), y_values, 1)
            p = np.poly1d(z)
            trend_y = [p(i) for i in range(len(y_values))]
            ax.plot(x_values, trend_y, '--', color=colors[1] if len(colors) > 1 else colors[0], alpha=0.7, label='Trend')
            ax.legend()

        # Set labels and title
        ax.set_xlabel(chart.options.get("xaxis_title", ""))
        ax.set_ylabel(chart.options.get("yaxis_title", "Value"))
        ax.set_title(chart.title)
        ax.grid(alpha=0.3)

        # Rotate x-axis labels if they're strings
        if x_values and isinstance(x_values[0], str):
            plt.xticks(rotation=45, ha='right')

        # Save to bytes
        from io import BytesIO
        buf = BytesIO()
        plt.tight_layout()
        plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')
        plt.close(fig)
        buf.seek(0)
        return buf.read()

    def generate_scatter_chart(
        self,
        chart: Chart,
        interactive: bool = True
    ) -> Union[str, bytes]:
        """
        Generate a scatter chart.

        Expected data structure:
            - {"x": [...], "y": [...]}

        Args:
            chart: Chart model with scatter chart data
            interactive: If True, generate interactive HTML

        Returns:
            HTML string or PNG bytes
        """
        data = chart.data
        x_values = data.get("x", [])
        y_values = data.get("y", [])

        if not x_values or not y_values:
            raise ValueError("Scatter chart requires 'x' and 'y' in data")

        colors = self._get_colors(chart)

        if interactive and PLOTLY_AVAILABLE:
            return self._generate_scatter_chart_plotly(chart, x_values, y_values, colors)
        elif MATPLOTLIB_AVAILABLE:
            return self._generate_scatter_chart_matplotlib(chart, x_values, y_values, colors)
        else:
            raise ImportError("No visualization library available")

    def _generate_scatter_chart_plotly(
        self,
        chart: Chart,
        x_values: List[float],
        y_values: List[float],
        colors: List[str]
    ) -> str:
        """Generate interactive scatter chart with plotly."""
        fig = go.Figure()

        fig.add_trace(go.Scatter(
            x=x_values,
            y=y_values,
            mode='markers',
            marker=dict(
                size=10,
                color=colors[0],
                line=dict(width=1, color='white')
            ),
            name=chart.title
        ))

        # Update layout
        fig.update_layout(
            title=chart.title,
            xaxis_title=chart.options.get("xaxis_title", "X"),
            yaxis_title=chart.options.get("yaxis_title", "Y"),
            template="plotly_white",
            hovermode="closest",
        )

        return fig.to_html(include_plotlyjs='cdn', full_html=False)

    def _generate_scatter_chart_matplotlib(
        self,
        chart: Chart,
        x_values: List[float],
        y_values: List[float],
        colors: List[str]
    ) -> bytes:
        """Generate static scatter chart with matplotlib."""
        fig, ax = plt.subplots(figsize=(10, 6))

        ax.scatter(x_values, y_values, c=colors[0], s=100, alpha=0.6, edgecolors='white', linewidth=1)

        ax.set_xlabel(chart.options.get("xaxis_title", "X"))
        ax.set_ylabel(chart.options.get("yaxis_title", "Y"))
        ax.set_title(chart.title)
        ax.grid(alpha=0.3)

        # Save to bytes
        from io import BytesIO
        buf = BytesIO()
        plt.tight_layout()
        plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')
        plt.close(fig)
        buf.seek(0)
        return buf.read()

    def generate_heatmap(
        self,
        chart: Chart,
        interactive: bool = True
    ) -> Union[str, bytes]:
        """
        Generate a heatmap.

        Expected data structure:
            - {"matrix": [[...]], "x_labels": [...], "y_labels": [...]}

        Args:
            chart: Chart model with heatmap data
            interactive: If True, generate interactive HTML

        Returns:
            HTML string or PNG bytes
        """
        data = chart.data
        matrix = data.get("matrix", [])
        x_labels = data.get("x_labels", [])
        y_labels = data.get("y_labels", [])

        if not matrix:
            raise ValueError("Heatmap requires 'matrix' in data")

        colors = self._get_colors(chart)

        if interactive and PLOTLY_AVAILABLE:
            return self._generate_heatmap_plotly(chart, matrix, x_labels, y_labels, colors)
        elif MATPLOTLIB_AVAILABLE:
            return self._generate_heatmap_matplotlib(chart, matrix, x_labels, y_labels, colors)
        else:
            raise ImportError("No visualization library available")

    def _generate_heatmap_plotly(
        self,
        chart: Chart,
        matrix: List[List[float]],
        x_labels: List[str],
        y_labels: List[str],
        colors: List[str]
    ) -> str:
        """Generate interactive heatmap with plotly."""
        # Use a color scale
        colorscale = chart.options.get("colorscale", "Viridis")

        fig = go.Figure(data=go.Heatmap(
            z=matrix,
            x=x_labels if x_labels else None,
            y=y_labels if y_labels else None,
            colorscale=colorscale,
            text=matrix,
            texttemplate="%{text:.2f}",
            textfont={"size": 10},
            hoverongaps=False
        ))

        fig.update_layout(
            title=chart.title,
            xaxis_title=chart.options.get("xaxis_title", ""),
            yaxis_title=chart.options.get("yaxis_title", ""),
            template="plotly_white",
        )

        return fig.to_html(include_plotlyjs='cdn', full_html=False)

    def _generate_heatmap_matplotlib(
        self,
        chart: Chart,
        matrix: List[List[float]],
        x_labels: List[str],
        y_labels: List[str],
        colors: List[str]
    ) -> bytes:
        """Generate static heatmap with matplotlib."""
        fig, ax = plt.subplots(figsize=(10, 8))

        # Create heatmap
        im = ax.imshow(matrix, cmap='viridis', aspect='auto')

        # Set ticks and labels
        if x_labels:
            ax.set_xticks(range(len(x_labels)))
            ax.set_xticklabels(x_labels, rotation=45, ha='right')
        if y_labels:
            ax.set_yticks(range(len(y_labels)))
            ax.set_yticklabels(y_labels)

        # Add colorbar
        plt.colorbar(im, ax=ax)

        # Add text annotations
        for i in range(len(matrix)):
            for j in range(len(matrix[0])):
                text = ax.text(j, i, f'{matrix[i][j]:.2f}',
                             ha="center", va="center", color="white", fontsize=8)

        ax.set_title(chart.title)

        # Save to bytes
        from io import BytesIO
        buf = BytesIO()
        plt.tight_layout()
        plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')
        plt.close(fig)
        buf.seek(0)
        return buf.read()

    def generate_pie_chart(
        self,
        chart: Chart,
        interactive: bool = True
    ) -> Union[str, bytes]:
        """
        Generate a pie chart.

        Expected data structure:
            - {"labels": [...], "values": [...]}

        Args:
            chart: Chart model with pie chart data
            interactive: If True, generate interactive HTML

        Returns:
            HTML string or PNG bytes
        """
        data = chart.data
        labels = data.get("labels", [])
        values = data.get("values", [])

        if not labels or not values:
            raise ValueError("Pie chart requires 'labels' and 'values' in data")

        colors = self._get_colors(chart)

        if interactive and PLOTLY_AVAILABLE:
            return self._generate_pie_chart_plotly(chart, labels, values, colors)
        elif MATPLOTLIB_AVAILABLE:
            return self._generate_pie_chart_matplotlib(chart, labels, values, colors)
        else:
            raise ImportError("No visualization library available")

    def _generate_pie_chart_plotly(
        self,
        chart: Chart,
        labels: List[str],
        values: List[float],
        colors: List[str]
    ) -> str:
        """Generate interactive pie chart with plotly."""
        fig = go.Figure(data=[go.Pie(
            labels=labels,
            values=values,
            marker=dict(colors=colors[:len(labels)]),
            textinfo='label+percent',
            hoverinfo='label+value+percent'
        )])

        fig.update_layout(
            title=chart.title,
            template="plotly_white",
        )

        return fig.to_html(include_plotlyjs='cdn', full_html=False)

    def _generate_pie_chart_matplotlib(
        self,
        chart: Chart,
        labels: List[str],
        values: List[float],
        colors: List[str]
    ) -> bytes:
        """Generate static pie chart with matplotlib."""
        fig, ax = plt.subplots(figsize=(10, 8))

        ax.pie(
            values,
            labels=labels,
            colors=colors[:len(labels)],
            autopct='%1.1f%%',
            startangle=90
        )

        ax.set_title(chart.title)

        # Save to bytes
        from io import BytesIO
        buf = BytesIO()
        plt.tight_layout()
        plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')
        plt.close(fig)
        buf.seek(0)
        return buf.read()

    def generate_radar_chart(
        self,
        chart: Chart,
        interactive: bool = True
    ) -> Union[str, bytes]:
        """
        Generate a radar (spider) chart.

        Expected data structure:
            - {"dimensions": [...], "scores": [...]}

        Args:
            chart: Chart model with radar chart data
            interactive: If True, generate interactive HTML

        Returns:
            HTML string or PNG bytes
        """
        data = chart.data
        dimensions = data.get("dimensions", [])
        scores = data.get("scores", [])

        if not dimensions or not scores:
            raise ValueError("Radar chart requires 'dimensions' and 'scores' in data")

        colors = self._get_colors(chart)

        if interactive and PLOTLY_AVAILABLE:
            return self._generate_radar_chart_plotly(chart, dimensions, scores, colors)
        elif MATPLOTLIB_AVAILABLE:
            return self._generate_radar_chart_matplotlib(chart, dimensions, scores, colors)
        else:
            raise ImportError("No visualization library available")

    def _generate_radar_chart_plotly(
        self,
        chart: Chart,
        dimensions: List[str],
        scores: List[float],
        colors: List[str]
    ) -> str:
        """Generate interactive radar chart with plotly."""
        fig = go.Figure()

        fig.add_trace(go.Scatterpolar(
            r=scores,
            theta=dimensions,
            fill='toself',
            fillcolor=colors[0],
            line=dict(color=colors[0]),
            opacity=0.6,
            name=chart.title
        ))

        fig.update_layout(
            polar=dict(
                radialaxis=dict(
                    visible=True,
                    range=[0, 1]
                )
            ),
            title=chart.title,
            template="plotly_white",
        )

        return fig.to_html(include_plotlyjs='cdn', full_html=False)

    def _generate_radar_chart_matplotlib(
        self,
        chart: Chart,
        dimensions: List[str],
        scores: List[float],
        colors: List[str]
    ) -> bytes:
        """Generate static radar chart with matplotlib."""
        # Number of variables
        num_vars = len(dimensions)

        # Compute angle for each axis
        angles = np.linspace(0, 2 * np.pi, num_vars, endpoint=False).tolist()

        # Complete the circle
        scores_plot = scores + [scores[0]]
        angles_plot = angles + [angles[0]]

        # Create figure
        fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(projection='polar'))

        # Plot data
        ax.plot(angles_plot, scores_plot, 'o-', linewidth=2, color=colors[0])
        ax.fill(angles_plot, scores_plot, alpha=0.25, color=colors[0])

        # Fix axis to go in the right order
        ax.set_theta_offset(np.pi / 2)
        ax.set_theta_direction(-1)

        # Set labels
        ax.set_xticks(angles)
        ax.set_xticklabels(dimensions)

        # Set y-axis limits
        ax.set_ylim(0, 1)

        ax.set_title(chart.title, pad=20)
        ax.grid(True)

        # Save to bytes
        from io import BytesIO
        buf = BytesIO()
        plt.tight_layout()
        plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')
        plt.close(fig)
        buf.seek(0)
        return buf.read()

    def generate_box_chart(
        self,
        chart: Chart,
        interactive: bool = True
    ) -> Union[str, bytes]:
        """
        Generate a box plot.

        Expected data structure:
            - {"data": [...], "labels": [...]} or
            - {"data": [[...], [...]], "labels": [...]}

        Args:
            chart: Chart model with box plot data
            interactive: If True, generate interactive HTML

        Returns:
            HTML string or PNG bytes
        """
        data = chart.data
        box_data = data.get("data", [])
        labels = data.get("labels", [])

        if not box_data:
            raise ValueError("Box chart requires 'data' in data")

        colors = self._get_colors(chart)

        if interactive and PLOTLY_AVAILABLE:
            return self._generate_box_chart_plotly(chart, box_data, labels, colors)
        elif MATPLOTLIB_AVAILABLE:
            return self._generate_box_chart_matplotlib(chart, box_data, labels, colors)
        else:
            raise ImportError("No visualization library available")

    def _generate_box_chart_plotly(
        self,
        chart: Chart,
        box_data: Union[List[float], List[List[float]]],
        labels: List[str],
        colors: List[str]
    ) -> str:
        """Generate interactive box plot with plotly."""
        fig = go.Figure()

        # Handle single or multiple box plots
        if box_data and isinstance(box_data[0], list):
            # Multiple box plots
            for i, (data_series, label) in enumerate(zip(box_data, labels)):
                fig.add_trace(go.Box(
                    y=data_series,
                    name=label,
                    marker_color=colors[i % len(colors)]
                ))
        else:
            # Single box plot
            fig.add_trace(go.Box(
                y=box_data,
                name=labels[0] if labels else chart.title,
                marker_color=colors[0]
            ))

        fig.update_layout(
            title=chart.title,
            yaxis_title=chart.options.get("yaxis_title", "Value"),
            template="plotly_white",
        )

        return fig.to_html(include_plotlyjs='cdn', full_html=False)

    def _generate_box_chart_matplotlib(
        self,
        chart: Chart,
        box_data: Union[List[float], List[List[float]]],
        labels: List[str],
        colors: List[str]
    ) -> bytes:
        """Generate static box plot with matplotlib."""
        fig, ax = plt.subplots(figsize=(10, 6))

        # Handle single or multiple box plots
        if box_data and isinstance(box_data[0], list):
            bp = ax.boxplot(box_data, labels=labels, patch_artist=True)
            # Color each box
            for patch, color in zip(bp['boxes'], colors):
                patch.set_facecolor(color)
        else:
            bp = ax.boxplot([box_data], labels=labels if labels else [chart.title], patch_artist=True)
            bp['boxes'][0].set_facecolor(colors[0])

        ax.set_ylabel(chart.options.get("yaxis_title", "Value"))
        ax.set_title(chart.title)
        ax.grid(axis='y', alpha=0.3)

        # Save to bytes
        from io import BytesIO
        buf = BytesIO()
        plt.tight_layout()
        plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')
        plt.close(fig)
        buf.seek(0)
        return buf.read()

    def generate_histogram(
        self,
        chart: Chart,
        interactive: bool = True
    ) -> Union[str, bytes]:
        """
        Generate a histogram.

        Expected data structure:
            - {"values": [...], "bins": int}

        Args:
            chart: Chart model with histogram data
            interactive: If True, generate interactive HTML

        Returns:
            HTML string or PNG bytes
        """
        data = chart.data
        values = data.get("values", [])
        bins = data.get("bins", 10)

        if not values:
            raise ValueError("Histogram requires 'values' in data")

        colors = self._get_colors(chart)

        if interactive and PLOTLY_AVAILABLE:
            return self._generate_histogram_plotly(chart, values, bins, colors)
        elif MATPLOTLIB_AVAILABLE:
            return self._generate_histogram_matplotlib(chart, values, bins, colors)
        else:
            raise ImportError("No visualization library available")

    def _generate_histogram_plotly(
        self,
        chart: Chart,
        values: List[float],
        bins: int,
        colors: List[str]
    ) -> str:
        """Generate interactive histogram with plotly."""
        fig = go.Figure()

        fig.add_trace(go.Histogram(
            x=values,
            nbinsx=bins,
            marker_color=colors[0],
            name=chart.title
        ))

        fig.update_layout(
            title=chart.title,
            xaxis_title=chart.options.get("xaxis_title", "Value"),
            yaxis_title=chart.options.get("yaxis_title", "Frequency"),
            template="plotly_white",
        )

        return fig.to_html(include_plotlyjs='cdn', full_html=False)

    def _generate_histogram_matplotlib(
        self,
        chart: Chart,
        values: List[float],
        bins: int,
        colors: List[str]
    ) -> bytes:
        """Generate static histogram with matplotlib."""
        fig, ax = plt.subplots(figsize=(10, 6))

        ax.hist(values, bins=bins, color=colors[0], alpha=0.7, edgecolor='black')

        ax.set_xlabel(chart.options.get("xaxis_title", "Value"))
        ax.set_ylabel(chart.options.get("yaxis_title", "Frequency"))
        ax.set_title(chart.title)
        ax.grid(axis='y', alpha=0.3)

        # Save to bytes
        from io import BytesIO
        buf = BytesIO()
        plt.tight_layout()
        plt.savefig(buf, format='png', dpi=150, bbox_inches='tight')
        plt.close(fig)
        buf.seek(0)
        return buf.read()

    def save_chart(
        self,
        chart: Chart,
        output_path: str,
        format: str = "html",
        interactive: bool = True
    ) -> None:
        """
        Generate and save a chart to a file.

        Args:
            chart: Chart model to generate
            output_path: Path where to save the chart
            format: Output format ("html", "png", "svg")
            interactive: If True, generate interactive chart (only for HTML)

        Raises:
            ValueError: If format is not supported
            IOError: If file cannot be written

        Example:
            >>> chart = Chart(
            ...     title="Test",
            ...     chart_type=ChartType.BAR,
            ...     data={"labels": ["A"], "values": [0.8]}
            ... )
            >>> engine.save_chart(chart, "output.html", format="html")
        """
        format = format.lower()

        if format == "html":
            if not PLOTLY_AVAILABLE:
                raise ImportError("plotly is required for HTML output")

            html_content = self.generate_chart(chart, interactive=True)

            # Wrap in full HTML if needed
            if not html_content.startswith("<!DOCTYPE") and not html_content.startswith("<html"):
                html_content = f"""<!DOCTYPE html>
<html>
<head>
    <meta charset="utf-8">
    <title>{chart.title}</title>
</head>
<body>
    {html_content}
</body>
</html>"""

            Path(output_path).write_text(html_content, encoding='utf-8')
            logger.info(f"Saved HTML chart to {output_path}")

        elif format in ["png", "svg"]:
            if not MATPLOTLIB_AVAILABLE:
                raise ImportError("matplotlib is required for PNG/SVG output")

            # Generate with matplotlib
            image_bytes = self.generate_chart(chart, interactive=False)

            if format == "svg":
                # Re-generate as SVG
                # This is a simplified approach - in production, you'd want to
                # modify the matplotlib generation to support SVG directly
                raise NotImplementedError("SVG format not yet implemented")

            Path(output_path).write_bytes(image_bytes)
            logger.info(f"Saved {format.upper()} chart to {output_path}")

        else:
            raise ValueError(
                f"Unsupported format: {format}. "
                f"Supported formats: html, png, svg"
            )

apply_theme(theme_name)

Apply a predefined color theme.

Parameters:

Name	Type	Description	Default
theme_name	str	Name of the theme to apply	required

Raises:

Type	Description
ValueError	If the theme doesn't exist

Example

engine = VisualizationEngine() engine.apply_theme("professional")

Source code in uaef/reporting/visualization.py

def apply_theme(self, theme_name: str) -> None:
    """
    Apply a predefined color theme.

    Args:
        theme_name: Name of the theme to apply

    Raises:
        ValueError: If the theme doesn't exist

    Example:
        >>> engine = VisualizationEngine()
        >>> engine.apply_theme("professional")
    """
    if theme_name not in self.THEMES:
        raise ValueError(
            f"Unknown theme: {theme_name}. "
            f"Available themes: {list(self.THEMES.keys())}"
        )

    self._current_theme = theme_name
    logger.info(f"Applied theme: {theme_name}")

generate_bar_chart(chart, interactive=True)

Generate a bar chart.

Expected data structure

• {"dimensions": [...], "scores": [...]} or
• {"labels": [...], "values": [...]}

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model with bar chart data	required
interactive	bool	If True, generate interactive HTML	True

Returns:

Type	Description
Union[str, bytes]	HTML string or PNG bytes

Example

chart = Chart( ... title="Scores", ... chart_type=ChartType.BAR, ... data={"labels": ["A", "B"], "values": [0.8, 0.9]} ... ) html = engine.generate_bar_chart(chart, interactive=True)

Source code in uaef/reporting/visualization.py

def generate_bar_chart(
    self,
    chart: Chart,
    interactive: bool = True
) -> Union[str, bytes]:
    """
    Generate a bar chart.

    Expected data structure:
        - {"dimensions": [...], "scores": [...]} or
        - {"labels": [...], "values": [...]}

    Args:
        chart: Chart model with bar chart data
        interactive: If True, generate interactive HTML

    Returns:
        HTML string or PNG bytes

    Example:
        >>> chart = Chart(
        ...     title="Scores",
        ...     chart_type=ChartType.BAR,
        ...     data={"labels": ["A", "B"], "values": [0.8, 0.9]}
        ... )
        >>> html = engine.generate_bar_chart(chart, interactive=True)
    """
    # Extract data
    data = chart.data
    labels = data.get("labels") or data.get("dimensions", [])
    values = data.get("values") or data.get("scores", [])

    if not labels or not values:
        raise ValueError("Bar chart requires 'labels' and 'values' in data")

    colors = self._get_colors(chart)

    if interactive and PLOTLY_AVAILABLE:
        return self._generate_bar_chart_plotly(chart, labels, values, colors)
    elif MATPLOTLIB_AVAILABLE:
        return self._generate_bar_chart_matplotlib(chart, labels, values, colors)
    else:
        raise ImportError("No visualization library available")

generate_box_chart(chart, interactive=True)

Generate a box plot.

Expected data structure

• {"data": [...], "labels": [...]} or
• {"data": [[...], [...]], "labels": [...]}

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model with box plot data	required
interactive	bool	If True, generate interactive HTML	True

Returns:

Type	Description
Union[str, bytes]	HTML string or PNG bytes

Source code in uaef/reporting/visualization.py

def generate_box_chart(
    self,
    chart: Chart,
    interactive: bool = True
) -> Union[str, bytes]:
    """
    Generate a box plot.

    Expected data structure:
        - {"data": [...], "labels": [...]} or
        - {"data": [[...], [...]], "labels": [...]}

    Args:
        chart: Chart model with box plot data
        interactive: If True, generate interactive HTML

    Returns:
        HTML string or PNG bytes
    """
    data = chart.data
    box_data = data.get("data", [])
    labels = data.get("labels", [])

    if not box_data:
        raise ValueError("Box chart requires 'data' in data")

    colors = self._get_colors(chart)

    if interactive and PLOTLY_AVAILABLE:
        return self._generate_box_chart_plotly(chart, box_data, labels, colors)
    elif MATPLOTLIB_AVAILABLE:
        return self._generate_box_chart_matplotlib(chart, box_data, labels, colors)
    else:
        raise ImportError("No visualization library available")

generate_chart(chart, interactive=True)

Generate a chart based on the chart type.

This is the main entry point for chart generation. It dispatches to the appropriate chart-specific method based on chart_type.

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model with data and configuration	required
interactive	bool	If True, generate interactive HTML (plotly). If False, generate static image (matplotlib).	True

Returns:

Type	Description
Union[str, bytes]	For interactive=True: HTML string
Union[str, bytes]	For interactive=False: PNG image bytes

Raises:

Type	Description
ValueError	If chart type is not supported
ImportError	If required library is not available

Example

chart = Chart( ... title="Test", ... chart_type=ChartType.BAR, ... data={"labels": ["A"], "values": [0.8]} ... ) html = engine.generate_chart(chart, interactive=True)

Source code in uaef/reporting/visualization.py

def generate_chart(
    self,
    chart: Chart,
    interactive: bool = True
) -> Union[str, bytes]:
    """
    Generate a chart based on the chart type.

    This is the main entry point for chart generation. It dispatches to
    the appropriate chart-specific method based on chart_type.

    Args:
        chart: Chart model with data and configuration
        interactive: If True, generate interactive HTML (plotly).
                    If False, generate static image (matplotlib).

    Returns:
        For interactive=True: HTML string
        For interactive=False: PNG image bytes

    Raises:
        ValueError: If chart type is not supported
        ImportError: If required library is not available

    Example:
        >>> chart = Chart(
        ...     title="Test",
        ...     chart_type=ChartType.BAR,
        ...     data={"labels": ["A"], "values": [0.8]}
        ... )
        >>> html = engine.generate_chart(chart, interactive=True)
    """
    logger.info(
        f"Generating {chart.chart_type.value} chart: '{chart.title}' "
        f"(interactive={interactive})"
    )

    # Dispatch to appropriate method
    chart_generators = {
        ChartType.BAR: self.generate_bar_chart,
        ChartType.LINE: self.generate_line_chart,
        ChartType.SCATTER: self.generate_scatter_chart,
        ChartType.HEATMAP: self.generate_heatmap,
        ChartType.PIE: self.generate_pie_chart,
        ChartType.RADAR: self.generate_radar_chart,
        ChartType.BOX: self.generate_box_chart,
        ChartType.HISTOGRAM: self.generate_histogram,
    }

    generator = chart_generators.get(chart.chart_type)
    if not generator:
        raise ValueError(f"Unsupported chart type: {chart.chart_type}")

    return generator(chart, interactive)

generate_heatmap(chart, interactive=True)

Generate a heatmap.

Expected data structure

• {"matrix": [[...]], "x_labels": [...], "y_labels": [...]}

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model with heatmap data	required
interactive	bool	If True, generate interactive HTML	True

Returns:

Type	Description
Union[str, bytes]	HTML string or PNG bytes

Source code in uaef/reporting/visualization.py

def generate_heatmap(
    self,
    chart: Chart,
    interactive: bool = True
) -> Union[str, bytes]:
    """
    Generate a heatmap.

    Expected data structure:
        - {"matrix": [[...]], "x_labels": [...], "y_labels": [...]}

    Args:
        chart: Chart model with heatmap data
        interactive: If True, generate interactive HTML

    Returns:
        HTML string or PNG bytes
    """
    data = chart.data
    matrix = data.get("matrix", [])
    x_labels = data.get("x_labels", [])
    y_labels = data.get("y_labels", [])

    if not matrix:
        raise ValueError("Heatmap requires 'matrix' in data")

    colors = self._get_colors(chart)

    if interactive and PLOTLY_AVAILABLE:
        return self._generate_heatmap_plotly(chart, matrix, x_labels, y_labels, colors)
    elif MATPLOTLIB_AVAILABLE:
        return self._generate_heatmap_matplotlib(chart, matrix, x_labels, y_labels, colors)
    else:
        raise ImportError("No visualization library available")

generate_histogram(chart, interactive=True)

Generate a histogram.

Expected data structure

• {"values": [...], "bins": int}

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model with histogram data	required
interactive	bool	If True, generate interactive HTML	True

Returns:

Type	Description
Union[str, bytes]	HTML string or PNG bytes

Source code in uaef/reporting/visualization.py

def generate_histogram(
    self,
    chart: Chart,
    interactive: bool = True
) -> Union[str, bytes]:
    """
    Generate a histogram.

    Expected data structure:
        - {"values": [...], "bins": int}

    Args:
        chart: Chart model with histogram data
        interactive: If True, generate interactive HTML

    Returns:
        HTML string or PNG bytes
    """
    data = chart.data
    values = data.get("values", [])
    bins = data.get("bins", 10)

    if not values:
        raise ValueError("Histogram requires 'values' in data")

    colors = self._get_colors(chart)

    if interactive and PLOTLY_AVAILABLE:
        return self._generate_histogram_plotly(chart, values, bins, colors)
    elif MATPLOTLIB_AVAILABLE:
        return self._generate_histogram_matplotlib(chart, values, bins, colors)
    else:
        raise ImportError("No visualization library available")

generate_line_chart(chart, interactive=True)

Generate a line chart.

Expected data structure

• {"timestamps": [...], "values": [...]} or
• {"x": [...], "y": [...]} or
• {"labels": [...], "values": [...]}

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model with line chart data	required
interactive	bool	If True, generate interactive HTML	True

Returns:

Type	Description
Union[str, bytes]	HTML string or PNG bytes

Source code in uaef/reporting/visualization.py

def generate_line_chart(
    self,
    chart: Chart,
    interactive: bool = True
) -> Union[str, bytes]:
    """
    Generate a line chart.

    Expected data structure:
        - {"timestamps": [...], "values": [...]} or
        - {"x": [...], "y": [...]} or
        - {"labels": [...], "values": [...]}

    Args:
        chart: Chart model with line chart data
        interactive: If True, generate interactive HTML

    Returns:
        HTML string or PNG bytes
    """
    data = chart.data
    x_values = data.get("x") or data.get("timestamps") or data.get("labels", [])
    y_values = data.get("y") or data.get("values", [])

    if not x_values or not y_values:
        raise ValueError("Line chart requires x and y values in data")

    colors = self._get_colors(chart)

    if interactive and PLOTLY_AVAILABLE:
        return self._generate_line_chart_plotly(chart, x_values, y_values, colors)
    elif MATPLOTLIB_AVAILABLE:
        return self._generate_line_chart_matplotlib(chart, x_values, y_values, colors)
    else:
        raise ImportError("No visualization library available")

generate_pie_chart(chart, interactive=True)

Generate a pie chart.

Expected data structure

• {"labels": [...], "values": [...]}

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model with pie chart data	required
interactive	bool	If True, generate interactive HTML	True

Returns:

Type	Description
Union[str, bytes]	HTML string or PNG bytes

Source code in uaef/reporting/visualization.py

def generate_pie_chart(
    self,
    chart: Chart,
    interactive: bool = True
) -> Union[str, bytes]:
    """
    Generate a pie chart.

    Expected data structure:
        - {"labels": [...], "values": [...]}

    Args:
        chart: Chart model with pie chart data
        interactive: If True, generate interactive HTML

    Returns:
        HTML string or PNG bytes
    """
    data = chart.data
    labels = data.get("labels", [])
    values = data.get("values", [])

    if not labels or not values:
        raise ValueError("Pie chart requires 'labels' and 'values' in data")

    colors = self._get_colors(chart)

    if interactive and PLOTLY_AVAILABLE:
        return self._generate_pie_chart_plotly(chart, labels, values, colors)
    elif MATPLOTLIB_AVAILABLE:
        return self._generate_pie_chart_matplotlib(chart, labels, values, colors)
    else:
        raise ImportError("No visualization library available")

generate_radar_chart(chart, interactive=True)

Generate a radar (spider) chart.

Expected data structure

• {"dimensions": [...], "scores": [...]}

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model with radar chart data	required
interactive	bool	If True, generate interactive HTML	True

Returns:

Type	Description
Union[str, bytes]	HTML string or PNG bytes

Source code in uaef/reporting/visualization.py

def generate_radar_chart(
    self,
    chart: Chart,
    interactive: bool = True
) -> Union[str, bytes]:
    """
    Generate a radar (spider) chart.

    Expected data structure:
        - {"dimensions": [...], "scores": [...]}

    Args:
        chart: Chart model with radar chart data
        interactive: If True, generate interactive HTML

    Returns:
        HTML string or PNG bytes
    """
    data = chart.data
    dimensions = data.get("dimensions", [])
    scores = data.get("scores", [])

    if not dimensions or not scores:
        raise ValueError("Radar chart requires 'dimensions' and 'scores' in data")

    colors = self._get_colors(chart)

    if interactive and PLOTLY_AVAILABLE:
        return self._generate_radar_chart_plotly(chart, dimensions, scores, colors)
    elif MATPLOTLIB_AVAILABLE:
        return self._generate_radar_chart_matplotlib(chart, dimensions, scores, colors)
    else:
        raise ImportError("No visualization library available")

generate_scatter_chart(chart, interactive=True)

Generate a scatter chart.

Expected data structure

• {"x": [...], "y": [...]}

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model with scatter chart data	required
interactive	bool	If True, generate interactive HTML	True

Returns:

Type	Description
Union[str, bytes]	HTML string or PNG bytes

Source code in uaef/reporting/visualization.py

def generate_scatter_chart(
    self,
    chart: Chart,
    interactive: bool = True
) -> Union[str, bytes]:
    """
    Generate a scatter chart.

    Expected data structure:
        - {"x": [...], "y": [...]}

    Args:
        chart: Chart model with scatter chart data
        interactive: If True, generate interactive HTML

    Returns:
        HTML string or PNG bytes
    """
    data = chart.data
    x_values = data.get("x", [])
    y_values = data.get("y", [])

    if not x_values or not y_values:
        raise ValueError("Scatter chart requires 'x' and 'y' in data")

    colors = self._get_colors(chart)

    if interactive and PLOTLY_AVAILABLE:
        return self._generate_scatter_chart_plotly(chart, x_values, y_values, colors)
    elif MATPLOTLIB_AVAILABLE:
        return self._generate_scatter_chart_matplotlib(chart, x_values, y_values, colors)
    else:
        raise ImportError("No visualization library available")

save_chart(chart, output_path, format='html', interactive=True)

Generate and save a chart to a file.

Parameters:

Name	Type	Description	Default
chart	Chart	Chart model to generate	required
output_path	str	Path where to save the chart	required
format	str	Output format ("html", "png", "svg")	'html'
interactive	bool	If True, generate interactive chart (only for HTML)	True

Raises:

Type	Description
ValueError	If format is not supported
IOError	If file cannot be written

Example

chart = Chart( ... title="Test", ... chart_type=ChartType.BAR, ... data={"labels": ["A"], "values": [0.8]} ... ) engine.save_chart(chart, "output.html", format="html")

Source code in uaef/reporting/visualization.py

    def save_chart(
        self,
        chart: Chart,
        output_path: str,
        format: str = "html",
        interactive: bool = True
    ) -> None:
        """
        Generate and save a chart to a file.

        Args:
            chart: Chart model to generate
            output_path: Path where to save the chart
            format: Output format ("html", "png", "svg")
            interactive: If True, generate interactive chart (only for HTML)

        Raises:
            ValueError: If format is not supported
            IOError: If file cannot be written

        Example:
            >>> chart = Chart(
            ...     title="Test",
            ...     chart_type=ChartType.BAR,
            ...     data={"labels": ["A"], "values": [0.8]}
            ... )
            >>> engine.save_chart(chart, "output.html", format="html")
        """
        format = format.lower()

        if format == "html":
            if not PLOTLY_AVAILABLE:
                raise ImportError("plotly is required for HTML output")

            html_content = self.generate_chart(chart, interactive=True)

            # Wrap in full HTML if needed
            if not html_content.startswith("<!DOCTYPE") and not html_content.startswith("<html"):
                html_content = f"""<!DOCTYPE html>
<html>
<head>
    <meta charset="utf-8">
    <title>{chart.title}</title>
</head>
<body>
    {html_content}
</body>
</html>"""

            Path(output_path).write_text(html_content, encoding='utf-8')
            logger.info(f"Saved HTML chart to {output_path}")

        elif format in ["png", "svg"]:
            if not MATPLOTLIB_AVAILABLE:
                raise ImportError("matplotlib is required for PNG/SVG output")

            # Generate with matplotlib
            image_bytes = self.generate_chart(chart, interactive=False)

            if format == "svg":
                # Re-generate as SVG
                # This is a simplified approach - in production, you'd want to
                # modify the matplotlib generation to support SVG directly
                raise NotImplementedError("SVG format not yet implemented")

            Path(output_path).write_bytes(image_bytes)
            logger.info(f"Saved {format.upper()} chart to {output_path}")

        else:
            raise ValueError(
                f"Unsupported format: {format}. "
                f"Supported formats: html, png, svg"
            )

uaef.reporting.pipeline

Reporting pipeline — single entry point for generating reports from evaluation results.

Provides generate_report() for single-run analysis and generate_comparison_report() for multi-run comparison with regression detection.

Output structure

output/reports// report.md — Main markdown report with embedded image links images/ — Generated plot images (PNG)

Examples:

>>> from uaef.reporting.pipeline import generate_report, generate_comparison_report
>>> from uaef.utils import load_metric_results
>>>
>>> results = load_metric_results("output/evaluation-results/run.json")
>>> report_dir = generate_report(results)
>>>
>>> runs = [load_metric_results(p) for p in paths]
>>> report_dir = generate_comparison_report(runs, labels=["Day 1", "Day 2", "Day 3"])

generate_comparison_report(runs, *, labels=None, title=None, output_dir=REPORTS_OUTPUT_DIR, report_name=None)

Generate a multi-run comparison report with regression detection.

Sections included: - Overall scores comparison table - Side-by-side metric comparison chart - Percentage change chart (vs baseline) - Full delta table (every metric across all runs) - Metric trend lines chart - Regressions & improvements summary (>5% threshold) - Dimension comparison radar overlay

Parameters:

Name	Type	Description	Default
runs	List[List[EvaluationResult]]	List of result lists, one per run (ordered chronologically)	required
labels	Optional[List[str]]	Human-readable label for each run	None
title	Optional[str]	Report title	None
output_dir	str	Base directory for report output	REPORTS_OUTPUT_DIR
report_name	Optional[str]	Folder name (auto-generated if None)	None

Returns:

Type	Description
str	Path to the report directory containing report.md and images/

Source code in uaef/reporting/pipeline.py

def generate_comparison_report(
    runs: List[List[EvaluationResult]],
    *,
    labels: Optional[List[str]] = None,
    title: Optional[str] = None,
    output_dir: str = REPORTS_OUTPUT_DIR,
    report_name: Optional[str] = None,
) -> str:
    """
    Generate a multi-run comparison report with regression detection.

    Sections included:
    - Overall scores comparison table
    - Side-by-side metric comparison chart
    - Percentage change chart (vs baseline)
    - Full delta table (every metric across all runs)
    - Metric trend lines chart
    - Regressions & improvements summary (>5% threshold)
    - Dimension comparison radar overlay

    Args:
        runs: List of result lists, one per run (ordered chronologically)
        labels: Human-readable label for each run
        title: Report title
        output_dir: Base directory for report output
        report_name: Folder name (auto-generated if None)

    Returns:
        Path to the report directory containing report.md and images/
    """
    if len(runs) < 2:
        raise ValueError("Need at least 2 runs to compare")

    if labels is None:
        labels = [f"Run {i+1}" for i in range(len(runs))]
    if len(labels) != len(runs):
        raise ValueError("labels length must match runs length")

    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    if report_name is None:
        report_name = f"comparison_report_{timestamp}"
    if title is None:
        title = f"Comparison Report — {len(runs)} runs"

    report_dir = Path(output_dir) / report_name
    images_dir = report_dir / "images"
    images_dir.mkdir(parents=True, exist_ok=True)

    logger.info(f"Generating comparison report for {len(runs)} runs")

    lines = []
    lines.append(f"# {title}")
    lines.append("")
    lines.append(f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M UTC')}*")
    lines.append(f"*Comparing {len(runs)} runs: {', '.join(labels)}*")
    lines.append("")

    # --- Overall Scores ---
    lines.append("## Overall Scores")
    lines.append("")
    lines.append("Summary statistics for each evaluation run.")
    lines.append("")
    lines.append("| Run | Avg Score | Pass Rate | Test Cases |")
    lines.append("|-----|-----------|-----------|------------|")
    for run, label in zip(runs, labels):
        avg = statistics.mean([r.overall_score for r in run])
        pr = sum(1 for r in run if r.passed) / len(run) * 100
        lines.append(f"| {label} | {avg:.3f} | {pr:.0f}% | {len(run)} |")
    lines.append("")

    # --- Side-by-Side Chart ---
    lines.append("## Side-by-Side Comparison")
    lines.append("")
    lines.append("Average metric scores for each run displayed side by side.")
    lines.append("")
    sidebyside_path = _generate_side_by_side_chart(runs, labels, images_dir)
    if sidebyside_path:
        lines.append(f"![Side-by-Side Comparison](images/{sidebyside_path.name})")
    lines.append("")

    # --- Percentage Change ---
    lines.append("## Percentage Change from Baseline")
    lines.append("")
    lines.append(f"Percentage change of each metric relative to *{labels[0]}*. Negative values indicate regression.")
    lines.append("")
    pct_path = _generate_percentage_change_chart(runs, labels, images_dir)
    if pct_path:
        lines.append(f"![Percentage Change](images/{pct_path.name})")
    lines.append("")

    # --- Delta Table ---
    lines.append("## Detailed Changes")
    lines.append("")
    lines.append(f"Per-metric averages across all runs with absolute delta (last run vs baseline).")
    lines.append("")
    lines.extend(_build_delta_table(runs, labels))
    lines.append("")

    # --- Metric Trends ---
    lines.append("## Metric Trends")
    lines.append("")
    lines.append("Line chart showing how the top changing metrics evolve across runs.")
    lines.append("")
    trend_path = _generate_trend_chart(runs, labels, images_dir)
    if trend_path:
        lines.append(f"![Metric Trends](images/{trend_path.name})")
    lines.append("")

    # --- Regressions & Improvements ---
    lines.append("## Regressions & Improvements")
    lines.append("")
    lines.append("Metrics that changed by more than 5% between baseline and the latest run.")
    lines.append("")
    lines.extend(_build_metric_trend_table(runs, labels))
    lines.append("")

    # --- Dimension Comparison Radar ---
    lines.append("## Dimension Comparison")
    lines.append("")
    lines.append("Radar overlay comparing dimension-level performance across all runs.")
    lines.append("")
    radar_path = _generate_comparison_radar(runs, labels, images_dir)
    if radar_path:
        lines.append(f"![Dimension Comparison](images/{radar_path.name})")
    lines.append("")

    report_path = report_dir / "report.md"
    report_path.write_text("\n".join(lines), encoding="utf-8")
    logger.info(f"Comparison report saved to {report_dir}")
    return str(report_dir)

generate_report(results, *, title=None, report_type='full', output_dir=REPORTS_OUTPUT_DIR, report_name=None, include_recommendations=True)

Generate a single-run evaluation report.

Sections included: - Executive summary with pass/fail badge - Dimension breakdown table + radar chart - Per-metric scores table - Statistical outlier analysis with 95% confidence intervals (full only) - Score distribution histograms (full only) - Metric correlation heatmap (full only) - Safety summary (full only) - Failures list - Recommendations

Parameters:

Name	Type	Description	Default
results	Union[EvaluationResult, List[EvaluationResult]]	EvaluationResult or list of results from a single evaluation run	required
title	Optional[str]	Report title (auto-generated if not provided)	None
report_type	str	"full" (all sections) or "executive" (summary only)	'full'
output_dir	str	Base directory for report output	REPORTS_OUTPUT_DIR
report_name	Optional[str]	Folder name (auto-generated with timestamp if None)	None
include_recommendations	bool	Whether to include the recommendations section	True

Returns:

Type	Description
str	Path to the report directory containing report.md and images/

Source code in uaef/reporting/pipeline.py

def generate_report(
    results: Union[EvaluationResult, List[EvaluationResult]],
    *,
    title: Optional[str] = None,
    report_type: str = "full",
    output_dir: str = REPORTS_OUTPUT_DIR,
    report_name: Optional[str] = None,
    include_recommendations: bool = True,
) -> str:
    """
    Generate a single-run evaluation report.

    Sections included:
    - Executive summary with pass/fail badge
    - Dimension breakdown table + radar chart
    - Per-metric scores table
    - Statistical outlier analysis with 95% confidence intervals (full only)
    - Score distribution histograms (full only)
    - Metric correlation heatmap (full only)
    - Safety summary (full only)
    - Failures list
    - Recommendations

    Args:
        results: EvaluationResult or list of results from a single evaluation run
        title: Report title (auto-generated if not provided)
        report_type: "full" (all sections) or "executive" (summary only)
        output_dir: Base directory for report output
        report_name: Folder name (auto-generated with timestamp if None)
        include_recommendations: Whether to include the recommendations section

    Returns:
        Path to the report directory containing report.md and images/
    """
    if isinstance(results, EvaluationResult):
        results = [results]
    if not results:
        raise ValueError("Cannot generate report from empty results")

    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    if report_name is None:
        report_name = f"{report_type}_report_{timestamp}"

    report_dir = Path(output_dir) / report_name
    images_dir = report_dir / "images"
    images_dir.mkdir(parents=True, exist_ok=True)

    if title is None:
        title = f"{report_type.title()} Report — {len(results)} test case(s)"

    logger.info(f"Generating {report_type} report for {len(results)} results")

    lines = []
    lines.append(f"# {title}")
    lines.append("")
    lines.append(f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M UTC')}*")
    lines.append("")

    # --- Executive Summary ---
    lines.append("## Executive Summary")
    lines.append("")
    lines.append("High-level overview of agent performance across all test cases.")
    lines.append("")
    lines.extend(_build_executive_summary(results))
    lines.append("")

    # --- Dimension Breakdown ---
    lines.append("## Dimension Breakdown")
    lines.append("")
    lines.append("Aggregate scores per evaluation dimension. Each dimension groups related metrics.")
    lines.append("")
    lines.extend(_build_dimension_table(results))
    lines.append("")
    radar_path = _generate_radar_chart(results, images_dir)
    if radar_path:
        lines.append(f"![Dimension Radar](images/{radar_path.name})")
        lines.append("")

    # --- Per-Metric Scores ---
    lines.append("## Per-Metric Scores")
    lines.append("")
    lines.append("Individual metric scores across all test cases.")
    lines.append("")
    lines.extend(_build_metric_table(results))
    lines.append("")

    if report_type == "full":
        # --- Statistical Outliers ---
        lines.append("## Statistical Outlier Analysis")
        lines.append("")
        lines.append("95% confidence intervals per metric. Outliers are scores beyond 2 standard deviations from the mean.")
        lines.append("")
        lines.extend(_build_outlier_section(results, images_dir))
        lines.append("")

        # --- Score Distributions ---
        lines.append("## Score Distributions")
        lines.append("")
        lines.append("Histograms showing the distribution shape of metric scores. Metrics with highest variance are shown.")
        lines.append("")
        hist_path = _generate_histogram(results, images_dir)
        if hist_path:
            lines.append(f"![Score Distributions](images/{hist_path.name})")
        else:
            lines.append("*Insufficient data for histograms (need ≥5 test cases).*")
        lines.append("")

        # --- Correlation Heatmap ---
        lines.append("## Metric Correlations")
        lines.append("")
        lines.append("Pearson correlation between metrics. High correlation indicates metrics that tend to move together.")
        lines.append("")
        corr_path = _generate_correlation_heatmap(results, images_dir)
        if corr_path:
            lines.append(f"![Correlation Heatmap](images/{corr_path.name})")
        else:
            lines.append("*Insufficient data for correlation analysis.*")
        lines.append("")

        # --- Safety Summary ---
        lines.append("## Safety Summary")
        lines.append("")
        lines.append("Responsible AI metrics overview. Concerns are flagged when scores drop below 0.7.")
        lines.append("")
        lines.extend(_build_safety_summary(results))
        lines.append("")

    # --- Failures ---
    failures_section = _build_failures_section(results)
    if failures_section:
        lines.append("## Failures")
        lines.append("")
        lines.append("Test cases that failed threshold checks.")
        lines.append("")
        lines.extend(failures_section)
        lines.append("")

    # --- Recommendations ---
    if include_recommendations:
        lines.append("## Recommendations")
        lines.append("")
        lines.append("Actionable suggestions based on dimension scores and metric variance.")
        lines.append("")
        lines.extend(_build_recommendations(results))
        lines.append("")

    report_path = report_dir / "report.md"
    report_path.write_text("\n".join(lines), encoding="utf-8")
    logger.info(f"Report saved to {report_dir}")
    return str(report_dir)

uaef.reporting.exporter

Report exporter for multiple output formats.

ReportExporter

Export reports and dashboards to various formats.

Supports JSON, HTML, PDF, and CSV export formats with professional styling and templates.

Examples:

>>> exporter = ReportExporter()
>>> 
>>> # Export to JSON
>>> json_str = exporter.export_to_json(report)
>>> 
>>> # Export to HTML file
>>> exporter.export_to_html(report, output_path="report.html")
>>> 
>>> # Export to PDF
>>> pdf_bytes = exporter.export_to_pdf(report)
>>> 
>>> # Export tabular data to CSV
>>> csv_str = exporter.export_to_csv(dashboard)

Source code in uaef/reporting/exporter.py

class ReportExporter:
    """
    Export reports and dashboards to various formats.

    Supports JSON, HTML, PDF, and CSV export formats with professional styling
    and templates.

    Examples:
        >>> exporter = ReportExporter()
        >>> 
        >>> # Export to JSON
        >>> json_str = exporter.export_to_json(report)
        >>> 
        >>> # Export to HTML file
        >>> exporter.export_to_html(report, output_path="report.html")
        >>> 
        >>> # Export to PDF
        >>> pdf_bytes = exporter.export_to_pdf(report)
        >>> 
        >>> # Export tabular data to CSV
        >>> csv_str = exporter.export_to_csv(dashboard)
    """

    def __init__(self):
        """Initialize the report exporter with Jinja2 templates."""
        # Set up Jinja2 environment for HTML templates
        self.jinja_env = None
        if JINJA2_AVAILABLE:
            try:
                self.jinja_env = Environment(  # nosemgrep: python.flask.security.xss.audit.direct-use-of-jinja2.direct-use-of-jinja2 -- False positive: HTML escaping is enabled via autoescape=select_autoescape(["html","xml"]); templates are packaged (PackageLoader), not user-supplied.
                    loader=PackageLoader("uaef.reporting", "templates"),
                    autoescape=select_autoescape(["html", "xml"])
                )
            except Exception:
                # If templates directory doesn't exist, jinja_env stays None
                pass

    def export_to_json(
        self,
        report: Union[Report, Dashboard],
        output_path: Optional[Union[str, Path]] = None,
        indent: int = 2
    ) -> str:
        """
        Export report or dashboard to JSON format.

        Uses Pydantic's model_dump() method to serialize the data with proper
        type handling for UUIDs, datetimes, and enums.

        Args:
            report: Report or Dashboard to export
            output_path: Optional file path to write JSON to
            indent: Number of spaces for JSON indentation (default: 2)

        Returns:
            JSON string representation

        Raises:
            ValueError: If report is None or invalid
            IOError: If output_path is specified but file cannot be written

        Examples:
            >>> exporter = ReportExporter()
            >>> json_str = exporter.export_to_json(report)
            >>> print(json_str[:100])
            {
              "report_id": "550e8400-e29b-41d4-a716-446655440000",
              "title": "Executive Report",
            ...

            >>> # Write to file
            >>> exporter.export_to_json(report, output_path="report.json")
        """
        if report is None:
            raise ValueError("Report cannot be None")

        try:
            # Use Pydantic's model_dump for proper serialization
            data = report.model_dump(mode="json")
            json_str = json.dumps(data, indent=indent, ensure_ascii=False)

            # Write to file if path specified
            if output_path:
                output_path = Path(output_path)
                output_path.parent.mkdir(parents=True, exist_ok=True)
                output_path.write_text(json_str, encoding="utf-8")

            return json_str

        except Exception as e:
            raise ValueError(f"Failed to export to JSON: {str(e)}") from e

    def export_to_html(
        self,
        report: Union[Report, Dashboard],
        output_path: Optional[Union[str, Path]] = None,
        template_name: Optional[str] = None
    ) -> str:
        """
        Export report or dashboard to HTML format with professional styling.

        Uses Jinja2 templates for rendering. If templates are not available,
        generates basic HTML structure.

        Args:
            report: Report or Dashboard to export
            output_path: Optional file path to write HTML to
            template_name: Optional custom template name (default: auto-detect)

        Returns:
            HTML string

        Raises:
            ValueError: If report is None or invalid
            IOError: If output_path is specified but file cannot be written

        Examples:
            >>> exporter = ReportExporter()
            >>> html = exporter.export_to_html(report)
            >>> 
            >>> # Write to file
            >>> exporter.export_to_html(report, output_path="report.html")
            >>> 
            >>> # Use custom template
            >>> html = exporter.export_to_html(
            ...     report,
            ...     template_name="custom_report.html"
            ... )
        """
        if report is None:
            raise ValueError("Report cannot be None")

        try:
            # Determine template name if not specified
            if template_name is None:
                if isinstance(report, Report):
                    template_name = "report.html"
                else:  # Dashboard
                    template_name = "dashboard.html"

            # Try to use Jinja2 template if available
            if self.jinja_env:
                try:
                    template = self.jinja_env.get_template(template_name)
                    html = template.render(  # nosemgrep: python.flask.security.xss.audit.direct-use-of-jinja2.direct-use-of-jinja2 -- False positive: rendered through the autoescaped Environment above (select_autoescape) using packaged templates; no untrusted template source.
                        report=report,
                        data=report.model_dump(mode="json")
                    )
                except Exception:
                    # Fall back to basic HTML if template not found
                    html = self._generate_basic_html(report)
            else:
                # Generate basic HTML if Jinja2 not available
                html = self._generate_basic_html(report)

            # Write to file if path specified
            if output_path:
                output_path = Path(output_path)
                output_path.parent.mkdir(parents=True, exist_ok=True)
                output_path.write_text(html, encoding="utf-8")

            return html

        except Exception as e:
            raise ValueError(f"Failed to export to HTML: {str(e)}") from e

    def _generate_basic_html(self, report: Union[Report, Dashboard]) -> str:
        """
        Generate basic HTML without templates.

        Args:
            report: Report or Dashboard to convert

        Returns:
            Basic HTML string with inline CSS
        """
        if isinstance(report, Report):
            return self._generate_report_html(report)
        else:
            return self._generate_dashboard_html(report)

    def _generate_report_html(self, report: Report) -> str:
        """Generate HTML for a Report."""
        html = f"""<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>{report.title}</title>
    <style>
        body {{
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
            line-height: 1.6;
            color: #333;
            max-width: 1200px;
            margin: 0 auto;
            padding: 20px;
            background-color: #f5f5f5;
        }}
        .header {{
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px;
            border-radius: 8px;
            margin-bottom: 30px;
        }}
        .header h1 {{
            margin: 0 0 10px 0;
            font-size: 2em;
        }}
        .header .meta {{
            opacity: 0.9;
            font-size: 0.9em;
        }}
        .summary {{
            background: white;
            padding: 20px;
            border-radius: 8px;
            margin-bottom: 20px;
            border-left: 4px solid #667eea;
        }}
        .section {{
            background: white;
            padding: 20px;
            border-radius: 8px;
            margin-bottom: 20px;
        }}
        .section h2 {{
            color: #667eea;
            margin-top: 0;
            border-bottom: 2px solid #f0f0f0;
            padding-bottom: 10px;
        }}
        .dashboard {{
            background: white;
            padding: 20px;
            border-radius: 8px;
            margin-bottom: 20px;
        }}
        .chart {{
            background: #f9f9f9;
            padding: 15px;
            border-radius: 4px;
            margin: 10px 0;
            border: 1px solid #e0e0e0;
        }}
        .chart h3 {{
            margin-top: 0;
            color: #555;
        }}
        .footer {{
            text-align: center;
            color: #888;
            margin-top: 40px;
            padding-top: 20px;
            border-top: 1px solid #ddd;
        }}
    </style>
</head>
<body>
    <div class="header">
        <h1>{report.title}</h1>
        <div class="meta">
            <strong>Type:</strong> {report.report_type.value} | 
            <strong>Created:</strong> {report.created_at.strftime('%Y-%m-%d %H:%M:%S UTC')}
        </div>
    </div>

    <div class="summary">
        <h2>Executive Summary</h2>
        <p>{report.summary}</p>
    </div>
"""

        # Add sections
        for section in report.sections:
            html += f"""
    <div class="section">
        <h2>{section['title']}</h2>
        <p>{section['content']}</p>
    </div>
"""

        # Add dashboard charts
        html += f"""
    <div class="dashboard">
        <h2>Dashboard: {report.dashboard.title}</h2>
"""
        if report.dashboard.description:
            html += f"        <p>{report.dashboard.description}</p>\n"

        for chart in report.dashboard.charts:
            html += f"""
        <div class="chart">
            <h3>{chart.title}</h3>
"""
            if chart.description:
                html += f"            <p>{chart.description}</p>\n"
            html += f"            <p><em>Chart Type:</em> {chart.chart_type.value}</p>\n"
            html += "        </div>\n"

        html += """    </div>

    <div class="footer">
        <p>Generated by Universal Agent Evaluation Framework (UAEF)</p>
    </div>
</body>
</html>
"""
        return html

    def _generate_dashboard_html(self, dashboard: Dashboard) -> str:
        """Generate HTML for a Dashboard."""
        html = f"""<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>{dashboard.title}</title>
    <style>
        body {{
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
            line-height: 1.6;
            color: #333;
            max-width: 1200px;
            margin: 0 auto;
            padding: 20px;
            background-color: #f5f5f5;
        }}
        .header {{
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px;
            border-radius: 8px;
            margin-bottom: 30px;
        }}
        .header h1 {{
            margin: 0 0 10px 0;
            font-size: 2em;
        }}
        .charts-grid {{
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
            gap: 20px;
        }}
        .chart {{
            background: white;
            padding: 20px;
            border-radius: 8px;
            border: 1px solid #e0e0e0;
        }}
        .chart h2 {{
            color: #667eea;
            margin-top: 0;
            font-size: 1.3em;
        }}
        .footer {{
            text-align: center;
            color: #888;
            margin-top: 40px;
            padding-top: 20px;
            border-top: 1px solid #ddd;
        }}
    </style>
</head>
<body>
    <div class="header">
        <h1>{dashboard.title}</h1>
"""
        if dashboard.description:
            html += f"        <p>{dashboard.description}</p>\n"
        html += f"""        <p><small>Created: {dashboard.created_at.strftime('%Y-%m-%d %H:%M:%S UTC')}</small></p>
    </div>

    <div class="charts-grid">
"""

        for chart in dashboard.charts:
            html += f"""
        <div class="chart">
            <h2>{chart.title}</h2>
"""
            if chart.description:
                html += f"            <p>{chart.description}</p>\n"
            html += f"            <p><strong>Type:</strong> {chart.chart_type.value}</p>\n"
            html += "        </div>\n"

        html += """    </div>

    <div class="footer">
        <p>Generated by Universal Agent Evaluation Framework (UAEF)</p>
    </div>
</body>
</html>
"""
        return html

    def export_to_pdf(
        self,
        report: Union[Report, Dashboard],
        output_path: Optional[Union[str, Path]] = None,
        use_weasyprint: bool = True
    ) -> bytes:
        """
        Export report or dashboard to PDF format.

        Supports two PDF generation methods:
        1. weasyprint (default): HTML-to-PDF conversion with CSS support
        2. reportlab: Direct PDF generation (fallback)

        Args:
            report: Report or Dashboard to export
            output_path: Optional file path to write PDF to
            use_weasyprint: Use weasyprint if available (default: True)

        Returns:
            PDF content as bytes

        Raises:
            ValueError: If report is None or invalid
            ImportError: If neither weasyprint nor reportlab is available
            IOError: If output_path is specified but file cannot be written

        Examples:
            >>> exporter = ReportExporter()
            >>> pdf_bytes = exporter.export_to_pdf(report)
            >>> 
            >>> # Write to file
            >>> exporter.export_to_pdf(report, output_path="report.pdf")
            >>> 
            >>> # Force reportlab usage
            >>> pdf_bytes = exporter.export_to_pdf(report, use_weasyprint=False)
        """
        if report is None:
            raise ValueError("Report cannot be None")

        try:
            # Try weasyprint first (better HTML/CSS support)
            if use_weasyprint:
                try:
                    from weasyprint import HTML

                    # Generate HTML
                    html = self.export_to_html(report)

                    # Convert to PDF
                    pdf_bytes = HTML(string=html).write_pdf()

                    # Write to file if path specified
                    if output_path:
                        output_path = Path(output_path)
                        output_path.parent.mkdir(parents=True, exist_ok=True)
                        output_path.write_bytes(pdf_bytes)

                    return pdf_bytes

                except ImportError:
                    # Fall back to reportlab
                    pass

            # Use reportlab as fallback
            try:
                from reportlab.lib import colors
                from reportlab.lib.pagesizes import letter
                from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
                from reportlab.lib.units import inch
                from reportlab.platypus import (
                    PageBreak,
                    Paragraph,
                    SimpleDocTemplate,
                    Spacer,
                    Table,
                    TableStyle,
                )

                # Create PDF buffer
                buffer = BytesIO()
                doc = SimpleDocTemplate(buffer, pagesize=letter)
                story = []
                styles = getSampleStyleSheet()

                # Custom styles
                title_style = ParagraphStyle(
                    'CustomTitle',
                    parent=styles['Heading1'],
                    fontSize=24,
                    textColor=colors.HexColor('#667eea'),
                    spaceAfter=30
                )
                heading_style = ParagraphStyle(
                    'CustomHeading',
                    parent=styles['Heading2'],
                    fontSize=16,
                    textColor=colors.HexColor('#667eea'),
                    spaceAfter=12
                )

                # Add content based on type
                if isinstance(report, Report):
                    # Title
                    story.append(Paragraph(report.title, title_style))
                    story.append(Spacer(1, 0.2 * inch))

                    # Metadata
                    meta_text = f"<b>Type:</b> {report.report_type.value} | <b>Created:</b> {report.created_at.strftime('%Y-%m-%d %H:%M:%S UTC')}"
                    story.append(Paragraph(meta_text, styles['Normal']))
                    story.append(Spacer(1, 0.3 * inch))

                    # Summary
                    story.append(Paragraph("Executive Summary", heading_style))
                    story.append(Paragraph(report.summary, styles['Normal']))
                    story.append(Spacer(1, 0.3 * inch))

                    # Sections
                    for section in report.sections:
                        story.append(Paragraph(section['title'], heading_style))
                        story.append(Paragraph(section['content'], styles['Normal']))
                        story.append(Spacer(1, 0.2 * inch))

                    # Dashboard info
                    story.append(PageBreak())
                    story.append(Paragraph(f"Dashboard: {report.dashboard.title}", heading_style))
                    if report.dashboard.description:
                        story.append(Paragraph(report.dashboard.description, styles['Normal']))
                    story.append(Spacer(1, 0.2 * inch))

                    # Charts
                    for chart in report.dashboard.charts:
                        chart_text = f"<b>{chart.title}</b> ({chart.chart_type.value})"
                        story.append(Paragraph(chart_text, styles['Normal']))
                        if chart.description:
                            story.append(Paragraph(chart.description, styles['Normal']))
                        story.append(Spacer(1, 0.1 * inch))

                else:  # Dashboard
                    # Title
                    story.append(Paragraph(report.title, title_style))
                    story.append(Spacer(1, 0.2 * inch))

                    if report.description:
                        story.append(Paragraph(report.description, styles['Normal']))
                        story.append(Spacer(1, 0.3 * inch))

                    # Charts
                    for chart in report.charts:
                        story.append(Paragraph(chart.title, heading_style))
                        chart_info = f"<b>Type:</b> {chart.chart_type.value}"
                        story.append(Paragraph(chart_info, styles['Normal']))
                        if chart.description:
                            story.append(Paragraph(chart.description, styles['Normal']))
                        story.append(Spacer(1, 0.2 * inch))

                # Build PDF
                doc.build(story)
                pdf_bytes = buffer.getvalue()
                buffer.close()

                # Write to file if path specified
                if output_path:
                    output_path = Path(output_path)
                    output_path.parent.mkdir(parents=True, exist_ok=True)
                    output_path.write_bytes(pdf_bytes)

                return pdf_bytes

            except ImportError as e:
                raise ImportError(
                    "PDF export requires either 'weasyprint' or 'reportlab'. "
                    "Install with: pip install weasyprint  OR  pip install reportlab"
                ) from e

        except Exception as e:
            raise ValueError(f"Failed to export to PDF: {str(e)}") from e

    def export_to_csv(
        self,
        data: Union[Report, Dashboard],
        output_path: Optional[Union[str, Path]] = None,
        include_metadata: bool = True
    ) -> str:
        """
        Export tabular data to CSV format.

        Extracts tabular data from charts and report sections. For reports,
        exports dimension scores, metric scores, and chart data. For dashboards,
        exports all chart data in tabular format.

        Args:
            data: Report or Dashboard to export
            output_path: Optional file path to write CSV to
            include_metadata: Include metadata rows at top (default: True)

        Returns:
            CSV string

        Raises:
            ValueError: If data is None or contains no tabular data
            IOError: If output_path is specified but file cannot be written

        Examples:
            >>> exporter = ReportExporter()
            >>> csv_str = exporter.export_to_csv(dashboard)
            >>> print(csv_str)
            Chart,Metric,Value
            Dimension Scores,Tool Calling,0.85
            Dimension Scores,Response Quality,0.92
            ...

            >>> # Write to file
            >>> exporter.export_to_csv(report, output_path="report.csv")
        """
        if data is None:
            raise ValueError("Data cannot be None")

        try:
            output = StringIO()
            writer = csv.writer(output)

            # Add metadata header if requested
            if include_metadata:
                if isinstance(data, Report):
                    writer.writerow(["Report Title", data.title])
                    writer.writerow(["Report Type", data.report_type.value])
                    writer.writerow(["Created", data.created_at.isoformat()])
                    writer.writerow([])  # Blank line
                else:  # Dashboard
                    writer.writerow(["Dashboard Title", data.title])
                    writer.writerow(["Created", data.created_at.isoformat()])
                    writer.writerow([])  # Blank line

            # Extract tabular data from charts
            dashboard = data.dashboard if isinstance(data, Report) else data

            for chart in dashboard.charts:
                # Write chart header
                writer.writerow([f"Chart: {chart.title}"])
                writer.writerow([f"Type: {chart.chart_type.value}"])

                # Extract data based on chart type
                chart_data = chart.data

                # Handle common data structures
                if "dimensions" in chart_data and "scores" in chart_data:
                    # Dimension scores
                    writer.writerow(["Dimension", "Score"])
                    for dim, score in zip(chart_data["dimensions"], chart_data["scores"]):
                        writer.writerow([dim, score])

                elif "metrics" in chart_data and "values" in chart_data:
                    # Metric values
                    writer.writerow(["Metric", "Value"])
                    for metric, value in zip(chart_data["metrics"], chart_data["values"]):
                        writer.writerow([metric, value])

                elif "timestamps" in chart_data and "values" in chart_data:
                    # Time series data
                    writer.writerow(["Timestamp", "Value"])
                    for ts, val in zip(chart_data["timestamps"], chart_data["values"]):
                        writer.writerow([ts, val])

                elif "matrix" in chart_data:
                    # Matrix data (heatmap)
                    matrix = chart_data["matrix"]
                    x_labels = chart_data.get("x_labels", [])
                    y_labels = chart_data.get("y_labels", [])

                    # Write header row
                    header = [""] + x_labels if x_labels else [""]
                    writer.writerow(header)

                    # Write data rows
                    for i, row in enumerate(matrix):
                        row_label = y_labels[i] if i < len(y_labels) else f"Row {i}"
                        writer.writerow([row_label] + row)

                elif "labels" in chart_data and "values" in chart_data:
                    # Generic labeled data
                    writer.writerow(["Label", "Value"])
                    for label, value in zip(chart_data["labels"], chart_data["values"]):
                        writer.writerow([label, value])

                else:
                    # Generic key-value pairs
                    writer.writerow(["Key", "Value"])
                    for key, value in chart_data.items():
                        if isinstance(value, (list, dict)):
                            value = json.dumps(value)
                        writer.writerow([key, value])

                writer.writerow([])  # Blank line between charts

            csv_str = output.getvalue()
            output.close()

            # Write to file if path specified
            if output_path:
                output_path = Path(output_path)
                output_path.parent.mkdir(parents=True, exist_ok=True)
                output_path.write_text(csv_str, encoding="utf-8")

            return csv_str

        except Exception as e:
            raise ValueError(f"Failed to export to CSV: {str(e)}") from e

    def export_to_markdown(
        self,
        data: Union[Report, Dashboard],
        output_path: Optional[Union[str, Path]] = None,
        include_metadata: bool = True,
    ) -> str:
        """
        Export a Report or Dashboard to Markdown format.

        Renders the summary, sections, and chart data as readable Markdown
        tables and bullet lists suitable for CI logs, PR comments, or docs.

        Args:
            data: Report or Dashboard to export
            output_path: Optional file path to write the Markdown to
            include_metadata: Include metadata header (default: True)

        Returns:
            Markdown string

        Raises:
            ValueError: If data is None

        Examples:
            >>> exporter = ReportExporter()
            >>> md = exporter.export_to_markdown(report)
            >>> print(md)

            >>> exporter.export_to_markdown(dashboard, output_path="report.md")
        """
        if data is None:
            raise ValueError("Data cannot be None")

        lines: list = []

        is_report = isinstance(data, Report)
        title = data.title
        dashboard = data.dashboard if is_report else data

        # Title
        lines.append(f"# {title}")
        lines.append("")

        # Metadata
        if include_metadata:
            created = data.created_at.strftime("%Y-%m-%d %H:%M UTC")
            lines.append(f"*Generated: {created}*")
            if is_report:
                lines.append(f"*Report type: {data.report_type.value}*")
            lines.append("")

        # Summary (reports only)
        if is_report and data.summary:
            lines.append("## Summary")
            lines.append("")
            lines.append(data.summary)
            lines.append("")

        # Sections (reports only)
        if is_report and data.sections:
            for section in data.sections:
                lines.append(f"## {section['title']}")
                lines.append("")
                lines.append(section["content"])
                lines.append("")

        # Charts as tables
        if dashboard.charts:
            lines.append("## Metrics")
            lines.append("")
            for chart in dashboard.charts:
                lines.append(f"### {chart.title}")
                if chart.description:
                    lines.append(f"_{chart.description}_")
                lines.append("")
                lines.extend(self._chart_to_markdown_table(chart))
                lines.append("")

        md_str = "\n".join(lines)

        if output_path:
            output_path = Path(output_path)
            output_path.parent.mkdir(parents=True, exist_ok=True)
            output_path.write_text(md_str, encoding="utf-8")

        return md_str

    @staticmethod
    def _chart_to_markdown_table(chart) -> list:
        """Convert a Chart's data dict into Markdown table lines."""
        lines: list = []
        chart_data = chart.data

        if "dimensions" in chart_data and "scores" in chart_data:
            lines.append("| Dimension | Score |")
            lines.append("|-----------|-------|")
            for dim, score in zip(chart_data["dimensions"], chart_data["scores"]):
                score_str = f"{score:.2f}" if isinstance(score, (int, float)) else str(score)
                lines.append(f"| {dim} | {score_str} |")

        elif "metrics" in chart_data and ("values" in chart_data or "scores" in chart_data):
            values = chart_data.get("values", chart_data.get("scores", []))
            lines.append("| Metric | Value |")
            lines.append("|--------|-------|")
            for metric, value in zip(chart_data["metrics"], values):
                val_str = f"{value:.2f}" if isinstance(value, (int, float)) else str(value)
                lines.append(f"| {metric} | {val_str} |")

        elif "labels" in chart_data and "values" in chart_data:
            lines.append("| Label | Value |")
            lines.append("|-------|-------|")
            for label, value in zip(chart_data["labels"], chart_data["values"]):
                val_str = f"{value:.2f}" if isinstance(value, (int, float)) else str(value)
                lines.append(f"| {label} | {val_str} |")

        elif "matrix" in chart_data:
            x_labels = chart_data.get("x_labels", [])
            y_labels = chart_data.get("y_labels", [])
            matrix = chart_data["matrix"]
            header = "| |" + "|".join(f" {x} " for x in x_labels) + "|"
            sep = "|---|" + "|".join("---" for _ in x_labels) + "|"
            lines.append(header)
            lines.append(sep)
            for i, row in enumerate(matrix):
                row_label = y_labels[i] if i < len(y_labels) else f"Row {i}"
                cells = "|".join(f" {v:.2f} " if isinstance(v, float) else f" {v} " for v in row)
                lines.append(f"| {row_label} |{cells}|")

        else:
            # Fallback: key-value list
            for key, value in chart_data.items():
                if isinstance(value, (list, dict)):
                    value = json.dumps(value, default=str)
                lines.append(f"- **{key}**: {value}")

        return lines

export_to_csv(data, output_path=None, include_metadata=True)

Export tabular data to CSV format.

Extracts tabular data from charts and report sections. For reports, exports dimension scores, metric scores, and chart data. For dashboards, exports all chart data in tabular format.

Parameters:

Name	Type	Description	Default
data	Union[Report, Dashboard]	Report or Dashboard to export	required
output_path	Optional[Union[str, Path]]	Optional file path to write CSV to	None
include_metadata	bool	Include metadata rows at top (default: True)	True

Returns:

Type	Description
str	CSV string

Raises:

Type	Description
ValueError	If data is None or contains no tabular data
IOError	If output_path is specified but file cannot be written

Examples:

>>> exporter = ReportExporter()
>>> csv_str = exporter.export_to_csv(dashboard)
>>> print(csv_str)
Chart,Metric,Value
Dimension Scores,Tool Calling,0.85
Dimension Scores,Response Quality,0.92
...

>>> # Write to file
>>> exporter.export_to_csv(report, output_path="report.csv")

Source code in uaef/reporting/exporter.py

def export_to_csv(
    self,
    data: Union[Report, Dashboard],
    output_path: Optional[Union[str, Path]] = None,
    include_metadata: bool = True
) -> str:
    """
    Export tabular data to CSV format.

    Extracts tabular data from charts and report sections. For reports,
    exports dimension scores, metric scores, and chart data. For dashboards,
    exports all chart data in tabular format.

    Args:
        data: Report or Dashboard to export
        output_path: Optional file path to write CSV to
        include_metadata: Include metadata rows at top (default: True)

    Returns:
        CSV string

    Raises:
        ValueError: If data is None or contains no tabular data
        IOError: If output_path is specified but file cannot be written

    Examples:
        >>> exporter = ReportExporter()
        >>> csv_str = exporter.export_to_csv(dashboard)
        >>> print(csv_str)
        Chart,Metric,Value
        Dimension Scores,Tool Calling,0.85
        Dimension Scores,Response Quality,0.92
        ...

        >>> # Write to file
        >>> exporter.export_to_csv(report, output_path="report.csv")
    """
    if data is None:
        raise ValueError("Data cannot be None")

    try:
        output = StringIO()
        writer = csv.writer(output)

        # Add metadata header if requested
        if include_metadata:
            if isinstance(data, Report):
                writer.writerow(["Report Title", data.title])
                writer.writerow(["Report Type", data.report_type.value])
                writer.writerow(["Created", data.created_at.isoformat()])
                writer.writerow([])  # Blank line
            else:  # Dashboard
                writer.writerow(["Dashboard Title", data.title])
                writer.writerow(["Created", data.created_at.isoformat()])
                writer.writerow([])  # Blank line

        # Extract tabular data from charts
        dashboard = data.dashboard if isinstance(data, Report) else data

        for chart in dashboard.charts:
            # Write chart header
            writer.writerow([f"Chart: {chart.title}"])
            writer.writerow([f"Type: {chart.chart_type.value}"])

            # Extract data based on chart type
            chart_data = chart.data

            # Handle common data structures
            if "dimensions" in chart_data and "scores" in chart_data:
                # Dimension scores
                writer.writerow(["Dimension", "Score"])
                for dim, score in zip(chart_data["dimensions"], chart_data["scores"]):
                    writer.writerow([dim, score])

            elif "metrics" in chart_data and "values" in chart_data:
                # Metric values
                writer.writerow(["Metric", "Value"])
                for metric, value in zip(chart_data["metrics"], chart_data["values"]):
                    writer.writerow([metric, value])

            elif "timestamps" in chart_data and "values" in chart_data:
                # Time series data
                writer.writerow(["Timestamp", "Value"])
                for ts, val in zip(chart_data["timestamps"], chart_data["values"]):
                    writer.writerow([ts, val])

            elif "matrix" in chart_data:
                # Matrix data (heatmap)
                matrix = chart_data["matrix"]
                x_labels = chart_data.get("x_labels", [])
                y_labels = chart_data.get("y_labels", [])

                # Write header row
                header = [""] + x_labels if x_labels else [""]
                writer.writerow(header)

                # Write data rows
                for i, row in enumerate(matrix):
                    row_label = y_labels[i] if i < len(y_labels) else f"Row {i}"
                    writer.writerow([row_label] + row)

            elif "labels" in chart_data and "values" in chart_data:
                # Generic labeled data
                writer.writerow(["Label", "Value"])
                for label, value in zip(chart_data["labels"], chart_data["values"]):
                    writer.writerow([label, value])

            else:
                # Generic key-value pairs
                writer.writerow(["Key", "Value"])
                for key, value in chart_data.items():
                    if isinstance(value, (list, dict)):
                        value = json.dumps(value)
                    writer.writerow([key, value])

            writer.writerow([])  # Blank line between charts

        csv_str = output.getvalue()
        output.close()

        # Write to file if path specified
        if output_path:
            output_path = Path(output_path)
            output_path.parent.mkdir(parents=True, exist_ok=True)
            output_path.write_text(csv_str, encoding="utf-8")

        return csv_str

    except Exception as e:
        raise ValueError(f"Failed to export to CSV: {str(e)}") from e

export_to_html(report, output_path=None, template_name=None)

Export report or dashboard to HTML format with professional styling.

Uses Jinja2 templates for rendering. If templates are not available, generates basic HTML structure.

Parameters:

Name	Type	Description	Default
report	Union[Report, Dashboard]	Report or Dashboard to export	required
output_path	Optional[Union[str, Path]]	Optional file path to write HTML to	None
template_name	Optional[str]	Optional custom template name (default: auto-detect)	None

Returns:

Type	Description
str	HTML string

Raises:

Type	Description
ValueError	If report is None or invalid
IOError	If output_path is specified but file cannot be written

Examples:

>>> exporter = ReportExporter()
>>> html = exporter.export_to_html(report)
>>> 
>>> # Write to file
>>> exporter.export_to_html(report, output_path="report.html")
>>> 
>>> # Use custom template
>>> html = exporter.export_to_html(
...     report,
...     template_name="custom_report.html"
... )

Source code in uaef/reporting/exporter.py

def export_to_html(
    self,
    report: Union[Report, Dashboard],
    output_path: Optional[Union[str, Path]] = None,
    template_name: Optional[str] = None
) -> str:
    """
    Export report or dashboard to HTML format with professional styling.

    Uses Jinja2 templates for rendering. If templates are not available,
    generates basic HTML structure.

    Args:
        report: Report or Dashboard to export
        output_path: Optional file path to write HTML to
        template_name: Optional custom template name (default: auto-detect)

    Returns:
        HTML string

    Raises:
        ValueError: If report is None or invalid
        IOError: If output_path is specified but file cannot be written

    Examples:
        >>> exporter = ReportExporter()
        >>> html = exporter.export_to_html(report)
        >>> 
        >>> # Write to file
        >>> exporter.export_to_html(report, output_path="report.html")
        >>> 
        >>> # Use custom template
        >>> html = exporter.export_to_html(
        ...     report,
        ...     template_name="custom_report.html"
        ... )
    """
    if report is None:
        raise ValueError("Report cannot be None")

    try:
        # Determine template name if not specified
        if template_name is None:
            if isinstance(report, Report):
                template_name = "report.html"
            else:  # Dashboard
                template_name = "dashboard.html"

        # Try to use Jinja2 template if available
        if self.jinja_env:
            try:
                template = self.jinja_env.get_template(template_name)
                html = template.render(  # nosemgrep: python.flask.security.xss.audit.direct-use-of-jinja2.direct-use-of-jinja2 -- False positive: rendered through the autoescaped Environment above (select_autoescape) using packaged templates; no untrusted template source.
                    report=report,
                    data=report.model_dump(mode="json")
                )
            except Exception:
                # Fall back to basic HTML if template not found
                html = self._generate_basic_html(report)
        else:
            # Generate basic HTML if Jinja2 not available
            html = self._generate_basic_html(report)

        # Write to file if path specified
        if output_path:
            output_path = Path(output_path)
            output_path.parent.mkdir(parents=True, exist_ok=True)
            output_path.write_text(html, encoding="utf-8")

        return html

    except Exception as e:
        raise ValueError(f"Failed to export to HTML: {str(e)}") from e

export_to_json(report, output_path=None, indent=2)

Export report or dashboard to JSON format.

Uses Pydantic's model_dump() method to serialize the data with proper type handling for UUIDs, datetimes, and enums.

Parameters:

Name	Type	Description	Default
report	Union[Report, Dashboard]	Report or Dashboard to export	required
output_path	Optional[Union[str, Path]]	Optional file path to write JSON to	None
indent	int	Number of spaces for JSON indentation (default: 2)	2

Returns:

Type	Description
str	JSON string representation

Raises:

Type	Description
ValueError	If report is None or invalid
IOError	If output_path is specified but file cannot be written

Examples:

>>> exporter = ReportExporter()
>>> json_str = exporter.export_to_json(report)
>>> print(json_str[:100])
{
  "report_id": "550e8400-e29b-41d4-a716-446655440000",
  "title": "Executive Report",
...

>>> # Write to file
>>> exporter.export_to_json(report, output_path="report.json")

Source code in uaef/reporting/exporter.py

def export_to_json(
    self,
    report: Union[Report, Dashboard],
    output_path: Optional[Union[str, Path]] = None,
    indent: int = 2
) -> str:
    """
    Export report or dashboard to JSON format.

    Uses Pydantic's model_dump() method to serialize the data with proper
    type handling for UUIDs, datetimes, and enums.

    Args:
        report: Report or Dashboard to export
        output_path: Optional file path to write JSON to
        indent: Number of spaces for JSON indentation (default: 2)

    Returns:
        JSON string representation

    Raises:
        ValueError: If report is None or invalid
        IOError: If output_path is specified but file cannot be written

    Examples:
        >>> exporter = ReportExporter()
        >>> json_str = exporter.export_to_json(report)
        >>> print(json_str[:100])
        {
          "report_id": "550e8400-e29b-41d4-a716-446655440000",
          "title": "Executive Report",
        ...

        >>> # Write to file
        >>> exporter.export_to_json(report, output_path="report.json")
    """
    if report is None:
        raise ValueError("Report cannot be None")

    try:
        # Use Pydantic's model_dump for proper serialization
        data = report.model_dump(mode="json")
        json_str = json.dumps(data, indent=indent, ensure_ascii=False)

        # Write to file if path specified
        if output_path:
            output_path = Path(output_path)
            output_path.parent.mkdir(parents=True, exist_ok=True)
            output_path.write_text(json_str, encoding="utf-8")

        return json_str

    except Exception as e:
        raise ValueError(f"Failed to export to JSON: {str(e)}") from e

export_to_markdown(data, output_path=None, include_metadata=True)

Export a Report or Dashboard to Markdown format.

Renders the summary, sections, and chart data as readable Markdown tables and bullet lists suitable for CI logs, PR comments, or docs.

Parameters:

Name	Type	Description	Default
data	Union[Report, Dashboard]	Report or Dashboard to export	required
output_path	Optional[Union[str, Path]]	Optional file path to write the Markdown to	None
include_metadata	bool	Include metadata header (default: True)	True

Returns:

Type	Description
str	Markdown string

Raises:

Type	Description
ValueError	If data is None

Examples:

>>> exporter = ReportExporter()
>>> md = exporter.export_to_markdown(report)
>>> print(md)

>>> exporter.export_to_markdown(dashboard, output_path="report.md")

Source code in uaef/reporting/exporter.py

def export_to_markdown(
    self,
    data: Union[Report, Dashboard],
    output_path: Optional[Union[str, Path]] = None,
    include_metadata: bool = True,
) -> str:
    """
    Export a Report or Dashboard to Markdown format.

    Renders the summary, sections, and chart data as readable Markdown
    tables and bullet lists suitable for CI logs, PR comments, or docs.

    Args:
        data: Report or Dashboard to export
        output_path: Optional file path to write the Markdown to
        include_metadata: Include metadata header (default: True)

    Returns:
        Markdown string

    Raises:
        ValueError: If data is None

    Examples:
        >>> exporter = ReportExporter()
        >>> md = exporter.export_to_markdown(report)
        >>> print(md)

        >>> exporter.export_to_markdown(dashboard, output_path="report.md")
    """
    if data is None:
        raise ValueError("Data cannot be None")

    lines: list = []

    is_report = isinstance(data, Report)
    title = data.title
    dashboard = data.dashboard if is_report else data

    # Title
    lines.append(f"# {title}")
    lines.append("")

    # Metadata
    if include_metadata:
        created = data.created_at.strftime("%Y-%m-%d %H:%M UTC")
        lines.append(f"*Generated: {created}*")
        if is_report:
            lines.append(f"*Report type: {data.report_type.value}*")
        lines.append("")

    # Summary (reports only)
    if is_report and data.summary:
        lines.append("## Summary")
        lines.append("")
        lines.append(data.summary)
        lines.append("")

    # Sections (reports only)
    if is_report and data.sections:
        for section in data.sections:
            lines.append(f"## {section['title']}")
            lines.append("")
            lines.append(section["content"])
            lines.append("")

    # Charts as tables
    if dashboard.charts:
        lines.append("## Metrics")
        lines.append("")
        for chart in dashboard.charts:
            lines.append(f"### {chart.title}")
            if chart.description:
                lines.append(f"_{chart.description}_")
            lines.append("")
            lines.extend(self._chart_to_markdown_table(chart))
            lines.append("")

    md_str = "\n".join(lines)

    if output_path:
        output_path = Path(output_path)
        output_path.parent.mkdir(parents=True, exist_ok=True)
        output_path.write_text(md_str, encoding="utf-8")

    return md_str

export_to_pdf(report, output_path=None, use_weasyprint=True)

Export report or dashboard to PDF format.

Supports two PDF generation methods: 1. weasyprint (default): HTML-to-PDF conversion with CSS support 2. reportlab: Direct PDF generation (fallback)

Parameters:

Name	Type	Description	Default
report	Union[Report, Dashboard]	Report or Dashboard to export	required
output_path	Optional[Union[str, Path]]	Optional file path to write PDF to	None
use_weasyprint	bool	Use weasyprint if available (default: True)	True

Returns:

Type	Description
bytes	PDF content as bytes

Raises:

Type	Description
ValueError	If report is None or invalid
ImportError	If neither weasyprint nor reportlab is available
IOError	If output_path is specified but file cannot be written

Examples:

>>> exporter = ReportExporter()
>>> pdf_bytes = exporter.export_to_pdf(report)
>>> 
>>> # Write to file
>>> exporter.export_to_pdf(report, output_path="report.pdf")
>>> 
>>> # Force reportlab usage
>>> pdf_bytes = exporter.export_to_pdf(report, use_weasyprint=False)

Source code in uaef/reporting/exporter.py

def export_to_pdf(
    self,
    report: Union[Report, Dashboard],
    output_path: Optional[Union[str, Path]] = None,
    use_weasyprint: bool = True
) -> bytes:
    """
    Export report or dashboard to PDF format.

    Supports two PDF generation methods:
    1. weasyprint (default): HTML-to-PDF conversion with CSS support
    2. reportlab: Direct PDF generation (fallback)

    Args:
        report: Report or Dashboard to export
        output_path: Optional file path to write PDF to
        use_weasyprint: Use weasyprint if available (default: True)

    Returns:
        PDF content as bytes

    Raises:
        ValueError: If report is None or invalid
        ImportError: If neither weasyprint nor reportlab is available
        IOError: If output_path is specified but file cannot be written

    Examples:
        >>> exporter = ReportExporter()
        >>> pdf_bytes = exporter.export_to_pdf(report)
        >>> 
        >>> # Write to file
        >>> exporter.export_to_pdf(report, output_path="report.pdf")
        >>> 
        >>> # Force reportlab usage
        >>> pdf_bytes = exporter.export_to_pdf(report, use_weasyprint=False)
    """
    if report is None:
        raise ValueError("Report cannot be None")

    try:
        # Try weasyprint first (better HTML/CSS support)
        if use_weasyprint:
            try:
                from weasyprint import HTML

                # Generate HTML
                html = self.export_to_html(report)

                # Convert to PDF
                pdf_bytes = HTML(string=html).write_pdf()

                # Write to file if path specified
                if output_path:
                    output_path = Path(output_path)
                    output_path.parent.mkdir(parents=True, exist_ok=True)
                    output_path.write_bytes(pdf_bytes)

                return pdf_bytes

            except ImportError:
                # Fall back to reportlab
                pass

        # Use reportlab as fallback
        try:
            from reportlab.lib import colors
            from reportlab.lib.pagesizes import letter
            from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
            from reportlab.lib.units import inch
            from reportlab.platypus import (
                PageBreak,
                Paragraph,
                SimpleDocTemplate,
                Spacer,
                Table,
                TableStyle,
            )

            # Create PDF buffer
            buffer = BytesIO()
            doc = SimpleDocTemplate(buffer, pagesize=letter)
            story = []
            styles = getSampleStyleSheet()

            # Custom styles
            title_style = ParagraphStyle(
                'CustomTitle',
                parent=styles['Heading1'],
                fontSize=24,
                textColor=colors.HexColor('#667eea'),
                spaceAfter=30
            )
            heading_style = ParagraphStyle(
                'CustomHeading',
                parent=styles['Heading2'],
                fontSize=16,
                textColor=colors.HexColor('#667eea'),
                spaceAfter=12
            )

            # Add content based on type
            if isinstance(report, Report):
                # Title
                story.append(Paragraph(report.title, title_style))
                story.append(Spacer(1, 0.2 * inch))

                # Metadata
                meta_text = f"<b>Type:</b> {report.report_type.value} | <b>Created:</b> {report.created_at.strftime('%Y-%m-%d %H:%M:%S UTC')}"
                story.append(Paragraph(meta_text, styles['Normal']))
                story.append(Spacer(1, 0.3 * inch))

                # Summary
                story.append(Paragraph("Executive Summary", heading_style))
                story.append(Paragraph(report.summary, styles['Normal']))
                story.append(Spacer(1, 0.3 * inch))

                # Sections
                for section in report.sections:
                    story.append(Paragraph(section['title'], heading_style))
                    story.append(Paragraph(section['content'], styles['Normal']))
                    story.append(Spacer(1, 0.2 * inch))

                # Dashboard info
                story.append(PageBreak())
                story.append(Paragraph(f"Dashboard: {report.dashboard.title}", heading_style))
                if report.dashboard.description:
                    story.append(Paragraph(report.dashboard.description, styles['Normal']))
                story.append(Spacer(1, 0.2 * inch))

                # Charts
                for chart in report.dashboard.charts:
                    chart_text = f"<b>{chart.title}</b> ({chart.chart_type.value})"
                    story.append(Paragraph(chart_text, styles['Normal']))
                    if chart.description:
                        story.append(Paragraph(chart.description, styles['Normal']))
                    story.append(Spacer(1, 0.1 * inch))

            else:  # Dashboard
                # Title
                story.append(Paragraph(report.title, title_style))
                story.append(Spacer(1, 0.2 * inch))

                if report.description:
                    story.append(Paragraph(report.description, styles['Normal']))
                    story.append(Spacer(1, 0.3 * inch))

                # Charts
                for chart in report.charts:
                    story.append(Paragraph(chart.title, heading_style))
                    chart_info = f"<b>Type:</b> {chart.chart_type.value}"
                    story.append(Paragraph(chart_info, styles['Normal']))
                    if chart.description:
                        story.append(Paragraph(chart.description, styles['Normal']))
                    story.append(Spacer(1, 0.2 * inch))

            # Build PDF
            doc.build(story)
            pdf_bytes = buffer.getvalue()
            buffer.close()

            # Write to file if path specified
            if output_path:
                output_path = Path(output_path)
                output_path.parent.mkdir(parents=True, exist_ok=True)
                output_path.write_bytes(pdf_bytes)

            return pdf_bytes

        except ImportError as e:
            raise ImportError(
                "PDF export requires either 'weasyprint' or 'reportlab'. "
                "Install with: pip install weasyprint  OR  pip install reportlab"
            ) from e

    except Exception as e:
        raise ValueError(f"Failed to export to PDF: {str(e)}") from e