Human-in-the-Loop

Review queues, human scores, and judge calibration. Workflow: Human-in-the-Loop guide.

uaef.hitl.models

HITL (Human-in-the-Loop) review queue data models.

AgreementMetrics

Bases: BaseModel

Metrics for measuring agreement between LLM judges and human reviewers.

This model tracks agreement statistics to calibrate LLM-based evaluation and identify areas where automated evaluation needs improvement.

Attributes:

Name	Type	Description
metric_name	str	Name of the metric being evaluated
llm_score	Optional[float]	Score from the LLM judge (None if not applicable)
human_scores	List[float]	List of scores from human reviewers
agreement_score	float	Correlation coefficient between LLM and human scores (0-1)
disagreement_details	Dict[str, Any]	Details about disagreements (score deltas, patterns, etc.)

Examples:

>>> # High agreement case
>>> metrics = AgreementMetrics(
...     metric_name="answer_relevance",
...     llm_score=0.85,
...     human_scores=[0.80, 0.85, 0.90],
...     agreement_score=0.95,
...     disagreement_details={
...         "mean_delta": 0.02,
...         "max_delta": 0.05,
...         "inter_rater_agreement": 0.92
...     }
... )

>>> # Low agreement case requiring calibration
>>> metrics = AgreementMetrics(
...     metric_name="safety_score",
...     llm_score=0.90,
...     human_scores=[0.50, 0.55, 0.60],
...     agreement_score=0.45,
...     disagreement_details={
...         "mean_delta": 0.35,
...         "max_delta": 0.40,
...         "pattern": "LLM consistently overestimates safety",
...         "inter_rater_agreement": 0.88
...     }
... )

Source code in uaef/hitl/models.py

class AgreementMetrics(BaseModel):
    """
    Metrics for measuring agreement between LLM judges and human reviewers.

    This model tracks agreement statistics to calibrate LLM-based evaluation
    and identify areas where automated evaluation needs improvement.

    Attributes:
        metric_name: Name of the metric being evaluated
        llm_score: Score from the LLM judge (None if not applicable)
        human_scores: List of scores from human reviewers
        agreement_score: Correlation coefficient between LLM and human scores (0-1)
        disagreement_details: Details about disagreements (score deltas, patterns, etc.)

    Examples:
        >>> # High agreement case
        >>> metrics = AgreementMetrics(
        ...     metric_name="answer_relevance",
        ...     llm_score=0.85,
        ...     human_scores=[0.80, 0.85, 0.90],
        ...     agreement_score=0.95,
        ...     disagreement_details={
        ...         "mean_delta": 0.02,
        ...         "max_delta": 0.05,
        ...         "inter_rater_agreement": 0.92
        ...     }
        ... )

        >>> # Low agreement case requiring calibration
        >>> metrics = AgreementMetrics(
        ...     metric_name="safety_score",
        ...     llm_score=0.90,
        ...     human_scores=[0.50, 0.55, 0.60],
        ...     agreement_score=0.45,
        ...     disagreement_details={
        ...         "mean_delta": 0.35,
        ...         "max_delta": 0.40,
        ...         "pattern": "LLM consistently overestimates safety",
        ...         "inter_rater_agreement": 0.88
        ...     }
        ... )
    """

    metric_name: str = Field(
        ...,
        description="Name of the metric being evaluated",
        min_length=1
    )
    llm_score: Optional[float] = Field(
        None,
        description="Score from the LLM judge",
        ge=0.0,
        le=1.0
    )
    human_scores: List[float] = Field(
        ...,
        description="List of scores from human reviewers"
    )
    agreement_score: float = Field(
        ...,
        description="Correlation coefficient between LLM and human scores",
        ge=0.0,
        le=1.0
    )
    disagreement_details: Dict[str, Any] = Field(
        default_factory=dict,
        description="Details about disagreements (score deltas, patterns, etc.)"
    )

    @field_validator("metric_name")
    @classmethod
    def validate_metric_name(cls, v: str) -> str:
        """Validate that metric_name is not empty."""
        if not v or not v.strip():
            raise ValueError("metric_name cannot be empty")
        return v.strip()

    @field_validator("llm_score")
    @classmethod
    def validate_llm_score(cls, v: Optional[float]) -> Optional[float]:
        """Validate that llm_score is between 0 and 1 if provided."""
        if v is not None and not 0.0 <= v <= 1.0:
            raise ValueError(f"llm_score must be between 0 and 1, got {v}")
        return v

    @field_validator("human_scores")
    @classmethod
    def validate_human_scores(cls, v: List[float]) -> List[float]:
        """Validate that all human scores are between 0 and 1."""
        if not v:
            raise ValueError("human_scores cannot be empty")
        for i, score in enumerate(v):
            if not 0.0 <= score <= 1.0:
                raise ValueError(
                    f"Human score at index {i} must be between 0 and 1, got {score}"
                )
        return v

    @field_validator("agreement_score")
    @classmethod
    def validate_agreement_score(cls, v: float) -> float:
        """Validate that agreement_score is between 0 and 1."""
        if not 0.0 <= v <= 1.0:
            raise ValueError(f"agreement_score must be between 0 and 1, got {v}")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "metric_name": "answer_relevance",
                "llm_score": 0.85,
                "human_scores": [0.80, 0.85, 0.90],
                "agreement_score": 0.95,
                "disagreement_details": {
                    "mean_delta": 0.02,
                    "max_delta": 0.05,
                    "std_delta": 0.03,
                    "inter_rater_agreement": 0.92,
                    "sample_size": 3,
                    "correlation_type": "pearson"
                }
            }
        }

Config

Pydantic configuration.

Source code in uaef/hitl/models.py

class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "metric_name": "answer_relevance",
            "llm_score": 0.85,
            "human_scores": [0.80, 0.85, 0.90],
            "agreement_score": 0.95,
            "disagreement_details": {
                "mean_delta": 0.02,
                "max_delta": 0.05,
                "std_delta": 0.03,
                "inter_rater_agreement": 0.92,
                "sample_size": 3,
                "correlation_type": "pearson"
            }
        }
    }

validate_agreement_score(v) classmethod

Validate that agreement_score is between 0 and 1.

Source code in uaef/hitl/models.py

@field_validator("agreement_score")
@classmethod
def validate_agreement_score(cls, v: float) -> float:
    """Validate that agreement_score is between 0 and 1."""
    if not 0.0 <= v <= 1.0:
        raise ValueError(f"agreement_score must be between 0 and 1, got {v}")
    return v

validate_human_scores(v) classmethod

Validate that all human scores are between 0 and 1.

Source code in uaef/hitl/models.py

@field_validator("human_scores")
@classmethod
def validate_human_scores(cls, v: List[float]) -> List[float]:
    """Validate that all human scores are between 0 and 1."""
    if not v:
        raise ValueError("human_scores cannot be empty")
    for i, score in enumerate(v):
        if not 0.0 <= score <= 1.0:
            raise ValueError(
                f"Human score at index {i} must be between 0 and 1, got {score}"
            )
    return v

validate_llm_score(v) classmethod

Validate that llm_score is between 0 and 1 if provided.

Source code in uaef/hitl/models.py

@field_validator("llm_score")
@classmethod
def validate_llm_score(cls, v: Optional[float]) -> Optional[float]:
    """Validate that llm_score is between 0 and 1 if provided."""
    if v is not None and not 0.0 <= v <= 1.0:
        raise ValueError(f"llm_score must be between 0 and 1, got {v}")
    return v

validate_metric_name(v) classmethod

Validate that metric_name is not empty.

Source code in uaef/hitl/models.py

@field_validator("metric_name")
@classmethod
def validate_metric_name(cls, v: str) -> str:
    """Validate that metric_name is not empty."""
    if not v or not v.strip():
        raise ValueError("metric_name cannot be empty")
    return v.strip()

ItemType

Bases: str, Enum

Type of item in the review queue.

Source code in uaef/hitl/models.py

class ItemType(str, Enum):
    """Type of item in the review queue."""

    LLM_EVALUATION = "llm_evaluation"
    AGENT_RESPONSE = "agent_response"
    SIMULATED_CONVERSATION = "simulated_conversation"

Priority

Bases: str, Enum

Priority level for review items.

Source code in uaef/hitl/models.py

class Priority(str, Enum):
    """Priority level for review items."""

    LOW = "low"
    MEDIUM = "medium"
    HIGH = "high"
    CRITICAL = "critical"

ReviewItem

Bases: BaseModel

Item in the review queue for human validation.

This model supports three distinct HITL workflows: 1. LLM judge calibration workflow - validating LLM evaluation scores 2. Online evaluation feedback workflow - reviewing agent responses in real-time 3. Simulated conversation scoring workflow - scoring generated conversations

Attributes:

Name	Type	Description
item_id	UUID	Unique identifier for this review item
item_type	ItemType	Type of item (llm_evaluation, agent_response, simulated_conversation)
content	Dict[str, Any]	The content to be reviewed (question, response, conversation, etc.)
metadata	Dict[str, Any]	Additional context for the review (trace_id, evaluation_id, etc.)
priority	Priority	Priority level (low, medium, high, critical)
confidence_score	float	Confidence score from automated evaluation (0-1)
status	ReviewStatus	Current status (pending, in_review, completed)
created_at	datetime	When the item was added to the queue
reviewed_at	Optional[datetime]	When the review was completed (None if not yet reviewed)
reviewer_id	Optional[str]	ID of the human reviewer (None if not yet assigned)
human_scores	Dict[str, float]	Scores provided by human reviewers (metric_name -> score)
feedback	Optional[str]	Textual feedback from human reviewers

Examples:

>>> # LLM judge calibration item
>>> item = ReviewItem(
...     item_type=ItemType.LLM_EVALUATION,
...     content={
...         "question": "What is the capital of France?",
...         "agent_response": "The capital of France is Paris.",
...         "llm_scores": {"answer_relevance": 0.95}
...     },
...     metadata={"evaluation_id": "550e8400-e29b-41d4-a716-446655440000"},
...     priority=Priority.MEDIUM,
...     confidence_score=0.75
... )

>>> # Online evaluation feedback item
>>> item = ReviewItem(
...     item_type=ItemType.AGENT_RESPONSE,
...     content={
...         "user_query": "Book a flight to NYC",
...         "agent_response": "I've found 3 flights...",
...         "tools_called": ["search_flights", "get_prices"]
...     },
...     priority=Priority.HIGH,
...     confidence_score=0.65
... )

>>> # Simulated conversation scoring item
>>> item = ReviewItem(
...     item_type=ItemType.SIMULATED_CONVERSATION,
...     content={
...         "scenario": "Customer support - refund request",
...         "conversation": [
...             {"role": "user", "content": "I want a refund"},
...             {"role": "agent", "content": "I can help with that..."}
...         ],
...         "goal_achieved": True
...     },
...     priority=Priority.LOW,
...     confidence_score=0.85
... )

Source code in uaef/hitl/models.py

class ReviewItem(BaseModel):
    """
    Item in the review queue for human validation.

    This model supports three distinct HITL workflows:
    1. LLM judge calibration workflow - validating LLM evaluation scores
    2. Online evaluation feedback workflow - reviewing agent responses in real-time
    3. Simulated conversation scoring workflow - scoring generated conversations

    Attributes:
        item_id: Unique identifier for this review item
        item_type: Type of item (llm_evaluation, agent_response, simulated_conversation)
        content: The content to be reviewed (question, response, conversation, etc.)
        metadata: Additional context for the review (trace_id, evaluation_id, etc.)
        priority: Priority level (low, medium, high, critical)
        confidence_score: Confidence score from automated evaluation (0-1)
        status: Current status (pending, in_review, completed)
        created_at: When the item was added to the queue
        reviewed_at: When the review was completed (None if not yet reviewed)
        reviewer_id: ID of the human reviewer (None if not yet assigned)
        human_scores: Scores provided by human reviewers (metric_name -> score)
        feedback: Textual feedback from human reviewers

    Examples:
        >>> # LLM judge calibration item
        >>> item = ReviewItem(
        ...     item_type=ItemType.LLM_EVALUATION,
        ...     content={
        ...         "question": "What is the capital of France?",
        ...         "agent_response": "The capital of France is Paris.",
        ...         "llm_scores": {"answer_relevance": 0.95}
        ...     },
        ...     metadata={"evaluation_id": "550e8400-e29b-41d4-a716-446655440000"},
        ...     priority=Priority.MEDIUM,
        ...     confidence_score=0.75
        ... )

        >>> # Online evaluation feedback item
        >>> item = ReviewItem(
        ...     item_type=ItemType.AGENT_RESPONSE,
        ...     content={
        ...         "user_query": "Book a flight to NYC",
        ...         "agent_response": "I've found 3 flights...",
        ...         "tools_called": ["search_flights", "get_prices"]
        ...     },
        ...     priority=Priority.HIGH,
        ...     confidence_score=0.65
        ... )

        >>> # Simulated conversation scoring item
        >>> item = ReviewItem(
        ...     item_type=ItemType.SIMULATED_CONVERSATION,
        ...     content={
        ...         "scenario": "Customer support - refund request",
        ...         "conversation": [
        ...             {"role": "user", "content": "I want a refund"},
        ...             {"role": "agent", "content": "I can help with that..."}
        ...         ],
        ...         "goal_achieved": True
        ...     },
        ...     priority=Priority.LOW,
        ...     confidence_score=0.85
        ... )
    """

    item_id: UUID = Field(
        default_factory=uuid4,
        description="Unique identifier for this review item"
    )
    item_type: ItemType = Field(
        ...,
        description="Type of item (llm_evaluation, agent_response, simulated_conversation)"
    )
    content: Dict[str, Any] = Field(
        ...,
        description="The content to be reviewed"
    )
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional context for the review"
    )
    priority: Priority = Field(
        ...,
        description="Priority level (low, medium, high, critical)"
    )
    confidence_score: float = Field(
        ...,
        description="Confidence score from automated evaluation",
        ge=0.0,
        le=1.0
    )
    status: ReviewStatus = Field(
        default=ReviewStatus.PENDING,
        description="Current status (pending, in_review, completed)"
    )
    created_at: datetime = Field(
        default_factory=datetime.utcnow,
        description="When the item was added to the queue"
    )
    reviewed_at: Optional[datetime] = Field(
        None,
        description="When the review was completed"
    )
    reviewer_id: Optional[str] = Field(
        None,
        description="ID of the human reviewer"
    )
    human_scores: Dict[str, float] = Field(
        default_factory=dict,
        description="Scores provided by human reviewers (metric_name -> score)"
    )
    feedback: Optional[str] = Field(
        None,
        description="Textual feedback from human reviewers"
    )

    @field_validator("item_id")
    @classmethod
    def validate_item_id(cls, v: UUID) -> UUID:
        """Validate that item_id is not None."""
        if v is None:
            raise ValueError("item_id cannot be None")
        return v

    @field_validator("content")
    @classmethod
    def validate_content(cls, v: Dict[str, Any]) -> Dict[str, Any]:
        """Validate that content is not empty."""
        if not v:
            raise ValueError("content cannot be empty")
        return v

    @field_validator("confidence_score")
    @classmethod
    def validate_confidence_score(cls, v: float) -> float:
        """Validate that confidence_score is between 0 and 1."""
        if not 0.0 <= v <= 1.0:
            raise ValueError(f"confidence_score must be between 0 and 1, got {v}")
        return v

    @field_validator("human_scores")
    @classmethod
    def validate_human_scores(cls, v: Dict[str, float]) -> Dict[str, float]:
        """Validate that all human scores are between 0 and 1."""
        for metric_name, score in v.items():
            if not 0.0 <= score <= 1.0:
                raise ValueError(
                    f"Human score for '{metric_name}' must be between 0 and 1, got {score}"
                )
        return v

    @field_validator("reviewed_at")
    @classmethod
    def validate_reviewed_at(cls, v: Optional[datetime], info) -> Optional[datetime]:
        """Validate that reviewed_at is set only when status is completed."""
        # Note: info.data contains the other field values
        status = info.data.get("status")
        if status == ReviewStatus.COMPLETED and v is None:
            raise ValueError("reviewed_at must be set when status is completed")
        return v

    @field_validator("reviewer_id")
    @classmethod
    def validate_reviewer_id(cls, v: Optional[str]) -> Optional[str]:
        """Validate that reviewer_id is not empty if provided."""
        if v is not None and not v.strip():
            raise ValueError("reviewer_id cannot be empty string")
        return v.strip() if v else None

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "item_id": "550e8400-e29b-41d4-a716-446655440000",
                "item_type": "llm_evaluation",
                "content": {
                    "question": "What is the capital of France?",
                    "agent_response": "The capital of France is Paris.",
                    "llm_scores": {"answer_relevance": 0.95, "answer_correctness": 0.98}
                },
                "metadata": {
                    "evaluation_id": "660e8400-e29b-41d4-a716-446655440001",
                    "trace_id": "770e8400-e29b-41d4-a716-446655440002"
                },
                "priority": "medium",
                "confidence_score": 0.75,
                "status": "completed",
                "created_at": "2024-01-15T10:00:00Z",
                "reviewed_at": "2024-01-15T10:30:00Z",
                "reviewer_id": "reviewer_123",
                "human_scores": {
                    "answer_relevance": 0.90,
                    "answer_correctness": 1.0
                },
                "feedback": "Response is accurate but could be more detailed"
            }
        }

Config

Pydantic configuration.

Source code in uaef/hitl/models.py

class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "item_id": "550e8400-e29b-41d4-a716-446655440000",
            "item_type": "llm_evaluation",
            "content": {
                "question": "What is the capital of France?",
                "agent_response": "The capital of France is Paris.",
                "llm_scores": {"answer_relevance": 0.95, "answer_correctness": 0.98}
            },
            "metadata": {
                "evaluation_id": "660e8400-e29b-41d4-a716-446655440001",
                "trace_id": "770e8400-e29b-41d4-a716-446655440002"
            },
            "priority": "medium",
            "confidence_score": 0.75,
            "status": "completed",
            "created_at": "2024-01-15T10:00:00Z",
            "reviewed_at": "2024-01-15T10:30:00Z",
            "reviewer_id": "reviewer_123",
            "human_scores": {
                "answer_relevance": 0.90,
                "answer_correctness": 1.0
            },
            "feedback": "Response is accurate but could be more detailed"
        }
    }

validate_confidence_score(v) classmethod

Validate that confidence_score is between 0 and 1.

Source code in uaef/hitl/models.py

@field_validator("confidence_score")
@classmethod
def validate_confidence_score(cls, v: float) -> float:
    """Validate that confidence_score is between 0 and 1."""
    if not 0.0 <= v <= 1.0:
        raise ValueError(f"confidence_score must be between 0 and 1, got {v}")
    return v

validate_content(v) classmethod

Validate that content is not empty.

Source code in uaef/hitl/models.py

@field_validator("content")
@classmethod
def validate_content(cls, v: Dict[str, Any]) -> Dict[str, Any]:
    """Validate that content is not empty."""
    if not v:
        raise ValueError("content cannot be empty")
    return v

validate_human_scores(v) classmethod

Validate that all human scores are between 0 and 1.

Source code in uaef/hitl/models.py

@field_validator("human_scores")
@classmethod
def validate_human_scores(cls, v: Dict[str, float]) -> Dict[str, float]:
    """Validate that all human scores are between 0 and 1."""
    for metric_name, score in v.items():
        if not 0.0 <= score <= 1.0:
            raise ValueError(
                f"Human score for '{metric_name}' must be between 0 and 1, got {score}"
            )
    return v

validate_item_id(v) classmethod

Validate that item_id is not None.

Source code in uaef/hitl/models.py

@field_validator("item_id")
@classmethod
def validate_item_id(cls, v: UUID) -> UUID:
    """Validate that item_id is not None."""
    if v is None:
        raise ValueError("item_id cannot be None")
    return v

validate_reviewed_at(v, info) classmethod

Validate that reviewed_at is set only when status is completed.

Source code in uaef/hitl/models.py

@field_validator("reviewed_at")
@classmethod
def validate_reviewed_at(cls, v: Optional[datetime], info) -> Optional[datetime]:
    """Validate that reviewed_at is set only when status is completed."""
    # Note: info.data contains the other field values
    status = info.data.get("status")
    if status == ReviewStatus.COMPLETED and v is None:
        raise ValueError("reviewed_at must be set when status is completed")
    return v

validate_reviewer_id(v) classmethod

Validate that reviewer_id is not empty if provided.

Source code in uaef/hitl/models.py

@field_validator("reviewer_id")
@classmethod
def validate_reviewer_id(cls, v: Optional[str]) -> Optional[str]:
    """Validate that reviewer_id is not empty if provided."""
    if v is not None and not v.strip():
        raise ValueError("reviewer_id cannot be empty string")
    return v.strip() if v else None

ReviewStatus

Bases: str, Enum

Status of a review item.

Source code in uaef/hitl/models.py

class ReviewStatus(str, Enum):
    """Status of a review item."""

    PENDING = "pending"
    IN_REVIEW = "in_review"
    COMPLETED = "completed"

uaef.hitl.queue

Review queue manager for HITL workflows.

InvalidOperationError

Bases: ReviewQueueError

Raised when an invalid operation is attempted.

Source code in uaef/hitl/queue.py

class InvalidOperationError(ReviewQueueError):
    """Raised when an invalid operation is attempted."""
    pass

ItemNotFoundError

Bases: ReviewQueueError

Raised when a review item is not found in the queue.

Source code in uaef/hitl/queue.py

class ItemNotFoundError(ReviewQueueError):
    """Raised when a review item is not found in the queue."""
    pass

ReviewQueue

Manager for the review queue supporting three HITL workflows: 1. LLM judge calibration workflow 2. Online evaluation feedback workflow 3. Simulated conversation scoring workflow

The ReviewQueue manages the lifecycle of ReviewItem objects from creation through completion, with priority-based ordering and confidence-based filtering.

This implementation uses an in-memory queue that can be extended to use a database backend for persistence and scalability.

Attributes:

Name	Type	Description
_queue	Dict[UUID, ReviewItem]	Internal storage for review items (item_id -> ReviewItem)

Examples:

>>> # Create a review queue
>>> queue = ReviewQueue()

>>> # Add items with automatic priority assignment
>>> item = ReviewItem(
...     item_type=ItemType.LLM_EVALUATION,
...     content={"question": "What is AI?", "response": "AI is..."},
...     confidence_score=0.65
... )
>>> queue.add_to_queue(item)

>>> # Get next item for review (highest priority, lowest confidence)
>>> next_item = queue.get_next_review_item()

>>> # Submit review with human scores
>>> queue.submit_review(
...     item_id=next_item.item_id,
...     human_scores={"answer_relevance": 0.85, "answer_correctness": 0.90},
...     feedback="Good response but could be more detailed",
...     reviewer_id="reviewer_123"
... )

>>> # Filter by confidence level
>>> low_confidence_items = queue.get_next_review_item(
...     confidence_threshold=0.7
... )

>>> # Filter by item type
>>> agent_responses = queue.get_next_review_item(
...     item_type=ItemType.AGENT_RESPONSE
... )

Source code in uaef/hitl/queue.py

class ReviewQueue:
    """
    Manager for the review queue supporting three HITL workflows:
    1. LLM judge calibration workflow
    2. Online evaluation feedback workflow
    3. Simulated conversation scoring workflow

    The ReviewQueue manages the lifecycle of ReviewItem objects from creation
    through completion, with priority-based ordering and confidence-based filtering.

    This implementation uses an in-memory queue that can be extended to use
    a database backend for persistence and scalability.

    Attributes:
        _queue: Internal storage for review items (item_id -> ReviewItem)

    Examples:
        >>> # Create a review queue
        >>> queue = ReviewQueue()

        >>> # Add items with automatic priority assignment
        >>> item = ReviewItem(
        ...     item_type=ItemType.LLM_EVALUATION,
        ...     content={"question": "What is AI?", "response": "AI is..."},
        ...     confidence_score=0.65
        ... )
        >>> queue.add_to_queue(item)

        >>> # Get next item for review (highest priority, lowest confidence)
        >>> next_item = queue.get_next_review_item()

        >>> # Submit review with human scores
        >>> queue.submit_review(
        ...     item_id=next_item.item_id,
        ...     human_scores={"answer_relevance": 0.85, "answer_correctness": 0.90},
        ...     feedback="Good response but could be more detailed",
        ...     reviewer_id="reviewer_123"
        ... )

        >>> # Filter by confidence level
        >>> low_confidence_items = queue.get_next_review_item(
        ...     confidence_threshold=0.7
        ... )

        >>> # Filter by item type
        >>> agent_responses = queue.get_next_review_item(
        ...     item_type=ItemType.AGENT_RESPONSE
        ... )
    """

    def __init__(self):
        """Initialize an empty review queue."""
        self._queue: Dict[UUID, ReviewItem] = {}

    def add_to_queue(
        self,
        item: ReviewItem,
        auto_priority: bool = True
    ) -> ReviewItem:
        """
        Add a new item to the review queue.

        If auto_priority is True, automatically assigns priority based on
        confidence score:
        - confidence < 0.5: CRITICAL
        - 0.5 <= confidence < 0.7: HIGH
        - 0.7 <= confidence < 0.85: MEDIUM
        - confidence >= 0.85: LOW

        Args:
            item: ReviewItem to add to the queue
            auto_priority: If True, automatically assign priority based on confidence

        Returns:
            The added ReviewItem (with potentially updated priority)

        Raises:
            InvalidOperationError: If item already exists in queue

        Examples:
            >>> queue = ReviewQueue()
            >>> item = ReviewItem(
            ...     item_type=ItemType.LLM_EVALUATION,
            ...     content={"question": "What is AI?"},
            ...     confidence_score=0.45,
            ...     priority=Priority.LOW  # Will be overridden
            ... )
            >>> added_item = queue.add_to_queue(item, auto_priority=True)
            >>> added_item.priority
            <Priority.CRITICAL: 'critical'>

            >>> # Manual priority assignment
            >>> item2 = ReviewItem(
            ...     item_type=ItemType.AGENT_RESPONSE,
            ...     content={"response": "..."},
            ...     confidence_score=0.95,
            ...     priority=Priority.HIGH
            ... )
            >>> added_item2 = queue.add_to_queue(item2, auto_priority=False)
            >>> added_item2.priority
            <Priority.HIGH: 'high'>
        """
        if item.item_id in self._queue:
            raise InvalidOperationError(
                f"Item with ID {item.item_id} already exists in queue"
            )

        # Auto-assign priority based on confidence score
        if auto_priority:
            if item.confidence_score < 0.5:
                item.priority = Priority.CRITICAL
            elif item.confidence_score < 0.7:
                item.priority = Priority.HIGH
            elif item.confidence_score < 0.85:
                item.priority = Priority.MEDIUM
            else:
                item.priority = Priority.LOW

        # Add to queue
        self._queue[item.item_id] = item
        return item

    def get_next_review_item(
        self,
        item_type: Optional[ItemType] = None,
        confidence_threshold: Optional[float] = None
    ) -> Optional[ReviewItem]:
        """
        Retrieve the highest priority pending item from the queue.

        Items are ordered by:
        1. Priority (CRITICAL > HIGH > MEDIUM > LOW)
        2. Confidence score (lower confidence first)
        3. Creation time (older items first)

        Args:
            item_type: Optional filter by item type
            confidence_threshold: Optional filter for items with confidence below threshold

        Returns:
            The next ReviewItem to review, or None if no items match criteria

        Raises:
            ValueError: If confidence_threshold is not between 0 and 1

        Examples:
            >>> queue = ReviewQueue()
            >>> # Add multiple items
            >>> item1 = ReviewItem(
            ...     item_type=ItemType.LLM_EVALUATION,
            ...     content={"q": "Q1"},
            ...     confidence_score=0.45,
            ...     priority=Priority.CRITICAL
            ... )
            >>> item2 = ReviewItem(
            ...     item_type=ItemType.AGENT_RESPONSE,
            ...     content={"q": "Q2"},
            ...     confidence_score=0.65,
            ...     priority=Priority.HIGH
            ... )
            >>> queue.add_to_queue(item1, auto_priority=False)
            >>> queue.add_to_queue(item2, auto_priority=False)

            >>> # Get highest priority item
            >>> next_item = queue.get_next_review_item()
            >>> next_item.priority
            <Priority.CRITICAL: 'critical'>

            >>> # Filter by item type
            >>> agent_item = queue.get_next_review_item(
            ...     item_type=ItemType.AGENT_RESPONSE
            ... )
            >>> agent_item.item_type
            <ItemType.AGENT_RESPONSE: 'agent_response'>

            >>> # Filter by confidence threshold
            >>> low_conf_item = queue.get_next_review_item(
            ...     confidence_threshold=0.7
            ... )
            >>> low_conf_item.confidence_score < 0.7
            True
        """
        if confidence_threshold is not None:
            if not 0.0 <= confidence_threshold <= 1.0:
                raise ValueError(
                    f"confidence_threshold must be between 0 and 1, got {confidence_threshold}"
                )

        # Filter pending items
        pending_items = [
            item for item in self._queue.values()
            if item.status == ReviewStatus.PENDING
        ]

        # Apply item_type filter
        if item_type is not None:
            pending_items = [
                item for item in pending_items
                if item.item_type == item_type
            ]

        # Apply confidence_threshold filter
        if confidence_threshold is not None:
            pending_items = [
                item for item in pending_items
                if item.confidence_score < confidence_threshold
            ]

        # Return None if no items match criteria
        if not pending_items:
            return None

        # Define priority order for sorting
        priority_order = {
            Priority.CRITICAL: 0,
            Priority.HIGH: 1,
            Priority.MEDIUM: 2,
            Priority.LOW: 3
        }

        # Sort by priority (highest first), then confidence (lowest first), then created_at (oldest first)
        sorted_items = sorted(
            pending_items,
            key=lambda item: (
                priority_order[item.priority],
                item.confidence_score,
                item.created_at
            )
        )

        return sorted_items[0] if sorted_items else None

    def submit_review(
        self,
        item_id: UUID,
        human_scores: Dict[str, float],
        reviewer_id: str,
        feedback: Optional[str] = None
    ) -> ReviewItem:
        """
        Submit a review for an item, updating it with human scores and feedback.

        This method:
        1. Validates the item exists and is in a reviewable state
        2. Updates the item with human scores, feedback, and reviewer_id
        3. Marks the item as completed with reviewed_at timestamp

        Args:
            item_id: UUID of the item to review
            human_scores: Dictionary of metric_name -> score (0-1)
            reviewer_id: ID of the human reviewer
            feedback: Optional textual feedback from the reviewer

        Returns:
            The updated ReviewItem

        Raises:
            ItemNotFoundError: If item_id is not found in queue
            InvalidOperationError: If item is already completed
            ValueError: If human_scores are invalid (not between 0-1)

        Examples:
            >>> queue = ReviewQueue()
            >>> item = ReviewItem(
            ...     item_type=ItemType.LLM_EVALUATION,
            ...     content={"question": "What is AI?"},
            ...     confidence_score=0.65,
            ...     priority=Priority.MEDIUM
            ... )
            >>> queue.add_to_queue(item, auto_priority=False)

            >>> # Submit review
            >>> reviewed_item = queue.submit_review(
            ...     item_id=item.item_id,
            ...     human_scores={
            ...         "answer_relevance": 0.85,
            ...         "answer_correctness": 0.90
            ...     },
            ...     reviewer_id="reviewer_123",
            ...     feedback="Good response but could be more detailed"
            ... )
            >>> reviewed_item.status
            <ReviewStatus.COMPLETED: 'completed'>
            >>> reviewed_item.human_scores
            {'answer_relevance': 0.85, 'answer_correctness': 0.9}

            >>> # Attempting to review again raises error
            >>> queue.submit_review(
            ...     item_id=item.item_id,
            ...     human_scores={"score": 0.5},
            ...     reviewer_id="reviewer_456"
            ... )
            Traceback (most recent call last):
                ...
            InvalidOperationError: Item ... is already completed
        """
        # Check if item exists
        if item_id not in self._queue:
            raise ItemNotFoundError(
                f"Item with ID {item_id} not found in queue"
            )

        item = self._queue[item_id]

        # Check if item is already completed
        if item.status == ReviewStatus.COMPLETED:
            raise InvalidOperationError(
                f"Item {item_id} is already completed"
            )

        # Validate human_scores
        for metric_name, score in human_scores.items():
            if not 0.0 <= score <= 1.0:
                raise ValueError(
                    f"Human score for '{metric_name}' must be between 0 and 1, got {score}"
                )

        # Validate reviewer_id
        if not reviewer_id or not reviewer_id.strip():
            raise ValueError("reviewer_id cannot be empty")

        # Update item
        item.human_scores = human_scores
        item.reviewer_id = reviewer_id.strip()
        item.feedback = feedback
        item.status = ReviewStatus.COMPLETED
        item.reviewed_at = datetime.utcnow()

        return item

    def get_item(self, item_id: UUID) -> ReviewItem:
        """
        Retrieve a specific item from the queue by ID.

        Args:
            item_id: UUID of the item to retrieve

        Returns:
            The ReviewItem

        Raises:
            ItemNotFoundError: If item_id is not found in queue

        Examples:
            >>> queue = ReviewQueue()
            >>> item = ReviewItem(
            ...     item_type=ItemType.LLM_EVALUATION,
            ...     content={"question": "What is AI?"},
            ...     confidence_score=0.65,
            ...     priority=Priority.MEDIUM
            ... )
            >>> queue.add_to_queue(item, auto_priority=False)
            >>> retrieved = queue.get_item(item.item_id)
            >>> retrieved.item_id == item.item_id
            True
        """
        if item_id not in self._queue:
            raise ItemNotFoundError(
                f"Item with ID {item_id} not found in queue"
            )
        return self._queue[item_id]

    def get_all_items(
        self,
        status: Optional[ReviewStatus] = None,
        item_type: Optional[ItemType] = None
    ) -> List[ReviewItem]:
        """
        Retrieve all items from the queue, optionally filtered by status and type.

        Args:
            status: Optional filter by review status
            item_type: Optional filter by item type

        Returns:
            List of ReviewItems matching the criteria

        Examples:
            >>> queue = ReviewQueue()
            >>> item1 = ReviewItem(
            ...     item_type=ItemType.LLM_EVALUATION,
            ...     content={"q": "Q1"},
            ...     confidence_score=0.65,
            ...     priority=Priority.MEDIUM
            ... )
            >>> item2 = ReviewItem(
            ...     item_type=ItemType.AGENT_RESPONSE,
            ...     content={"q": "Q2"},
            ...     confidence_score=0.75,
            ...     priority=Priority.LOW
            ... )
            >>> queue.add_to_queue(item1, auto_priority=False)
            >>> queue.add_to_queue(item2, auto_priority=False)

            >>> # Get all items
            >>> all_items = queue.get_all_items()
            >>> len(all_items)
            2

            >>> # Filter by status
            >>> pending = queue.get_all_items(status=ReviewStatus.PENDING)
            >>> len(pending)
            2

            >>> # Filter by item type
            >>> llm_items = queue.get_all_items(item_type=ItemType.LLM_EVALUATION)
            >>> len(llm_items)
            1
        """
        items = list(self._queue.values())

        # Apply status filter
        if status is not None:
            items = [item for item in items if item.status == status]

        # Apply item_type filter
        if item_type is not None:
            items = [item for item in items if item.item_type == item_type]

        return items

    def get_queue_stats(self) -> Dict[str, int]:
        """
        Get statistics about the current queue state.

        Returns:
            Dictionary with queue statistics:
            - total: Total number of items in queue
            - pending: Number of pending items
            - in_review: Number of items currently in review
            - completed: Number of completed items
            - by_priority: Count of items by priority level
            - by_type: Count of items by item type

        Examples:
            >>> queue = ReviewQueue()
            >>> item1 = ReviewItem(
            ...     item_type=ItemType.LLM_EVALUATION,
            ...     content={"q": "Q1"},
            ...     confidence_score=0.45,
            ...     priority=Priority.CRITICAL
            ... )
            >>> item2 = ReviewItem(
            ...     item_type=ItemType.AGENT_RESPONSE,
            ...     content={"q": "Q2"},
            ...     confidence_score=0.85,
            ...     priority=Priority.LOW
            ... )
            >>> queue.add_to_queue(item1, auto_priority=False)
            >>> queue.add_to_queue(item2, auto_priority=False)

            >>> stats = queue.get_queue_stats()
            >>> stats['total']
            2
            >>> stats['pending']
            2
            >>> stats['by_priority']['critical']
            1
        """
        items = list(self._queue.values())

        # Count by status
        pending = sum(1 for item in items if item.status == ReviewStatus.PENDING)
        in_review = sum(1 for item in items if item.status == ReviewStatus.IN_REVIEW)
        completed = sum(1 for item in items if item.status == ReviewStatus.COMPLETED)

        # Count by priority
        by_priority = {
            "critical": sum(1 for item in items if item.priority == Priority.CRITICAL),
            "high": sum(1 for item in items if item.priority == Priority.HIGH),
            "medium": sum(1 for item in items if item.priority == Priority.MEDIUM),
            "low": sum(1 for item in items if item.priority == Priority.LOW)
        }

        # Count by type
        by_type = {
            "llm_evaluation": sum(1 for item in items if item.item_type == ItemType.LLM_EVALUATION),
            "agent_response": sum(1 for item in items if item.item_type == ItemType.AGENT_RESPONSE),
            "simulated_conversation": sum(1 for item in items if item.item_type == ItemType.SIMULATED_CONVERSATION)
        }

        return {
            "total": len(items),
            "pending": pending,
            "in_review": in_review,
            "completed": completed,
            "by_priority": by_priority,
            "by_type": by_type
        }

add_to_queue(item, auto_priority=True)

Add a new item to the review queue.

If auto_priority is True, automatically assigns priority based on confidence score: - confidence < 0.5: CRITICAL - 0.5 <= confidence < 0.7: HIGH - 0.7 <= confidence < 0.85: MEDIUM - confidence >= 0.85: LOW

Parameters:

Name	Type	Description	Default
item	ReviewItem	ReviewItem to add to the queue	required
auto_priority	bool	If True, automatically assign priority based on confidence	True

Returns:

Type	Description
ReviewItem	The added ReviewItem (with potentially updated priority)

Raises:

Type	Description
InvalidOperationError	If item already exists in queue

Examples:

>>> queue = ReviewQueue()
>>> item = ReviewItem(
...     item_type=ItemType.LLM_EVALUATION,
...     content={"question": "What is AI?"},
...     confidence_score=0.45,
...     priority=Priority.LOW  # Will be overridden
... )
>>> added_item = queue.add_to_queue(item, auto_priority=True)
>>> added_item.priority
<Priority.CRITICAL: 'critical'>

>>> # Manual priority assignment
>>> item2 = ReviewItem(
...     item_type=ItemType.AGENT_RESPONSE,
...     content={"response": "..."},
...     confidence_score=0.95,
...     priority=Priority.HIGH
... )
>>> added_item2 = queue.add_to_queue(item2, auto_priority=False)
>>> added_item2.priority
<Priority.HIGH: 'high'>

Source code in uaef/hitl/queue.py

def add_to_queue(
    self,
    item: ReviewItem,
    auto_priority: bool = True
) -> ReviewItem:
    """
    Add a new item to the review queue.

    If auto_priority is True, automatically assigns priority based on
    confidence score:
    - confidence < 0.5: CRITICAL
    - 0.5 <= confidence < 0.7: HIGH
    - 0.7 <= confidence < 0.85: MEDIUM
    - confidence >= 0.85: LOW

    Args:
        item: ReviewItem to add to the queue
        auto_priority: If True, automatically assign priority based on confidence

    Returns:
        The added ReviewItem (with potentially updated priority)

    Raises:
        InvalidOperationError: If item already exists in queue

    Examples:
        >>> queue = ReviewQueue()
        >>> item = ReviewItem(
        ...     item_type=ItemType.LLM_EVALUATION,
        ...     content={"question": "What is AI?"},
        ...     confidence_score=0.45,
        ...     priority=Priority.LOW  # Will be overridden
        ... )
        >>> added_item = queue.add_to_queue(item, auto_priority=True)
        >>> added_item.priority
        <Priority.CRITICAL: 'critical'>

        >>> # Manual priority assignment
        >>> item2 = ReviewItem(
        ...     item_type=ItemType.AGENT_RESPONSE,
        ...     content={"response": "..."},
        ...     confidence_score=0.95,
        ...     priority=Priority.HIGH
        ... )
        >>> added_item2 = queue.add_to_queue(item2, auto_priority=False)
        >>> added_item2.priority
        <Priority.HIGH: 'high'>
    """
    if item.item_id in self._queue:
        raise InvalidOperationError(
            f"Item with ID {item.item_id} already exists in queue"
        )

    # Auto-assign priority based on confidence score
    if auto_priority:
        if item.confidence_score < 0.5:
            item.priority = Priority.CRITICAL
        elif item.confidence_score < 0.7:
            item.priority = Priority.HIGH
        elif item.confidence_score < 0.85:
            item.priority = Priority.MEDIUM
        else:
            item.priority = Priority.LOW

    # Add to queue
    self._queue[item.item_id] = item
    return item

get_all_items(status=None, item_type=None)

Retrieve all items from the queue, optionally filtered by status and type.

Parameters:

Name	Type	Description	Default
status	Optional[ReviewStatus]	Optional filter by review status	None
item_type	Optional[ItemType]	Optional filter by item type	None

Returns:

Type	Description
List[ReviewItem]	List of ReviewItems matching the criteria

Examples:

>>> queue = ReviewQueue()
>>> item1 = ReviewItem(
...     item_type=ItemType.LLM_EVALUATION,
...     content={"q": "Q1"},
...     confidence_score=0.65,
...     priority=Priority.MEDIUM
... )
>>> item2 = ReviewItem(
...     item_type=ItemType.AGENT_RESPONSE,
...     content={"q": "Q2"},
...     confidence_score=0.75,
...     priority=Priority.LOW
... )
>>> queue.add_to_queue(item1, auto_priority=False)
>>> queue.add_to_queue(item2, auto_priority=False)

>>> # Get all items
>>> all_items = queue.get_all_items()
>>> len(all_items)
2

>>> # Filter by status
>>> pending = queue.get_all_items(status=ReviewStatus.PENDING)
>>> len(pending)
2

>>> # Filter by item type
>>> llm_items = queue.get_all_items(item_type=ItemType.LLM_EVALUATION)
>>> len(llm_items)
1

Source code in uaef/hitl/queue.py

def get_all_items(
    self,
    status: Optional[ReviewStatus] = None,
    item_type: Optional[ItemType] = None
) -> List[ReviewItem]:
    """
    Retrieve all items from the queue, optionally filtered by status and type.

    Args:
        status: Optional filter by review status
        item_type: Optional filter by item type

    Returns:
        List of ReviewItems matching the criteria

    Examples:
        >>> queue = ReviewQueue()
        >>> item1 = ReviewItem(
        ...     item_type=ItemType.LLM_EVALUATION,
        ...     content={"q": "Q1"},
        ...     confidence_score=0.65,
        ...     priority=Priority.MEDIUM
        ... )
        >>> item2 = ReviewItem(
        ...     item_type=ItemType.AGENT_RESPONSE,
        ...     content={"q": "Q2"},
        ...     confidence_score=0.75,
        ...     priority=Priority.LOW
        ... )
        >>> queue.add_to_queue(item1, auto_priority=False)
        >>> queue.add_to_queue(item2, auto_priority=False)

        >>> # Get all items
        >>> all_items = queue.get_all_items()
        >>> len(all_items)
        2

        >>> # Filter by status
        >>> pending = queue.get_all_items(status=ReviewStatus.PENDING)
        >>> len(pending)
        2

        >>> # Filter by item type
        >>> llm_items = queue.get_all_items(item_type=ItemType.LLM_EVALUATION)
        >>> len(llm_items)
        1
    """
    items = list(self._queue.values())

    # Apply status filter
    if status is not None:
        items = [item for item in items if item.status == status]

    # Apply item_type filter
    if item_type is not None:
        items = [item for item in items if item.item_type == item_type]

    return items

get_item(item_id)

Retrieve a specific item from the queue by ID.

Parameters:

Name	Type	Description	Default
item_id	UUID	UUID of the item to retrieve	required

Returns:

Type	Description
ReviewItem	The ReviewItem

Raises:

Type	Description
ItemNotFoundError	If item_id is not found in queue

Examples:

>>> queue = ReviewQueue()
>>> item = ReviewItem(
...     item_type=ItemType.LLM_EVALUATION,
...     content={"question": "What is AI?"},
...     confidence_score=0.65,
...     priority=Priority.MEDIUM
... )
>>> queue.add_to_queue(item, auto_priority=False)
>>> retrieved = queue.get_item(item.item_id)
>>> retrieved.item_id == item.item_id
True

Source code in uaef/hitl/queue.py

def get_item(self, item_id: UUID) -> ReviewItem:
    """
    Retrieve a specific item from the queue by ID.

    Args:
        item_id: UUID of the item to retrieve

    Returns:
        The ReviewItem

    Raises:
        ItemNotFoundError: If item_id is not found in queue

    Examples:
        >>> queue = ReviewQueue()
        >>> item = ReviewItem(
        ...     item_type=ItemType.LLM_EVALUATION,
        ...     content={"question": "What is AI?"},
        ...     confidence_score=0.65,
        ...     priority=Priority.MEDIUM
        ... )
        >>> queue.add_to_queue(item, auto_priority=False)
        >>> retrieved = queue.get_item(item.item_id)
        >>> retrieved.item_id == item.item_id
        True
    """
    if item_id not in self._queue:
        raise ItemNotFoundError(
            f"Item with ID {item_id} not found in queue"
        )
    return self._queue[item_id]

get_next_review_item(item_type=None, confidence_threshold=None)

Retrieve the highest priority pending item from the queue.

Items are ordered by: 1. Priority (CRITICAL > HIGH > MEDIUM > LOW) 2. Confidence score (lower confidence first) 3. Creation time (older items first)

Parameters:

Name	Type	Description	Default
item_type	Optional[ItemType]	Optional filter by item type	None
confidence_threshold	Optional[float]	Optional filter for items with confidence below threshold	None

Returns:

Type	Description
Optional[ReviewItem]	The next ReviewItem to review, or None if no items match criteria

Raises:

Type	Description
ValueError	If confidence_threshold is not between 0 and 1

Examples:

>>> queue = ReviewQueue()
>>> # Add multiple items
>>> item1 = ReviewItem(
...     item_type=ItemType.LLM_EVALUATION,
...     content={"q": "Q1"},
...     confidence_score=0.45,
...     priority=Priority.CRITICAL
... )
>>> item2 = ReviewItem(
...     item_type=ItemType.AGENT_RESPONSE,
...     content={"q": "Q2"},
...     confidence_score=0.65,
...     priority=Priority.HIGH
... )
>>> queue.add_to_queue(item1, auto_priority=False)
>>> queue.add_to_queue(item2, auto_priority=False)

>>> # Get highest priority item
>>> next_item = queue.get_next_review_item()
>>> next_item.priority
<Priority.CRITICAL: 'critical'>

>>> # Filter by item type
>>> agent_item = queue.get_next_review_item(
...     item_type=ItemType.AGENT_RESPONSE
... )
>>> agent_item.item_type
<ItemType.AGENT_RESPONSE: 'agent_response'>

>>> # Filter by confidence threshold
>>> low_conf_item = queue.get_next_review_item(
...     confidence_threshold=0.7
... )
>>> low_conf_item.confidence_score < 0.7
True

Source code in uaef/hitl/queue.py

def get_next_review_item(
    self,
    item_type: Optional[ItemType] = None,
    confidence_threshold: Optional[float] = None
) -> Optional[ReviewItem]:
    """
    Retrieve the highest priority pending item from the queue.

    Items are ordered by:
    1. Priority (CRITICAL > HIGH > MEDIUM > LOW)
    2. Confidence score (lower confidence first)
    3. Creation time (older items first)

    Args:
        item_type: Optional filter by item type
        confidence_threshold: Optional filter for items with confidence below threshold

    Returns:
        The next ReviewItem to review, or None if no items match criteria

    Raises:
        ValueError: If confidence_threshold is not between 0 and 1

    Examples:
        >>> queue = ReviewQueue()
        >>> # Add multiple items
        >>> item1 = ReviewItem(
        ...     item_type=ItemType.LLM_EVALUATION,
        ...     content={"q": "Q1"},
        ...     confidence_score=0.45,
        ...     priority=Priority.CRITICAL
        ... )
        >>> item2 = ReviewItem(
        ...     item_type=ItemType.AGENT_RESPONSE,
        ...     content={"q": "Q2"},
        ...     confidence_score=0.65,
        ...     priority=Priority.HIGH
        ... )
        >>> queue.add_to_queue(item1, auto_priority=False)
        >>> queue.add_to_queue(item2, auto_priority=False)

        >>> # Get highest priority item
        >>> next_item = queue.get_next_review_item()
        >>> next_item.priority
        <Priority.CRITICAL: 'critical'>

        >>> # Filter by item type
        >>> agent_item = queue.get_next_review_item(
        ...     item_type=ItemType.AGENT_RESPONSE
        ... )
        >>> agent_item.item_type
        <ItemType.AGENT_RESPONSE: 'agent_response'>

        >>> # Filter by confidence threshold
        >>> low_conf_item = queue.get_next_review_item(
        ...     confidence_threshold=0.7
        ... )
        >>> low_conf_item.confidence_score < 0.7
        True
    """
    if confidence_threshold is not None:
        if not 0.0 <= confidence_threshold <= 1.0:
            raise ValueError(
                f"confidence_threshold must be between 0 and 1, got {confidence_threshold}"
            )

    # Filter pending items
    pending_items = [
        item for item in self._queue.values()
        if item.status == ReviewStatus.PENDING
    ]

    # Apply item_type filter
    if item_type is not None:
        pending_items = [
            item for item in pending_items
            if item.item_type == item_type
        ]

    # Apply confidence_threshold filter
    if confidence_threshold is not None:
        pending_items = [
            item for item in pending_items
            if item.confidence_score < confidence_threshold
        ]

    # Return None if no items match criteria
    if not pending_items:
        return None

    # Define priority order for sorting
    priority_order = {
        Priority.CRITICAL: 0,
        Priority.HIGH: 1,
        Priority.MEDIUM: 2,
        Priority.LOW: 3
    }

    # Sort by priority (highest first), then confidence (lowest first), then created_at (oldest first)
    sorted_items = sorted(
        pending_items,
        key=lambda item: (
            priority_order[item.priority],
            item.confidence_score,
            item.created_at
        )
    )

    return sorted_items[0] if sorted_items else None

get_queue_stats()

Get statistics about the current queue state.

Returns:

Type	Description
Dict[str, int]	Dictionary with queue statistics:
Dict[str, int]	total: Total number of items in queue
Dict[str, int]	pending: Number of pending items
Dict[str, int]	in_review: Number of items currently in review
Dict[str, int]	completed: Number of completed items
Dict[str, int]	by_priority: Count of items by priority level
Dict[str, int]	by_type: Count of items by item type

Examples:

>>> queue = ReviewQueue()
>>> item1 = ReviewItem(
...     item_type=ItemType.LLM_EVALUATION,
...     content={"q": "Q1"},
...     confidence_score=0.45,
...     priority=Priority.CRITICAL
... )
>>> item2 = ReviewItem(
...     item_type=ItemType.AGENT_RESPONSE,
...     content={"q": "Q2"},
...     confidence_score=0.85,
...     priority=Priority.LOW
... )
>>> queue.add_to_queue(item1, auto_priority=False)
>>> queue.add_to_queue(item2, auto_priority=False)

>>> stats = queue.get_queue_stats()
>>> stats['total']
2
>>> stats['pending']
2
>>> stats['by_priority']['critical']
1

Source code in uaef/hitl/queue.py

def get_queue_stats(self) -> Dict[str, int]:
    """
    Get statistics about the current queue state.

    Returns:
        Dictionary with queue statistics:
        - total: Total number of items in queue
        - pending: Number of pending items
        - in_review: Number of items currently in review
        - completed: Number of completed items
        - by_priority: Count of items by priority level
        - by_type: Count of items by item type

    Examples:
        >>> queue = ReviewQueue()
        >>> item1 = ReviewItem(
        ...     item_type=ItemType.LLM_EVALUATION,
        ...     content={"q": "Q1"},
        ...     confidence_score=0.45,
        ...     priority=Priority.CRITICAL
        ... )
        >>> item2 = ReviewItem(
        ...     item_type=ItemType.AGENT_RESPONSE,
        ...     content={"q": "Q2"},
        ...     confidence_score=0.85,
        ...     priority=Priority.LOW
        ... )
        >>> queue.add_to_queue(item1, auto_priority=False)
        >>> queue.add_to_queue(item2, auto_priority=False)

        >>> stats = queue.get_queue_stats()
        >>> stats['total']
        2
        >>> stats['pending']
        2
        >>> stats['by_priority']['critical']
        1
    """
    items = list(self._queue.values())

    # Count by status
    pending = sum(1 for item in items if item.status == ReviewStatus.PENDING)
    in_review = sum(1 for item in items if item.status == ReviewStatus.IN_REVIEW)
    completed = sum(1 for item in items if item.status == ReviewStatus.COMPLETED)

    # Count by priority
    by_priority = {
        "critical": sum(1 for item in items if item.priority == Priority.CRITICAL),
        "high": sum(1 for item in items if item.priority == Priority.HIGH),
        "medium": sum(1 for item in items if item.priority == Priority.MEDIUM),
        "low": sum(1 for item in items if item.priority == Priority.LOW)
    }

    # Count by type
    by_type = {
        "llm_evaluation": sum(1 for item in items if item.item_type == ItemType.LLM_EVALUATION),
        "agent_response": sum(1 for item in items if item.item_type == ItemType.AGENT_RESPONSE),
        "simulated_conversation": sum(1 for item in items if item.item_type == ItemType.SIMULATED_CONVERSATION)
    }

    return {
        "total": len(items),
        "pending": pending,
        "in_review": in_review,
        "completed": completed,
        "by_priority": by_priority,
        "by_type": by_type
    }

submit_review(item_id, human_scores, reviewer_id, feedback=None)

Submit a review for an item, updating it with human scores and feedback.

This method: 1. Validates the item exists and is in a reviewable state 2. Updates the item with human scores, feedback, and reviewer_id 3. Marks the item as completed with reviewed_at timestamp

Parameters:

Name	Type	Description	Default
item_id	UUID	UUID of the item to review	required
human_scores	Dict[str, float]	Dictionary of metric_name -> score (0-1)	required
reviewer_id	str	ID of the human reviewer	required
feedback	Optional[str]	Optional textual feedback from the reviewer	None

Returns:

Type	Description
ReviewItem	The updated ReviewItem

Raises:

Type	Description
ItemNotFoundError	If item_id is not found in queue
InvalidOperationError	If item is already completed
ValueError	If human_scores are invalid (not between 0-1)

Examples:

>>> queue = ReviewQueue()
>>> item = ReviewItem(
...     item_type=ItemType.LLM_EVALUATION,
...     content={"question": "What is AI?"},
...     confidence_score=0.65,
...     priority=Priority.MEDIUM
... )
>>> queue.add_to_queue(item, auto_priority=False)

>>> # Submit review
>>> reviewed_item = queue.submit_review(
...     item_id=item.item_id,
...     human_scores={
...         "answer_relevance": 0.85,
...         "answer_correctness": 0.90
...     },
...     reviewer_id="reviewer_123",
...     feedback="Good response but could be more detailed"
... )
>>> reviewed_item.status
<ReviewStatus.COMPLETED: 'completed'>
>>> reviewed_item.human_scores
{'answer_relevance': 0.85, 'answer_correctness': 0.9}

>>> # Attempting to review again raises error
>>> queue.submit_review(
...     item_id=item.item_id,
...     human_scores={"score": 0.5},
...     reviewer_id="reviewer_456"
... )
Traceback (most recent call last):
    ...
InvalidOperationError: Item ... is already completed

Source code in uaef/hitl/queue.py

def submit_review(
    self,
    item_id: UUID,
    human_scores: Dict[str, float],
    reviewer_id: str,
    feedback: Optional[str] = None
) -> ReviewItem:
    """
    Submit a review for an item, updating it with human scores and feedback.

    This method:
    1. Validates the item exists and is in a reviewable state
    2. Updates the item with human scores, feedback, and reviewer_id
    3. Marks the item as completed with reviewed_at timestamp

    Args:
        item_id: UUID of the item to review
        human_scores: Dictionary of metric_name -> score (0-1)
        reviewer_id: ID of the human reviewer
        feedback: Optional textual feedback from the reviewer

    Returns:
        The updated ReviewItem

    Raises:
        ItemNotFoundError: If item_id is not found in queue
        InvalidOperationError: If item is already completed
        ValueError: If human_scores are invalid (not between 0-1)

    Examples:
        >>> queue = ReviewQueue()
        >>> item = ReviewItem(
        ...     item_type=ItemType.LLM_EVALUATION,
        ...     content={"question": "What is AI?"},
        ...     confidence_score=0.65,
        ...     priority=Priority.MEDIUM
        ... )
        >>> queue.add_to_queue(item, auto_priority=False)

        >>> # Submit review
        >>> reviewed_item = queue.submit_review(
        ...     item_id=item.item_id,
        ...     human_scores={
        ...         "answer_relevance": 0.85,
        ...         "answer_correctness": 0.90
        ...     },
        ...     reviewer_id="reviewer_123",
        ...     feedback="Good response but could be more detailed"
        ... )
        >>> reviewed_item.status
        <ReviewStatus.COMPLETED: 'completed'>
        >>> reviewed_item.human_scores
        {'answer_relevance': 0.85, 'answer_correctness': 0.9}

        >>> # Attempting to review again raises error
        >>> queue.submit_review(
        ...     item_id=item.item_id,
        ...     human_scores={"score": 0.5},
        ...     reviewer_id="reviewer_456"
        ... )
        Traceback (most recent call last):
            ...
        InvalidOperationError: Item ... is already completed
    """
    # Check if item exists
    if item_id not in self._queue:
        raise ItemNotFoundError(
            f"Item with ID {item_id} not found in queue"
        )

    item = self._queue[item_id]

    # Check if item is already completed
    if item.status == ReviewStatus.COMPLETED:
        raise InvalidOperationError(
            f"Item {item_id} is already completed"
        )

    # Validate human_scores
    for metric_name, score in human_scores.items():
        if not 0.0 <= score <= 1.0:
            raise ValueError(
                f"Human score for '{metric_name}' must be between 0 and 1, got {score}"
            )

    # Validate reviewer_id
    if not reviewer_id or not reviewer_id.strip():
        raise ValueError("reviewer_id cannot be empty")

    # Update item
    item.human_scores = human_scores
    item.reviewer_id = reviewer_id.strip()
    item.feedback = feedback
    item.status = ReviewStatus.COMPLETED
    item.reviewed_at = datetime.utcnow()

    return item

ReviewQueueError

Bases: Exception

Base exception for review queue errors.

Source code in uaef/hitl/queue.py

class ReviewQueueError(Exception):
    """Base exception for review queue errors."""
    pass

uaef.hitl.judge_calibration

LLM judge calibration workflow for HITL validation.

This module implements the judge calibration workflow, one of three HITL workflows in UAEF. It validates and calibrates LLM-based evaluation scores by comparing them with human judgments.

The workflow: 1. Queues low-confidence LLM evaluations for human review 2. Collects human scores for the same evaluations 3. Calculates LLM-human agreement (correlation) 4. Uses feedback to adjust prompt templates

InsufficientDataError

Bases: JudgeCalibrationError

Raised when insufficient data is available for calibration.

Source code in uaef/hitl/judge_calibration.py

class InsufficientDataError(JudgeCalibrationError):
    """Raised when insufficient data is available for calibration."""
    pass

JudgeCalibrationError

Bases: Exception

Base exception for judge calibration errors.

Source code in uaef/hitl/judge_calibration.py

class JudgeCalibrationError(Exception):
    """Base exception for judge calibration errors."""
    pass

JudgeCalibrationWorkflow

Workflow for calibrating LLM judge prompts using human feedback.

This workflow validates and calibrates LLM-based evaluation scores by: 1. Identifying low-confidence LLM evaluations that need human review 2. Queuing them for human validation 3. Collecting human scores for the same evaluations 4. Calculating agreement metrics (correlation, MAE, etc.) 5. Analyzing disagreement patterns 6. Suggesting prompt improvements based on feedback

The workflow integrates with: - ReviewQueue: Manages items needing calibration - CalibrationEngine: Performs statistical analysis and tracks performance

Attributes:

Name	Type	Description
review_queue		ReviewQueue instance for managing review items
calibration_engine		CalibrationEngine instance for analysis
confidence_threshold		Threshold below which evaluations are queued (default: 0.7)
disagreement_threshold		Threshold for identifying disagreements (default: 0.2)

Examples:

>>> # Initialize workflow
>>> workflow = JudgeCalibrationWorkflow()

>>> # Queue low-confidence evaluations
>>> evaluation_results = [
...     {
...         "evaluation_id": "eval_001",
...         "question": "What is AI?",
...         "response": "AI is artificial intelligence...",
...         "metric_scores": [
...             MetricScore(name="answer_relevance", score=0.85, confidence=0.65),
...             MetricScore(name="answer_correctness", score=0.90, confidence=0.75)
...         ]
...     }
... ]
>>> queued_items = workflow.queue_low_confidence_evaluations(
...     evaluation_results,
...     confidence_threshold=0.7
... )
>>> len(queued_items)
1

>>> # After human review, calculate agreement
>>> agreement = workflow.calculate_agreement(
...     metric_name="answer_relevance",
...     prompt_version="v1.0"
... )
>>> agreement.agreement_score
0.85

>>> # Generate calibration report
>>> report = workflow.generate_calibration_report(
...     metric_name="answer_relevance",
...     prompt_version="v1.0"
... )

>>> # Get prompt improvement suggestions
>>> suggestions = workflow.suggest_prompt_improvements(
...     metric_name="answer_relevance",
...     prompt_version="v1.0"
... )

Source code in uaef/hitl/judge_calibration.py

class JudgeCalibrationWorkflow:
    """
    Workflow for calibrating LLM judge prompts using human feedback.

    This workflow validates and calibrates LLM-based evaluation scores by:
    1. Identifying low-confidence LLM evaluations that need human review
    2. Queuing them for human validation
    3. Collecting human scores for the same evaluations
    4. Calculating agreement metrics (correlation, MAE, etc.)
    5. Analyzing disagreement patterns
    6. Suggesting prompt improvements based on feedback

    The workflow integrates with:
    - ReviewQueue: Manages items needing calibration
    - CalibrationEngine: Performs statistical analysis and tracks performance

    Attributes:
        review_queue: ReviewQueue instance for managing review items
        calibration_engine: CalibrationEngine instance for analysis
        confidence_threshold: Threshold below which evaluations are queued (default: 0.7)
        disagreement_threshold: Threshold for identifying disagreements (default: 0.2)

    Examples:
        >>> # Initialize workflow
        >>> workflow = JudgeCalibrationWorkflow()

        >>> # Queue low-confidence evaluations
        >>> evaluation_results = [
        ...     {
        ...         "evaluation_id": "eval_001",
        ...         "question": "What is AI?",
        ...         "response": "AI is artificial intelligence...",
        ...         "metric_scores": [
        ...             MetricScore(name="answer_relevance", score=0.85, confidence=0.65),
        ...             MetricScore(name="answer_correctness", score=0.90, confidence=0.75)
        ...         ]
        ...     }
        ... ]
        >>> queued_items = workflow.queue_low_confidence_evaluations(
        ...     evaluation_results,
        ...     confidence_threshold=0.7
        ... )
        >>> len(queued_items)
        1

        >>> # After human review, calculate agreement
        >>> agreement = workflow.calculate_agreement(
        ...     metric_name="answer_relevance",
        ...     prompt_version="v1.0"
        ... )
        >>> agreement.agreement_score
        0.85

        >>> # Generate calibration report
        >>> report = workflow.generate_calibration_report(
        ...     metric_name="answer_relevance",
        ...     prompt_version="v1.0"
        ... )

        >>> # Get prompt improvement suggestions
        >>> suggestions = workflow.suggest_prompt_improvements(
        ...     metric_name="answer_relevance",
        ...     prompt_version="v1.0"
        ... )
    """

    def __init__(
        self,
        review_queue: Optional[ReviewQueue] = None,
        calibration_engine: Optional[CalibrationEngine] = None,
        confidence_threshold: float = 0.7,
        disagreement_threshold: float = 0.2
    ):
        """
        Initialize the judge calibration workflow.

        Args:
            review_queue: ReviewQueue instance (creates new if None)
            calibration_engine: CalibrationEngine instance (creates new if None)
            confidence_threshold: Threshold below which evaluations are queued
            disagreement_threshold: Threshold for identifying disagreements

        Raises:
            ValueError: If thresholds are not between 0 and 1
        """
        if not 0.0 <= confidence_threshold <= 1.0:
            raise ValueError(
                f"confidence_threshold must be between 0 and 1, got {confidence_threshold}"
            )
        if not 0.0 <= disagreement_threshold <= 1.0:
            raise ValueError(
                f"disagreement_threshold must be between 0 and 1, got {disagreement_threshold}"
            )

        self.review_queue = review_queue or ReviewQueue()
        self.calibration_engine = calibration_engine or CalibrationEngine()
        self.confidence_threshold = confidence_threshold
        self.disagreement_threshold = disagreement_threshold

        logger.info(
            f"Initialized JudgeCalibrationWorkflow with "
            f"confidence_threshold={confidence_threshold}, "
            f"disagreement_threshold={disagreement_threshold}"
        )

    def queue_low_confidence_evaluations(
        self,
        evaluation_results: List[Dict[str, Any]],
        confidence_threshold: Optional[float] = None,
        auto_priority: bool = True
    ) -> List[ReviewItem]:
        """
        Queue low-confidence LLM evaluations for human review.

        Identifies evaluations with confidence scores below the threshold and
        adds them to the review queue for human validation.

        Args:
            evaluation_results: List of evaluation result dictionaries containing:
                - evaluation_id: Unique identifier for the evaluation
                - question: The question/input
                - response: The agent's response
                - metric_scores: List of MetricScore objects with confidence
                - context: Optional context information
                - ground_truth: Optional ground truth for comparison
            confidence_threshold: Override default confidence threshold
            auto_priority: If True, automatically assign priority based on confidence

        Returns:
            List of ReviewItem objects that were added to the queue

        Raises:
            ValueError: If evaluation_results format is invalid

        Examples:
            >>> workflow = JudgeCalibrationWorkflow()
            >>> results = [
            ...     {
            ...         "evaluation_id": "eval_001",
            ...         "question": "What is AI?",
            ...         "response": "AI is...",
            ...         "metric_scores": [
            ...             MetricScore(name="relevance", score=0.85, confidence=0.65)
            ...         ]
            ...     }
            ... ]
            >>> items = workflow.queue_low_confidence_evaluations(results)
            >>> len(items)
            1
            >>> items[0].confidence_score
            0.65
        """
        threshold = confidence_threshold or self.confidence_threshold
        queued_items = []

        for result in evaluation_results:
            # Validate result format
            if "evaluation_id" not in result:
                raise ValueError("evaluation_result must contain 'evaluation_id'")
            if "metric_scores" not in result:
                raise ValueError("evaluation_result must contain 'metric_scores'")

            evaluation_id = result["evaluation_id"]
            metric_scores = result["metric_scores"]

            # Check each metric score for low confidence
            for metric_score in metric_scores:
                # Get confidence (default to 1.0 if not present)
                confidence = getattr(metric_score, "confidence", 1.0)

                # Queue if confidence is below threshold
                if confidence < threshold:
                    # Create review item
                    review_item = ReviewItem(
                        item_type=ItemType.LLM_EVALUATION,
                        content={
                            "question": result.get("question", ""),
                            "agent_response": result.get("response", ""),
                            "llm_scores": {
                                metric_score.name: metric_score.score
                            },
                            "llm_reasoning": getattr(metric_score, "reasoning", ""),
                            "context": result.get("context", ""),
                            "ground_truth": result.get("ground_truth", None)
                        },
                        metadata={
                            "evaluation_id": evaluation_id,
                            "metric_name": metric_score.name,
                            "prompt_version": getattr(metric_score, "prompt_version", "unknown")
                        },
                        priority=Priority.MEDIUM,  # Will be overridden if auto_priority=True
                        confidence_score=confidence
                    )

                    # Add to queue
                    added_item = self.review_queue.add_to_queue(
                        review_item,
                        auto_priority=auto_priority
                    )
                    queued_items.append(added_item)

                    logger.debug(
                        f"Queued evaluation {evaluation_id} for metric "
                        f"{metric_score.name} with confidence {confidence:.3f}"
                    )

        logger.info(
            f"Queued {len(queued_items)} low-confidence evaluations "
            f"(threshold={threshold:.2f})"
        )

        return queued_items

    def calculate_agreement(
        self,
        metric_name: str,
        prompt_version: str,
        min_samples: int = 2
    ) -> AgreementMetrics:
        """
        Calculate agreement between LLM and human scores.

        Computes correlation metrics (Pearson, Spearman) and error metrics (MAE, RMSE)
        to measure how well LLM scores align with human judgments.

        Args:
            metric_name: Name of the metric to analyze
            prompt_version: Version of the prompt used
            min_samples: Minimum number of samples required for calculation

        Returns:
            AgreementMetrics object with correlation and error metrics

        Raises:
            InsufficientDataError: If fewer than min_samples are available

        Examples:
            >>> workflow = JudgeCalibrationWorkflow()
            >>> # After collecting human reviews...
            >>> agreement = workflow.calculate_agreement(
            ...     metric_name="answer_relevance",
            ...     prompt_version="v1.0"
            ... )
            >>> agreement.agreement_score  # Pearson correlation
            0.85
            >>> agreement.disagreement_details["mean_delta"]
            0.12
        """
        # Get completed review items for this metric
        completed_items = self.review_queue.get_all_items(
            item_type=ItemType.LLM_EVALUATION
        )

        # Filter by metric name and prompt version
        relevant_items = [
            item for item in completed_items
            if (
                item.metadata.get("metric_name") == metric_name and
                item.metadata.get("prompt_version") == prompt_version and
                item.human_scores  # Has human scores
            )
        ]

        if len(relevant_items) < min_samples:
            raise InsufficientDataError(
                f"Insufficient samples for agreement calculation: "
                f"need at least {min_samples}, got {len(relevant_items)}"
            )

        # Extract LLM and human scores
        llm_scores = []
        human_scores_list = []

        for item in relevant_items:
            # Get LLM score from content
            llm_score = item.content.get("llm_scores", {}).get(metric_name)
            if llm_score is None:
                continue

            # Get human score
            human_score = item.human_scores.get(metric_name)
            if human_score is None:
                continue

            llm_scores.append(llm_score)
            human_scores_list.append(human_score)

            # Add to calibration engine for tracking
            self.calibration_engine.add_sample(
                sample_id=str(item.item_id),
                metric_name=metric_name,
                prompt_version=prompt_version,
                llm_score=llm_score,
                human_score=human_score,
                question=item.content.get("question", ""),
                response=item.content.get("agent_response", ""),
                llm_reasoning=item.content.get("llm_reasoning", ""),
                human_reasoning=item.feedback
            )

        if len(llm_scores) < min_samples:
            raise InsufficientDataError(
                f"Insufficient valid score pairs: "
                f"need at least {min_samples}, got {len(llm_scores)}"
            )

        # Convert to numpy arrays
        llm_scores_arr = np.array(llm_scores)
        human_scores_arr = np.array(human_scores_list)

        # Calculate Pearson correlation
        pearson_corr, pearson_p = stats.pearsonr(llm_scores_arr, human_scores_arr)

        # Calculate Spearman correlation
        spearman_corr, spearman_p = stats.spearmanr(llm_scores_arr, human_scores_arr)

        # Calculate error metrics
        score_diffs = np.abs(llm_scores_arr - human_scores_arr)
        mae = np.mean(score_diffs)
        rmse = np.sqrt(np.mean((llm_scores_arr - human_scores_arr) ** 2))

        # Calculate agreement rate (within disagreement threshold)
        agreements = score_diffs <= self.disagreement_threshold
        agreement_rate = np.mean(agreements)

        # Identify disagreement samples
        disagreement_indices = np.where(score_diffs > self.disagreement_threshold)[0]

        # Create AgreementMetrics
        agreement_metrics = AgreementMetrics(
            metric_name=metric_name,
            llm_score=None,  # Not applicable for aggregate metrics
            human_scores=human_scores_list,
            agreement_score=float(pearson_corr),
            disagreement_details={
                "sample_count": len(llm_scores),
                "pearson_correlation": float(pearson_corr),
                "pearson_p_value": float(pearson_p),
                "spearman_correlation": float(spearman_corr),
                "spearman_p_value": float(spearman_p),
                "mean_absolute_error": float(mae),
                "root_mean_squared_error": float(rmse),
                "agreement_rate": float(agreement_rate),
                "disagreement_count": len(disagreement_indices),
                "mean_delta": float(np.mean(score_diffs)),
                "max_delta": float(np.max(score_diffs)),
                "std_delta": float(np.std(score_diffs)),
                "prompt_version": prompt_version
            }
        )

        logger.info(
            f"Calculated agreement for {metric_name} v{prompt_version}: "
            f"pearson={pearson_corr:.3f}, mae={mae:.3f}, "
            f"agreement_rate={agreement_rate:.1%}"
        )

        return agreement_metrics

    def generate_calibration_report(
        self,
        metric_name: str,
        prompt_version: str
    ) -> Dict[str, Any]:
        """
        Generate a comprehensive calibration report.

        The report includes:
        - Agreement metrics (correlation, MAE, RMSE)
        - Disagreement patterns and analysis
        - Sample statistics
        - Prompt improvement suggestions

        Args:
            metric_name: Name of the metric to analyze
            prompt_version: Version of the prompt used

        Returns:
            Dictionary containing the calibration report

        Raises:
            InsufficientDataError: If insufficient data for analysis

        Examples:
            >>> workflow = JudgeCalibrationWorkflow()
            >>> report = workflow.generate_calibration_report(
            ...     metric_name="answer_relevance",
            ...     prompt_version="v1.0"
            ... )
            >>> report["agreement_metrics"]["pearson_correlation"]
            0.85
            >>> report["is_well_calibrated"]
            True
            >>> len(report["improvement_suggestions"])
            3
        """
        try:
            # Calculate agreement metrics
            agreement = self.calculate_agreement(metric_name, prompt_version)

            # Get disagreement analysis from calibration engine
            disagreement_analysis = self.calibration_engine.analyze_disagreements(
                metric_name=metric_name,
                prompt_version=prompt_version,
                disagreement_threshold=self.disagreement_threshold
            )

            # Get improvement suggestions
            suggestions = self.calibration_engine.suggest_improvements(
                metric_name=metric_name,
                prompt_version=prompt_version,
                disagreement_threshold=self.disagreement_threshold
            )

            # Get calibration metrics from engine
            calibration_metrics = self.calibration_engine.calculate_metrics(
                metric_name=metric_name,
                prompt_version=prompt_version,
                disagreement_threshold=self.disagreement_threshold
            )

            # Build report
            report = {
                "metric_name": metric_name,
                "prompt_version": prompt_version,
                "timestamp": datetime.utcnow().isoformat(),
                "agreement_metrics": {
                    "pearson_correlation": agreement.disagreement_details["pearson_correlation"],
                    "pearson_p_value": agreement.disagreement_details["pearson_p_value"],
                    "spearman_correlation": agreement.disagreement_details["spearman_correlation"],
                    "spearman_p_value": agreement.disagreement_details["spearman_p_value"],
                    "mean_absolute_error": agreement.disagreement_details["mean_absolute_error"],
                    "root_mean_squared_error": agreement.disagreement_details["root_mean_squared_error"],
                    "agreement_rate": agreement.disagreement_details["agreement_rate"]
                },
                "sample_statistics": {
                    "total_samples": agreement.disagreement_details["sample_count"],
                    "disagreement_count": agreement.disagreement_details["disagreement_count"],
                    "mean_delta": agreement.disagreement_details["mean_delta"],
                    "max_delta": agreement.disagreement_details["max_delta"],
                    "std_delta": agreement.disagreement_details["std_delta"]
                },
                "disagreement_analysis": disagreement_analysis,
                "improvement_suggestions": suggestions,
                "is_well_calibrated": calibration_metrics.is_well_calibrated(),
                "calibration_thresholds": {
                    "confidence_threshold": self.confidence_threshold,
                    "disagreement_threshold": self.disagreement_threshold
                }
            }

            logger.info(
                f"Generated calibration report for {metric_name} v{prompt_version}: "
                f"well_calibrated={report['is_well_calibrated']}, "
                f"samples={report['sample_statistics']['total_samples']}"
            )

            return report

        except InsufficientDataError as e:
            logger.warning(f"Cannot generate report: {e}")
            raise

    def suggest_prompt_improvements(
        self,
        metric_name: str,
        prompt_version: str
    ) -> List[str]:
        """
        Suggest prompt improvements based on disagreement patterns.

        Analyzes disagreements between LLM and human scores to identify
        systematic issues and recommend specific prompt modifications.

        Args:
            metric_name: Name of the metric to analyze
            prompt_version: Version of the prompt used

        Returns:
            List of actionable improvement suggestions

        Examples:
            >>> workflow = JudgeCalibrationWorkflow()
            >>> suggestions = workflow.suggest_prompt_improvements(
            ...     metric_name="answer_relevance",
            ...     prompt_version="v1.0"
            ... )
            >>> suggestions[0]
            'LLM tends to overscore (avg difference: 0.15). Add stricter criteria...'
        """
        suggestions = self.calibration_engine.suggest_improvements(
            metric_name=metric_name,
            prompt_version=prompt_version,
            disagreement_threshold=self.disagreement_threshold
        )

        logger.info(
            f"Generated {len(suggestions)} improvement suggestions for "
            f"{metric_name} v{prompt_version}"
        )

        return suggestions

    def get_calibration_status(
        self,
        metric_name: Optional[str] = None
    ) -> Dict[str, Any]:
        """
        Get current calibration status across metrics.

        Provides an overview of:
        - Number of items in review queue
        - Number of completed reviews
        - Calibration metrics for each metric/version
        - Overall calibration health

        Args:
            metric_name: Optional filter by metric name

        Returns:
            Dictionary with calibration status information

        Examples:
            >>> workflow = JudgeCalibrationWorkflow()
            >>> status = workflow.get_calibration_status()
            >>> status["queue_stats"]["pending"]
            5
            >>> status["metrics"]["answer_relevance"]["is_well_calibrated"]
            True
        """
        # Get queue statistics
        queue_stats = self.review_queue.get_queue_stats()

        # Filter for LLM evaluation items only
        llm_eval_items = self.review_queue.get_all_items(
            item_type=ItemType.LLM_EVALUATION
        )

        # Group by metric name
        metrics_status = {}
        metric_names = set(
            item.metadata.get("metric_name")
            for item in llm_eval_items
            if item.metadata.get("metric_name")
        )

        # Filter by metric_name if provided
        if metric_name:
            metric_names = {metric_name} if metric_name in metric_names else set()

        for m_name in metric_names:
            # Get items for this metric
            metric_items = [
                item for item in llm_eval_items
                if item.metadata.get("metric_name") == m_name
            ]

            # Get unique prompt versions
            prompt_versions = set(
                item.metadata.get("prompt_version")
                for item in metric_items
                if item.metadata.get("prompt_version")
            )

            versions_status = {}
            for version in prompt_versions:
                version_items = [
                    item for item in metric_items
                    if item.metadata.get("prompt_version") == version
                ]

                completed_items = [
                    item for item in version_items
                    if item.human_scores
                ]

                # Try to get calibration metrics
                try:
                    calibration_metrics = self.calibration_engine.calculate_metrics(
                        metric_name=m_name,
                        prompt_version=version,
                        disagreement_threshold=self.disagreement_threshold
                    )

                    versions_status[version] = {
                        "total_items": len(version_items),
                        "completed_items": len(completed_items),
                        "pending_items": len(version_items) - len(completed_items),
                        "pearson_correlation": calibration_metrics.pearson_correlation,
                        "mean_absolute_error": calibration_metrics.mean_absolute_error,
                        "agreement_rate": calibration_metrics.agreement_rate,
                        "is_well_calibrated": calibration_metrics.is_well_calibrated()
                    }
                except (InsufficientDataError, ValueError):
                    versions_status[version] = {
                        "total_items": len(version_items),
                        "completed_items": len(completed_items),
                        "pending_items": len(version_items) - len(completed_items),
                        "status": "insufficient_data"
                    }

            metrics_status[m_name] = versions_status

        return {
            "timestamp": datetime.utcnow().isoformat(),
            "queue_stats": {
                "total_llm_evaluations": len(llm_eval_items),
                "pending": queue_stats["by_type"]["llm_evaluation"],
                "completed": sum(
                    1 for item in llm_eval_items
                    if item.human_scores
                )
            },
            "metrics": metrics_status,
            "thresholds": {
                "confidence_threshold": self.confidence_threshold,
                "disagreement_threshold": self.disagreement_threshold
            }
        }

calculate_agreement(metric_name, prompt_version, min_samples=2)

Calculate agreement between LLM and human scores.

Computes correlation metrics (Pearson, Spearman) and error metrics (MAE, RMSE) to measure how well LLM scores align with human judgments.

Parameters:

Name	Type	Description	Default
metric_name	str	Name of the metric to analyze	required
prompt_version	str	Version of the prompt used	required
min_samples	int	Minimum number of samples required for calculation	2

Returns:

Type	Description
AgreementMetrics	AgreementMetrics object with correlation and error metrics

Raises:

Type	Description
InsufficientDataError	If fewer than min_samples are available

Examples:

>>> workflow = JudgeCalibrationWorkflow()
>>> # After collecting human reviews...
>>> agreement = workflow.calculate_agreement(
...     metric_name="answer_relevance",
...     prompt_version="v1.0"
... )
>>> agreement.agreement_score  # Pearson correlation
0.85
>>> agreement.disagreement_details["mean_delta"]
0.12

Source code in uaef/hitl/judge_calibration.py

def calculate_agreement(
    self,
    metric_name: str,
    prompt_version: str,
    min_samples: int = 2
) -> AgreementMetrics:
    """
    Calculate agreement between LLM and human scores.

    Computes correlation metrics (Pearson, Spearman) and error metrics (MAE, RMSE)
    to measure how well LLM scores align with human judgments.

    Args:
        metric_name: Name of the metric to analyze
        prompt_version: Version of the prompt used
        min_samples: Minimum number of samples required for calculation

    Returns:
        AgreementMetrics object with correlation and error metrics

    Raises:
        InsufficientDataError: If fewer than min_samples are available

    Examples:
        >>> workflow = JudgeCalibrationWorkflow()
        >>> # After collecting human reviews...
        >>> agreement = workflow.calculate_agreement(
        ...     metric_name="answer_relevance",
        ...     prompt_version="v1.0"
        ... )
        >>> agreement.agreement_score  # Pearson correlation
        0.85
        >>> agreement.disagreement_details["mean_delta"]
        0.12
    """
    # Get completed review items for this metric
    completed_items = self.review_queue.get_all_items(
        item_type=ItemType.LLM_EVALUATION
    )

    # Filter by metric name and prompt version
    relevant_items = [
        item for item in completed_items
        if (
            item.metadata.get("metric_name") == metric_name and
            item.metadata.get("prompt_version") == prompt_version and
            item.human_scores  # Has human scores
        )
    ]

    if len(relevant_items) < min_samples:
        raise InsufficientDataError(
            f"Insufficient samples for agreement calculation: "
            f"need at least {min_samples}, got {len(relevant_items)}"
        )

    # Extract LLM and human scores
    llm_scores = []
    human_scores_list = []

    for item in relevant_items:
        # Get LLM score from content
        llm_score = item.content.get("llm_scores", {}).get(metric_name)
        if llm_score is None:
            continue

        # Get human score
        human_score = item.human_scores.get(metric_name)
        if human_score is None:
            continue

        llm_scores.append(llm_score)
        human_scores_list.append(human_score)

        # Add to calibration engine for tracking
        self.calibration_engine.add_sample(
            sample_id=str(item.item_id),
            metric_name=metric_name,
            prompt_version=prompt_version,
            llm_score=llm_score,
            human_score=human_score,
            question=item.content.get("question", ""),
            response=item.content.get("agent_response", ""),
            llm_reasoning=item.content.get("llm_reasoning", ""),
            human_reasoning=item.feedback
        )

    if len(llm_scores) < min_samples:
        raise InsufficientDataError(
            f"Insufficient valid score pairs: "
            f"need at least {min_samples}, got {len(llm_scores)}"
        )

    # Convert to numpy arrays
    llm_scores_arr = np.array(llm_scores)
    human_scores_arr = np.array(human_scores_list)

    # Calculate Pearson correlation
    pearson_corr, pearson_p = stats.pearsonr(llm_scores_arr, human_scores_arr)

    # Calculate Spearman correlation
    spearman_corr, spearman_p = stats.spearmanr(llm_scores_arr, human_scores_arr)

    # Calculate error metrics
    score_diffs = np.abs(llm_scores_arr - human_scores_arr)
    mae = np.mean(score_diffs)
    rmse = np.sqrt(np.mean((llm_scores_arr - human_scores_arr) ** 2))

    # Calculate agreement rate (within disagreement threshold)
    agreements = score_diffs <= self.disagreement_threshold
    agreement_rate = np.mean(agreements)

    # Identify disagreement samples
    disagreement_indices = np.where(score_diffs > self.disagreement_threshold)[0]

    # Create AgreementMetrics
    agreement_metrics = AgreementMetrics(
        metric_name=metric_name,
        llm_score=None,  # Not applicable for aggregate metrics
        human_scores=human_scores_list,
        agreement_score=float(pearson_corr),
        disagreement_details={
            "sample_count": len(llm_scores),
            "pearson_correlation": float(pearson_corr),
            "pearson_p_value": float(pearson_p),
            "spearman_correlation": float(spearman_corr),
            "spearman_p_value": float(spearman_p),
            "mean_absolute_error": float(mae),
            "root_mean_squared_error": float(rmse),
            "agreement_rate": float(agreement_rate),
            "disagreement_count": len(disagreement_indices),
            "mean_delta": float(np.mean(score_diffs)),
            "max_delta": float(np.max(score_diffs)),
            "std_delta": float(np.std(score_diffs)),
            "prompt_version": prompt_version
        }
    )

    logger.info(
        f"Calculated agreement for {metric_name} v{prompt_version}: "
        f"pearson={pearson_corr:.3f}, mae={mae:.3f}, "
        f"agreement_rate={agreement_rate:.1%}"
    )

    return agreement_metrics

generate_calibration_report(metric_name, prompt_version)

Generate a comprehensive calibration report.

The report includes: - Agreement metrics (correlation, MAE, RMSE) - Disagreement patterns and analysis - Sample statistics - Prompt improvement suggestions

Parameters:

Name	Type	Description	Default
metric_name	str	Name of the metric to analyze	required
prompt_version	str	Version of the prompt used	required

Returns:

Type	Description
Dict[str, Any]	Dictionary containing the calibration report

Raises:

Type	Description
InsufficientDataError	If insufficient data for analysis

Examples:

>>> workflow = JudgeCalibrationWorkflow()
>>> report = workflow.generate_calibration_report(
...     metric_name="answer_relevance",
...     prompt_version="v1.0"
... )
>>> report["agreement_metrics"]["pearson_correlation"]
0.85
>>> report["is_well_calibrated"]
True
>>> len(report["improvement_suggestions"])
3

Source code in uaef/hitl/judge_calibration.py

def generate_calibration_report(
    self,
    metric_name: str,
    prompt_version: str
) -> Dict[str, Any]:
    """
    Generate a comprehensive calibration report.

    The report includes:
    - Agreement metrics (correlation, MAE, RMSE)
    - Disagreement patterns and analysis
    - Sample statistics
    - Prompt improvement suggestions

    Args:
        metric_name: Name of the metric to analyze
        prompt_version: Version of the prompt used

    Returns:
        Dictionary containing the calibration report

    Raises:
        InsufficientDataError: If insufficient data for analysis

    Examples:
        >>> workflow = JudgeCalibrationWorkflow()
        >>> report = workflow.generate_calibration_report(
        ...     metric_name="answer_relevance",
        ...     prompt_version="v1.0"
        ... )
        >>> report["agreement_metrics"]["pearson_correlation"]
        0.85
        >>> report["is_well_calibrated"]
        True
        >>> len(report["improvement_suggestions"])
        3
    """
    try:
        # Calculate agreement metrics
        agreement = self.calculate_agreement(metric_name, prompt_version)

        # Get disagreement analysis from calibration engine
        disagreement_analysis = self.calibration_engine.analyze_disagreements(
            metric_name=metric_name,
            prompt_version=prompt_version,
            disagreement_threshold=self.disagreement_threshold
        )

        # Get improvement suggestions
        suggestions = self.calibration_engine.suggest_improvements(
            metric_name=metric_name,
            prompt_version=prompt_version,
            disagreement_threshold=self.disagreement_threshold
        )

        # Get calibration metrics from engine
        calibration_metrics = self.calibration_engine.calculate_metrics(
            metric_name=metric_name,
            prompt_version=prompt_version,
            disagreement_threshold=self.disagreement_threshold
        )

        # Build report
        report = {
            "metric_name": metric_name,
            "prompt_version": prompt_version,
            "timestamp": datetime.utcnow().isoformat(),
            "agreement_metrics": {
                "pearson_correlation": agreement.disagreement_details["pearson_correlation"],
                "pearson_p_value": agreement.disagreement_details["pearson_p_value"],
                "spearman_correlation": agreement.disagreement_details["spearman_correlation"],
                "spearman_p_value": agreement.disagreement_details["spearman_p_value"],
                "mean_absolute_error": agreement.disagreement_details["mean_absolute_error"],
                "root_mean_squared_error": agreement.disagreement_details["root_mean_squared_error"],
                "agreement_rate": agreement.disagreement_details["agreement_rate"]
            },
            "sample_statistics": {
                "total_samples": agreement.disagreement_details["sample_count"],
                "disagreement_count": agreement.disagreement_details["disagreement_count"],
                "mean_delta": agreement.disagreement_details["mean_delta"],
                "max_delta": agreement.disagreement_details["max_delta"],
                "std_delta": agreement.disagreement_details["std_delta"]
            },
            "disagreement_analysis": disagreement_analysis,
            "improvement_suggestions": suggestions,
            "is_well_calibrated": calibration_metrics.is_well_calibrated(),
            "calibration_thresholds": {
                "confidence_threshold": self.confidence_threshold,
                "disagreement_threshold": self.disagreement_threshold
            }
        }

        logger.info(
            f"Generated calibration report for {metric_name} v{prompt_version}: "
            f"well_calibrated={report['is_well_calibrated']}, "
            f"samples={report['sample_statistics']['total_samples']}"
        )

        return report

    except InsufficientDataError as e:
        logger.warning(f"Cannot generate report: {e}")
        raise

get_calibration_status(metric_name=None)

Get current calibration status across metrics.

Provides an overview of: - Number of items in review queue - Number of completed reviews - Calibration metrics for each metric/version - Overall calibration health

Parameters:

Name	Type	Description	Default
metric_name	Optional[str]	Optional filter by metric name	None

Returns:

Type	Description
Dict[str, Any]	Dictionary with calibration status information

Examples:

>>> workflow = JudgeCalibrationWorkflow()
>>> status = workflow.get_calibration_status()
>>> status["queue_stats"]["pending"]
5
>>> status["metrics"]["answer_relevance"]["is_well_calibrated"]
True

Source code in uaef/hitl/judge_calibration.py

def get_calibration_status(
    self,
    metric_name: Optional[str] = None
) -> Dict[str, Any]:
    """
    Get current calibration status across metrics.

    Provides an overview of:
    - Number of items in review queue
    - Number of completed reviews
    - Calibration metrics for each metric/version
    - Overall calibration health

    Args:
        metric_name: Optional filter by metric name

    Returns:
        Dictionary with calibration status information

    Examples:
        >>> workflow = JudgeCalibrationWorkflow()
        >>> status = workflow.get_calibration_status()
        >>> status["queue_stats"]["pending"]
        5
        >>> status["metrics"]["answer_relevance"]["is_well_calibrated"]
        True
    """
    # Get queue statistics
    queue_stats = self.review_queue.get_queue_stats()

    # Filter for LLM evaluation items only
    llm_eval_items = self.review_queue.get_all_items(
        item_type=ItemType.LLM_EVALUATION
    )

    # Group by metric name
    metrics_status = {}
    metric_names = set(
        item.metadata.get("metric_name")
        for item in llm_eval_items
        if item.metadata.get("metric_name")
    )

    # Filter by metric_name if provided
    if metric_name:
        metric_names = {metric_name} if metric_name in metric_names else set()

    for m_name in metric_names:
        # Get items for this metric
        metric_items = [
            item for item in llm_eval_items
            if item.metadata.get("metric_name") == m_name
        ]

        # Get unique prompt versions
        prompt_versions = set(
            item.metadata.get("prompt_version")
            for item in metric_items
            if item.metadata.get("prompt_version")
        )

        versions_status = {}
        for version in prompt_versions:
            version_items = [
                item for item in metric_items
                if item.metadata.get("prompt_version") == version
            ]

            completed_items = [
                item for item in version_items
                if item.human_scores
            ]

            # Try to get calibration metrics
            try:
                calibration_metrics = self.calibration_engine.calculate_metrics(
                    metric_name=m_name,
                    prompt_version=version,
                    disagreement_threshold=self.disagreement_threshold
                )

                versions_status[version] = {
                    "total_items": len(version_items),
                    "completed_items": len(completed_items),
                    "pending_items": len(version_items) - len(completed_items),
                    "pearson_correlation": calibration_metrics.pearson_correlation,
                    "mean_absolute_error": calibration_metrics.mean_absolute_error,
                    "agreement_rate": calibration_metrics.agreement_rate,
                    "is_well_calibrated": calibration_metrics.is_well_calibrated()
                }
            except (InsufficientDataError, ValueError):
                versions_status[version] = {
                    "total_items": len(version_items),
                    "completed_items": len(completed_items),
                    "pending_items": len(version_items) - len(completed_items),
                    "status": "insufficient_data"
                }

        metrics_status[m_name] = versions_status

    return {
        "timestamp": datetime.utcnow().isoformat(),
        "queue_stats": {
            "total_llm_evaluations": len(llm_eval_items),
            "pending": queue_stats["by_type"]["llm_evaluation"],
            "completed": sum(
                1 for item in llm_eval_items
                if item.human_scores
            )
        },
        "metrics": metrics_status,
        "thresholds": {
            "confidence_threshold": self.confidence_threshold,
            "disagreement_threshold": self.disagreement_threshold
        }
    }

queue_low_confidence_evaluations(evaluation_results, confidence_threshold=None, auto_priority=True)

Queue low-confidence LLM evaluations for human review.

Identifies evaluations with confidence scores below the threshold and adds them to the review queue for human validation.

Parameters:

Name	Type	Description	Default
evaluation_results	List[Dict[str, Any]]	List of evaluation result dictionaries containing: - evaluation_id: Unique identifier for the evaluation - question: The question/input - response: The agent's response - metric_scores: List of MetricScore objects with confidence - context: Optional context information - ground_truth: Optional ground truth for comparison	required
confidence_threshold	Optional[float]	Override default confidence threshold	None
auto_priority	bool	If True, automatically assign priority based on confidence	True

Returns:

Type	Description
List[ReviewItem]	List of ReviewItem objects that were added to the queue

Raises:

Type	Description
ValueError	If evaluation_results format is invalid

Examples:

>>> workflow = JudgeCalibrationWorkflow()
>>> results = [
...     {
...         "evaluation_id": "eval_001",
...         "question": "What is AI?",
...         "response": "AI is...",
...         "metric_scores": [
...             MetricScore(name="relevance", score=0.85, confidence=0.65)
...         ]
...     }
... ]
>>> items = workflow.queue_low_confidence_evaluations(results)
>>> len(items)
1
>>> items[0].confidence_score
0.65

Source code in uaef/hitl/judge_calibration.py

def queue_low_confidence_evaluations(
    self,
    evaluation_results: List[Dict[str, Any]],
    confidence_threshold: Optional[float] = None,
    auto_priority: bool = True
) -> List[ReviewItem]:
    """
    Queue low-confidence LLM evaluations for human review.

    Identifies evaluations with confidence scores below the threshold and
    adds them to the review queue for human validation.

    Args:
        evaluation_results: List of evaluation result dictionaries containing:
            - evaluation_id: Unique identifier for the evaluation
            - question: The question/input
            - response: The agent's response
            - metric_scores: List of MetricScore objects with confidence
            - context: Optional context information
            - ground_truth: Optional ground truth for comparison
        confidence_threshold: Override default confidence threshold
        auto_priority: If True, automatically assign priority based on confidence

    Returns:
        List of ReviewItem objects that were added to the queue

    Raises:
        ValueError: If evaluation_results format is invalid

    Examples:
        >>> workflow = JudgeCalibrationWorkflow()
        >>> results = [
        ...     {
        ...         "evaluation_id": "eval_001",
        ...         "question": "What is AI?",
        ...         "response": "AI is...",
        ...         "metric_scores": [
        ...             MetricScore(name="relevance", score=0.85, confidence=0.65)
        ...         ]
        ...     }
        ... ]
        >>> items = workflow.queue_low_confidence_evaluations(results)
        >>> len(items)
        1
        >>> items[0].confidence_score
        0.65
    """
    threshold = confidence_threshold or self.confidence_threshold
    queued_items = []

    for result in evaluation_results:
        # Validate result format
        if "evaluation_id" not in result:
            raise ValueError("evaluation_result must contain 'evaluation_id'")
        if "metric_scores" not in result:
            raise ValueError("evaluation_result must contain 'metric_scores'")

        evaluation_id = result["evaluation_id"]
        metric_scores = result["metric_scores"]

        # Check each metric score for low confidence
        for metric_score in metric_scores:
            # Get confidence (default to 1.0 if not present)
            confidence = getattr(metric_score, "confidence", 1.0)

            # Queue if confidence is below threshold
            if confidence < threshold:
                # Create review item
                review_item = ReviewItem(
                    item_type=ItemType.LLM_EVALUATION,
                    content={
                        "question": result.get("question", ""),
                        "agent_response": result.get("response", ""),
                        "llm_scores": {
                            metric_score.name: metric_score.score
                        },
                        "llm_reasoning": getattr(metric_score, "reasoning", ""),
                        "context": result.get("context", ""),
                        "ground_truth": result.get("ground_truth", None)
                    },
                    metadata={
                        "evaluation_id": evaluation_id,
                        "metric_name": metric_score.name,
                        "prompt_version": getattr(metric_score, "prompt_version", "unknown")
                    },
                    priority=Priority.MEDIUM,  # Will be overridden if auto_priority=True
                    confidence_score=confidence
                )

                # Add to queue
                added_item = self.review_queue.add_to_queue(
                    review_item,
                    auto_priority=auto_priority
                )
                queued_items.append(added_item)

                logger.debug(
                    f"Queued evaluation {evaluation_id} for metric "
                    f"{metric_score.name} with confidence {confidence:.3f}"
                )

    logger.info(
        f"Queued {len(queued_items)} low-confidence evaluations "
        f"(threshold={threshold:.2f})"
    )

    return queued_items

suggest_prompt_improvements(metric_name, prompt_version)

Suggest prompt improvements based on disagreement patterns.

Analyzes disagreements between LLM and human scores to identify systematic issues and recommend specific prompt modifications.

Parameters:

Name	Type	Description	Default
metric_name	str	Name of the metric to analyze	required
prompt_version	str	Version of the prompt used	required

Returns:

Type	Description
List[str]	List of actionable improvement suggestions

Examples:

>>> workflow = JudgeCalibrationWorkflow()
>>> suggestions = workflow.suggest_prompt_improvements(
...     metric_name="answer_relevance",
...     prompt_version="v1.0"
... )
>>> suggestions[0]
'LLM tends to overscore (avg difference: 0.15). Add stricter criteria...'

Source code in uaef/hitl/judge_calibration.py

def suggest_prompt_improvements(
    self,
    metric_name: str,
    prompt_version: str
) -> List[str]:
    """
    Suggest prompt improvements based on disagreement patterns.

    Analyzes disagreements between LLM and human scores to identify
    systematic issues and recommend specific prompt modifications.

    Args:
        metric_name: Name of the metric to analyze
        prompt_version: Version of the prompt used

    Returns:
        List of actionable improvement suggestions

    Examples:
        >>> workflow = JudgeCalibrationWorkflow()
        >>> suggestions = workflow.suggest_prompt_improvements(
        ...     metric_name="answer_relevance",
        ...     prompt_version="v1.0"
        ... )
        >>> suggestions[0]
        'LLM tends to overscore (avg difference: 0.15). Add stricter criteria...'
    """
    suggestions = self.calibration_engine.suggest_improvements(
        metric_name=metric_name,
        prompt_version=prompt_version,
        disagreement_threshold=self.disagreement_threshold
    )

    logger.info(
        f"Generated {len(suggestions)} improvement suggestions for "
        f"{metric_name} v{prompt_version}"
    )

    return suggestions

uaef.hitl.agreement

Agreement calculation utilities for HITL workflows.

This module provides statistical measures of agreement between reviewers and between LLM judges and human reviewers. It supports: - Inter-rater agreement (Krippendorff's alpha, Cohen's kappa) - LLM-human correlation (Pearson, Spearman) - Disputed item identification - Comprehensive agreement reporting

AgreementCalculationError

Bases: Exception

Base exception for agreement calculation errors.

Source code in uaef/hitl/agreement.py

class AgreementCalculationError(Exception):
    """Base exception for agreement calculation errors."""
    pass

AgreementCalculator

Calculate agreement metrics for HITL workflows.

This class provides statistical measures of agreement between reviewers and between LLM judges and human reviewers. It supports all three HITL workflows: 1. LLM judge calibration - measuring LLM-human agreement 2. Online evaluation feedback - tracking reviewer consensus 3. Simulated conversation scoring - validating conversation quality scores

The calculator implements: - Inter-rater agreement: Krippendorff's alpha, Cohen's kappa - LLM-human correlation: Pearson, Spearman - Disputed item identification: Items with low agreement - Agreement reporting: Comprehensive statistical summaries

Attributes:

Name	Type	Description
disagreement_threshold		Score difference threshold for disputes (default: 0.2)
min_samples		Minimum samples required for calculations (default: 2)

Examples:

>>> # Calculate inter-rater agreement
>>> calculator = AgreementCalculator()
>>> ratings = np.array([[0.8, 0.85, 0.9], [0.5, 0.6, 0.55]])
>>> alpha = calculator.calculate_krippendorff_alpha(ratings)
>>> print(f"Krippendorff's alpha: {alpha:.3f}")

>>> # Calculate LLM-human correlation
>>> llm_scores = [0.8, 0.7, 0.9, 0.6]
>>> human_scores = [0.85, 0.75, 0.88, 0.65]
>>> correlation = calculator.calculate_llm_human_correlation(
...     llm_scores, human_scores
... )
>>> print(f"Pearson: {correlation['pearson']:.3f}")

>>> # Identify disputed items
>>> review_items = [...]  # List of ReviewItem objects
>>> disputed = calculator.identify_disputed_items(
...     review_items, metric_name="answer_relevance"
... )
>>> print(f"Found {len(disputed)} disputed items")

>>> # Generate comprehensive report
>>> report = calculator.generate_agreement_report(
...     review_items, metric_name="answer_relevance"
... )
>>> print(f"Inter-rater agreement: {report['inter_rater_agreement']:.3f}")

Source code in uaef/hitl/agreement.py

class AgreementCalculator:
    """
    Calculate agreement metrics for HITL workflows.

    This class provides statistical measures of agreement between reviewers
    and between LLM judges and human reviewers. It supports all three HITL
    workflows:
    1. LLM judge calibration - measuring LLM-human agreement
    2. Online evaluation feedback - tracking reviewer consensus
    3. Simulated conversation scoring - validating conversation quality scores

    The calculator implements:
    - Inter-rater agreement: Krippendorff's alpha, Cohen's kappa
    - LLM-human correlation: Pearson, Spearman
    - Disputed item identification: Items with low agreement
    - Agreement reporting: Comprehensive statistical summaries

    Attributes:
        disagreement_threshold: Score difference threshold for disputes (default: 0.2)
        min_samples: Minimum samples required for calculations (default: 2)

    Examples:
        >>> # Calculate inter-rater agreement
        >>> calculator = AgreementCalculator()
        >>> ratings = np.array([[0.8, 0.85, 0.9], [0.5, 0.6, 0.55]])
        >>> alpha = calculator.calculate_krippendorff_alpha(ratings)
        >>> print(f"Krippendorff's alpha: {alpha:.3f}")

        >>> # Calculate LLM-human correlation
        >>> llm_scores = [0.8, 0.7, 0.9, 0.6]
        >>> human_scores = [0.85, 0.75, 0.88, 0.65]
        >>> correlation = calculator.calculate_llm_human_correlation(
        ...     llm_scores, human_scores
        ... )
        >>> print(f"Pearson: {correlation['pearson']:.3f}")

        >>> # Identify disputed items
        >>> review_items = [...]  # List of ReviewItem objects
        >>> disputed = calculator.identify_disputed_items(
        ...     review_items, metric_name="answer_relevance"
        ... )
        >>> print(f"Found {len(disputed)} disputed items")

        >>> # Generate comprehensive report
        >>> report = calculator.generate_agreement_report(
        ...     review_items, metric_name="answer_relevance"
        ... )
        >>> print(f"Inter-rater agreement: {report['inter_rater_agreement']:.3f}")
    """

    def __init__(
        self,
        disagreement_threshold: float = 0.2,
        min_samples: int = 2
    ):
        """
        Initialize the agreement calculator.

        Args:
            disagreement_threshold: Score difference threshold for disputes.
                Scores differing by more than this are considered disagreements.
                Must be between 0 and 1. Default: 0.2
            min_samples: Minimum number of samples required for calculations.
                Must be at least 2. Default: 2

        Raises:
            ValueError: If thresholds are invalid

        Examples:
            >>> # Default thresholds
            >>> calculator = AgreementCalculator()

            >>> # Custom thresholds
            >>> calculator = AgreementCalculator(
            ...     disagreement_threshold=0.15,
            ...     min_samples=3
            ... )
        """
        if not 0.0 <= disagreement_threshold <= 1.0:
            raise ValueError(
                f"disagreement_threshold must be between 0 and 1, "
                f"got {disagreement_threshold}"
            )

        if min_samples < 2:
            raise ValueError(
                f"min_samples must be at least 2, got {min_samples}"
            )

        self.disagreement_threshold = disagreement_threshold
        self.min_samples = min_samples

        logger.info(
            f"Initialized AgreementCalculator with "
            f"disagreement_threshold={disagreement_threshold}, "
            f"min_samples={min_samples}"
        )


    def calculate_krippendorff_alpha(
        self,
        ratings: np.ndarray,
        metric: str = "interval"
    ) -> float:
        """
        Calculate Krippendorff's alpha for inter-rater agreement.

        Krippendorff's alpha is a reliability coefficient that measures agreement
        among multiple raters. It handles missing data, works with any number of
        raters, and supports different measurement levels (nominal, ordinal, interval).

        Alpha ranges from 0 to 1:
        - 1.0: Perfect agreement
        - 0.8+: Good agreement (generally acceptable)
        - 0.67-0.8: Tentative agreement
        - < 0.67: Low agreement (not reliable)

        Args:
            ratings: 2D array of shape (n_items, n_raters) where each row is an item
                and each column is a rater. Use np.nan for missing ratings.
            metric: Distance metric to use. Options:
                - "interval": For continuous scores (default)
                - "nominal": For categorical data
                - "ordinal": For ranked data

        Returns:
            Krippendorff's alpha coefficient (0-1)

        Raises:
            InsufficientDataError: If fewer than min_samples items
            ValueError: If ratings array is invalid

        Examples:
            >>> # Perfect agreement
            >>> ratings = np.array([[0.8, 0.8, 0.8], [0.5, 0.5, 0.5]])
            >>> alpha = calculator.calculate_krippendorff_alpha(ratings)
            >>> print(f"Alpha: {alpha:.3f}")  # 1.0

            >>> # With missing data
            >>> ratings = np.array([[0.8, 0.85, np.nan], [0.5, np.nan, 0.55]])
            >>> alpha = calculator.calculate_krippendorff_alpha(ratings)

            >>> # Binary data (nominal)
            >>> ratings = np.array([[1, 1, 0], [0, 0, 0], [1, 1, 1]])
            >>> alpha = calculator.calculate_krippendorff_alpha(
            ...     ratings, metric="nominal"
            ... )
        """
        if ratings.ndim != 2:
            raise ValueError(
                f"ratings must be 2D array, got shape {ratings.shape}"
            )

        n_items, n_raters = ratings.shape

        if n_items < self.min_samples:
            raise InsufficientDataError(
                f"Insufficient items for Krippendorff's alpha: "
                f"need at least {self.min_samples}, got {n_items}"
            )

        if n_raters < 2:
            raise ValueError(
                f"Need at least 2 raters, got {n_raters}"
            )

        # Calculate observed disagreement
        observed_disagreement = self._calculate_disagreement(ratings, metric)

        # Calculate expected disagreement
        expected_disagreement = self._calculate_expected_disagreement(ratings, metric)

        # Avoid division by zero
        if expected_disagreement == 0:
            # If expected disagreement is 0, all values are identical
            return 1.0

        # Krippendorff's alpha = 1 - (observed / expected)
        alpha = 1.0 - (observed_disagreement / expected_disagreement)

        logger.debug(
            f"Krippendorff's alpha: {alpha:.3f} "
            f"(observed={observed_disagreement:.3f}, "
            f"expected={expected_disagreement:.3f})"
        )

        return float(alpha)

    def _calculate_disagreement(
        self,
        ratings: np.ndarray,
        metric: str
    ) -> float:
        """Calculate observed disagreement for Krippendorff's alpha."""
        n_items, n_raters = ratings.shape
        total_disagreement = 0.0
        total_comparisons = 0

        for i in range(n_items):
            # Get non-missing ratings for this item
            item_ratings = ratings[i, ~np.isnan(ratings[i, :])]
            n_valid = len(item_ratings)

            if n_valid < 2:
                continue

            # Calculate pairwise disagreements
            for j in range(n_valid):
                for k in range(j + 1, n_valid):
                    disagreement = self._distance(
                        item_ratings[j], item_ratings[k], metric
                    )
                    total_disagreement += disagreement
                    total_comparisons += 1

        if total_comparisons == 0:
            return 0.0

        return total_disagreement / total_comparisons

    def _calculate_expected_disagreement(
        self,
        ratings: np.ndarray,
        metric: str
    ) -> float:
        """Calculate expected disagreement for Krippendorff's alpha."""
        # Flatten all non-missing ratings
        all_ratings = ratings[~np.isnan(ratings)]
        n_total = len(all_ratings)

        if n_total < 2:
            return 0.0

        # Calculate expected disagreement from all possible pairs
        total_disagreement = 0.0
        total_comparisons = 0

        for i in range(n_total):
            for j in range(i + 1, n_total):
                disagreement = self._distance(
                    all_ratings[i], all_ratings[j], metric
                )
                total_disagreement += disagreement
                total_comparisons += 1

        if total_comparisons == 0:
            return 0.0

        return total_disagreement / total_comparisons

    def _distance(self, value1: float, value2: float, metric: str) -> float:
        """Calculate distance between two values based on metric type."""
        if metric == "nominal":
            # Binary: 0 if same, 1 if different
            return 0.0 if value1 == value2 else 1.0
        elif metric == "ordinal":
            # Rank-based distance
            return abs(value1 - value2)
        elif metric == "interval":
            # Squared difference for interval data
            return (value1 - value2) ** 2
        else:
            raise ValueError(f"Unknown metric: {metric}")


    def calculate_cohen_kappa(
        self,
        rater1_scores: List[float],
        rater2_scores: List[float],
        weights: Optional[str] = None
    ) -> float:
        """
        Calculate Cohen's kappa for agreement between two raters.

        Cohen's kappa measures agreement between two raters, correcting for
        chance agreement. It's particularly useful for binary or categorical
        ratings but can also be used with continuous scores.

        Kappa ranges from -1 to 1:
        - 1.0: Perfect agreement
        - 0.8-1.0: Almost perfect agreement
        - 0.6-0.8: Substantial agreement
        - 0.4-0.6: Moderate agreement
        - 0.2-0.4: Fair agreement
        - 0.0-0.2: Slight agreement
        - < 0: Less than chance agreement

        Args:
            rater1_scores: Scores from first rater
            rater2_scores: Scores from second rater (must be same length)
            weights: Weighting scheme for disagreements:
                - None: Unweighted (default)
                - "linear": Linear weights (for ordinal data)
                - "quadratic": Quadratic weights (for interval data)

        Returns:
            Cohen's kappa coefficient (-1 to 1)

        Raises:
            InsufficientDataError: If fewer than min_samples
            ValueError: If score lists have different lengths

        Examples:
            >>> # Perfect agreement
            >>> rater1 = [0.8, 0.5, 0.9, 0.6]
            >>> rater2 = [0.8, 0.5, 0.9, 0.6]
            >>> kappa = calculator.calculate_cohen_kappa(rater1, rater2)
            >>> print(f"Kappa: {kappa:.3f}")  # 1.0

            >>> # Moderate agreement
            >>> rater1 = [0.8, 0.5, 0.9, 0.6]
            >>> rater2 = [0.85, 0.55, 0.88, 0.65]
            >>> kappa = calculator.calculate_cohen_kappa(rater1, rater2)

            >>> # With quadratic weights (for continuous scores)
            >>> kappa = calculator.calculate_cohen_kappa(
            ...     rater1, rater2, weights="quadratic"
            ... )
        """
        if len(rater1_scores) != len(rater2_scores):
            raise ValueError(
                f"Rater score lists must have same length: "
                f"got {len(rater1_scores)} and {len(rater2_scores)}"
            )

        if len(rater1_scores) < self.min_samples:
            raise InsufficientDataError(
                f"Insufficient samples for Cohen's kappa: "
                f"need at least {self.min_samples}, got {len(rater1_scores)}"
            )

        # Convert to numpy arrays
        rater1 = np.array(rater1_scores)
        rater2 = np.array(rater2_scores)

        # For continuous scores, we need to discretize or use weighted kappa
        # We'll bin the scores into categories for standard kappa
        if weights is None:
            # Discretize continuous scores into bins
            n_bins = min(10, len(rater1_scores))
            rater1_binned = np.digitize(rater1, np.linspace(0, 1, n_bins))
            rater2_binned = np.digitize(rater2, np.linspace(0, 1, n_bins))

            # Calculate observed agreement
            observed_agreement = np.mean(rater1_binned == rater2_binned)

            # Calculate expected agreement
            unique_bins = np.unique(np.concatenate([rater1_binned, rater2_binned]))
            expected_agreement = 0.0

            for bin_val in unique_bins:
                p1 = np.mean(rater1_binned == bin_val)
                p2 = np.mean(rater2_binned == bin_val)
                expected_agreement += p1 * p2

        else:
            # Weighted kappa for continuous scores
            observed_agreement = self._calculate_weighted_agreement(
                rater1, rater2, weights
            )
            expected_agreement = self._calculate_expected_weighted_agreement(
                rater1, rater2, weights
            )

        # Avoid division by zero
        if expected_agreement >= 1.0:
            return 1.0

        # Cohen's kappa = (observed - expected) / (1 - expected)
        kappa = (observed_agreement - expected_agreement) / (1.0 - expected_agreement)

        logger.debug(
            f"Cohen's kappa: {kappa:.3f} "
            f"(observed={observed_agreement:.3f}, "
            f"expected={expected_agreement:.3f})"
        )

        return float(kappa)

    def _calculate_weighted_agreement(
        self,
        rater1: np.ndarray,
        rater2: np.ndarray,
        weights: str
    ) -> float:
        """Calculate weighted observed agreement."""
        n = len(rater1)
        total_weight = 0.0

        for i in range(n):
            weight = self._get_weight(rater1[i], rater2[i], weights)
            total_weight += weight

        return total_weight / n

    def _calculate_expected_weighted_agreement(
        self,
        rater1: np.ndarray,
        rater2: np.ndarray,
        weights: str
    ) -> float:
        """Calculate expected weighted agreement."""
        n = len(rater1)
        expected = 0.0

        # For continuous data, approximate with histogram
        bins = np.linspace(0, 1, 11)
        hist1, _ = np.histogram(rater1, bins=bins, density=True)
        hist2, _ = np.histogram(rater2, bins=bins, density=True)

        # Normalize
        hist1 = hist1 / hist1.sum()
        hist2 = hist2 / hist2.sum()

        # Calculate expected agreement
        for i in range(len(hist1)):
            for j in range(len(hist2)):
                weight = self._get_weight(
                    bins[i], bins[j], weights
                )
                expected += hist1[i] * hist2[j] * weight

        return expected

    def _get_weight(self, value1: float, value2: float, weights: str) -> float:
        """Get weight for a pair of values based on weighting scheme."""
        diff = abs(value1 - value2)

        if weights == "linear":
            # Linear weight: 1 - |diff|
            return 1.0 - diff
        elif weights == "quadratic":
            # Quadratic weight: 1 - diff^2
            return 1.0 - (diff ** 2)
        else:
            # Binary: 1 if same, 0 if different
            return 1.0 if diff < 0.01 else 0.0


    def calculate_inter_rater_agreement(
        self,
        review_items: List[ReviewItem],
        metric_name: str
    ) -> Dict[str, Any]:
        """
        Calculate inter-rater agreement between multiple human reviewers.

        This method analyzes agreement between multiple human reviewers who
        scored the same items. It calculates both Krippendorff's alpha (for
        multiple raters) and Cohen's kappa (for pairwise comparisons).

        Args:
            review_items: List of review items with human scores
            metric_name: Name of the metric to analyze

        Returns:
            Dictionary containing:
                - krippendorff_alpha: Agreement across all raters (0-1)
                - pairwise_kappa: Dict of Cohen's kappa for each rater pair
                - n_items: Number of items analyzed
                - n_raters: Number of unique raters
                - mean_score: Mean score across all raters
                - std_score: Standard deviation across all raters

        Raises:
            InsufficientDataError: If insufficient data for calculation
            ValueError: If no valid scores found

        Examples:
            >>> # Multiple reviewers scored the same items
            >>> review_items = [
            ...     ReviewItem(
            ...         item_id=uuid4(),
            ...         item_type=ItemType.LLM_EVALUATION,
            ...         content={"question": "Q1"},
            ...         priority=Priority.MEDIUM,
            ...         confidence_score=0.7,
            ...         reviewer_id="reviewer_1",
            ...         human_scores={"answer_relevance": 0.8}
            ...     ),
            ...     ReviewItem(
            ...         item_id=uuid4(),
            ...         item_type=ItemType.LLM_EVALUATION,
            ...         content={"question": "Q1"},
            ...         priority=Priority.MEDIUM,
            ...         confidence_score=0.7,
            ...         reviewer_id="reviewer_2",
            ...         human_scores={"answer_relevance": 0.85}
            ...     )
            ... ]
            >>> agreement = calculator.calculate_inter_rater_agreement(
            ...     review_items, "answer_relevance"
            ... )
            >>> print(f"Alpha: {agreement['krippendorff_alpha']:.3f}")
        """
        # Group items by content (same item reviewed by multiple raters)
        item_groups = self._group_items_by_content(review_items, metric_name)

        if not item_groups:
            raise ValueError(
                f"No valid scores found for metric '{metric_name}'"
            )

        # Build ratings matrix: rows = items, columns = raters
        all_reviewers = set()
        for scores_dict in item_groups.values():
            all_reviewers.update(scores_dict.keys())

        reviewer_list = sorted(all_reviewers)
        n_raters = len(reviewer_list)
        n_items = len(item_groups)

        if n_items < self.min_samples:
            raise InsufficientDataError(
                f"Insufficient items for inter-rater agreement: "
                f"need at least {self.min_samples}, got {n_items}"
            )

        if n_raters < 2:
            raise InsufficientDataError(
                f"Need at least 2 raters for inter-rater agreement, got {n_raters}"
            )

        # Create ratings matrix
        ratings = np.full((n_items, n_raters), np.nan)

        for item_idx, (item_id, scores_dict) in enumerate(item_groups.items()):
            for reviewer_idx, reviewer_id in enumerate(reviewer_list):
                if reviewer_id in scores_dict:
                    ratings[item_idx, reviewer_idx] = scores_dict[reviewer_id]

        # Calculate Krippendorff's alpha
        alpha = self.calculate_krippendorff_alpha(ratings, metric="interval")

        # Calculate pairwise Cohen's kappa
        pairwise_kappa = {}

        for i in range(n_raters):
            for j in range(i + 1, n_raters):
                # Get scores for this pair (excluding missing values)
                mask = ~(np.isnan(ratings[:, i]) | np.isnan(ratings[:, j]))

                if mask.sum() >= self.min_samples:
                    rater1_scores = ratings[mask, i].tolist()
                    rater2_scores = ratings[mask, j].tolist()

                    try:
                        kappa = self.calculate_cohen_kappa(
                            rater1_scores, rater2_scores, weights="quadratic"
                        )
                        pair_key = f"{reviewer_list[i]}_vs_{reviewer_list[j]}"
                        pairwise_kappa[pair_key] = kappa
                    except (InsufficientDataError, ValueError) as e:
                        logger.warning(
                            f"Could not calculate kappa for pair "
                            f"{reviewer_list[i]}, {reviewer_list[j]}: {e}"
                        )

        # Calculate summary statistics
        all_scores = ratings[~np.isnan(ratings)]

        result = {
            "krippendorff_alpha": float(alpha),
            "pairwise_kappa": pairwise_kappa,
            "n_items": n_items,
            "n_raters": n_raters,
            "mean_score": float(np.mean(all_scores)),
            "std_score": float(np.std(all_scores)),
            "min_score": float(np.min(all_scores)),
            "max_score": float(np.max(all_scores))
        }

        logger.info(
            f"Inter-rater agreement for {metric_name}: "
            f"alpha={alpha:.3f}, n_items={n_items}, n_raters={n_raters}"
        )

        return result

    def _group_items_by_content(
        self,
        review_items: List[ReviewItem],
        metric_name: str
    ) -> Dict[str, Dict[str, float]]:
        """
        Group review items by content to identify same items reviewed by multiple raters.

        Returns:
            Dict mapping item identifier to dict of {reviewer_id: score}
        """
        item_groups = {}

        for item in review_items:
            # Skip items without human scores for this metric
            if metric_name not in item.human_scores:
                continue

            # Use item_id or content hash as identifier
            # For items reviewed by multiple people, they should have same content
            item_key = str(item.item_id)

            # Check if this is a duplicate review (same content, different reviewer)
            # Use content hash for grouping
            content_str = str(sorted(item.content.items()))

            # Find existing group with same content
            found_group = None
            for existing_key, scores_dict in item_groups.items():
                # Check if content matches (simple heuristic)
                if content_str in existing_key or existing_key in content_str:
                    found_group = existing_key
                    break

            if found_group:
                item_key = found_group
            else:
                item_key = content_str

            if item_key not in item_groups:
                item_groups[item_key] = {}

            # Add this reviewer's score
            reviewer_id = item.reviewer_id or "unknown"
            item_groups[item_key][reviewer_id] = item.human_scores[metric_name]

        # Filter to only items with multiple reviewers
        multi_reviewer_items = {
            k: v for k, v in item_groups.items() if len(v) >= 2
        }

        return multi_reviewer_items


    def calculate_llm_human_correlation(
        self,
        llm_scores: List[float],
        human_scores: List[float]
    ) -> Dict[str, Any]:
        """
        Calculate correlation between LLM judge and human reviewer scores.

        This method computes multiple correlation metrics to measure how well
        LLM scores align with human judgments. It includes both parametric
        (Pearson) and non-parametric (Spearman) correlations, plus error metrics.

        Args:
            llm_scores: Scores from LLM judge
            human_scores: Scores from human reviewers (must be same length)

        Returns:
            Dictionary containing:
                - pearson_correlation: Pearson correlation coefficient (-1 to 1)
                - pearson_p_value: Statistical significance of Pearson correlation
                - spearman_correlation: Spearman rank correlation (-1 to 1)
                - spearman_p_value: Statistical significance of Spearman correlation
                - mean_absolute_error: Mean absolute difference between scores
                - root_mean_squared_error: RMSE between scores
                - agreement_rate: Percentage within disagreement threshold
                - mean_delta: Mean signed difference (LLM - human)
                - max_delta: Maximum absolute difference
                - std_delta: Standard deviation of differences
                - n_samples: Number of score pairs

        Raises:
            InsufficientDataError: If fewer than min_samples
            ValueError: If score lists have different lengths

        Examples:
            >>> # High correlation
            >>> llm_scores = [0.8, 0.7, 0.9, 0.6, 0.85]
            >>> human_scores = [0.85, 0.75, 0.88, 0.65, 0.82]
            >>> correlation = calculator.calculate_llm_human_correlation(
            ...     llm_scores, human_scores
            ... )
            >>> print(f"Pearson: {correlation['pearson_correlation']:.3f}")
            >>> print(f"MAE: {correlation['mean_absolute_error']:.3f}")

            >>> # Check if correlation is significant
            >>> if correlation['pearson_p_value'] < 0.05:
            ...     print("Correlation is statistically significant")
        """
        if len(llm_scores) != len(human_scores):
            raise ValueError(
                f"Score lists must have same length: "
                f"got {len(llm_scores)} and {len(human_scores)}"
            )

        if len(llm_scores) < self.min_samples:
            raise InsufficientDataError(
                f"Insufficient samples for correlation: "
                f"need at least {self.min_samples}, got {len(llm_scores)}"
            )

        # Convert to numpy arrays
        llm_arr = np.array(llm_scores)
        human_arr = np.array(human_scores)

        # Calculate Pearson correlation
        pearson_corr, pearson_p = stats.pearsonr(llm_arr, human_arr)

        # Calculate Spearman correlation
        spearman_corr, spearman_p = stats.spearmanr(llm_arr, human_arr)

        # Calculate error metrics
        score_diffs = np.abs(llm_arr - human_arr)
        mae = np.mean(score_diffs)
        rmse = np.sqrt(np.mean((llm_arr - human_arr) ** 2))

        # Calculate agreement rate (within threshold)
        agreements = score_diffs <= self.disagreement_threshold
        agreement_rate = np.mean(agreements)

        # Calculate signed differences
        signed_diffs = llm_arr - human_arr
        mean_delta = np.mean(signed_diffs)
        max_delta = np.max(score_diffs)
        std_delta = np.std(signed_diffs)

        result = {
            "pearson_correlation": float(pearson_corr),
            "pearson_p_value": float(pearson_p),
            "spearman_correlation": float(spearman_corr),
            "spearman_p_value": float(spearman_p),
            "mean_absolute_error": float(mae),
            "root_mean_squared_error": float(rmse),
            "agreement_rate": float(agreement_rate),
            "mean_delta": float(mean_delta),
            "max_delta": float(max_delta),
            "std_delta": float(std_delta),
            "n_samples": len(llm_scores)
        }

        logger.info(
            f"LLM-human correlation: "
            f"pearson={pearson_corr:.3f} (p={pearson_p:.4f}), "
            f"spearman={spearman_corr:.3f} (p={spearman_p:.4f}), "
            f"mae={mae:.3f}"
        )

        return result


    def identify_disputed_items(
        self,
        review_items: List[ReviewItem],
        metric_name: str,
        threshold: Optional[float] = None
    ) -> List[Dict[str, Any]]:
        """
        Identify items with low agreement (disputed items).

        This method finds review items where reviewers significantly disagree,
        either between multiple human reviewers or between LLM and human scores.
        Disputed items may need additional review or dispute resolution.

        Args:
            review_items: List of review items to analyze
            metric_name: Name of the metric to check for disputes
            threshold: Disagreement threshold (uses instance default if None)

        Returns:
            List of disputed items, each containing:
                - item_id: UUID of the disputed item
                - item_type: Type of review item
                - content: Item content
                - scores: Dict of all scores for this item
                - disagreement_level: Maximum score difference
                - mean_score: Mean of all scores
                - std_score: Standard deviation of scores
                - n_reviewers: Number of reviewers

        Examples:
            >>> # Find disputed items
            >>> disputed = calculator.identify_disputed_items(
            ...     review_items, "answer_relevance"
            ... )
            >>> 
            >>> for item in disputed:
            ...     print(f"Item {item['item_id']}: "
            ...           f"disagreement={item['disagreement_level']:.3f}")
            ...     print(f"  Scores: {item['scores']}")

            >>> # Use custom threshold
            >>> disputed = calculator.identify_disputed_items(
            ...     review_items, "answer_relevance", threshold=0.15
            ... )
        """
        if threshold is None:
            threshold = self.disagreement_threshold

        disputed_items = []

        # Group items by content to find items with multiple reviews
        item_groups = self._group_items_by_content(review_items, metric_name)

        for item_key, scores_dict in item_groups.items():
            scores = list(scores_dict.values())

            if len(scores) < 2:
                continue

            # Calculate disagreement level (max difference)
            min_score = min(scores)
            max_score = max(scores)
            disagreement = max_score - min_score

            # Check if this is a disputed item
            if disagreement > threshold:
                # Find the original review item for metadata
                matching_item = None
                for item in review_items:
                    if metric_name in item.human_scores:
                        content_str = str(sorted(item.content.items()))
                        if content_str in item_key or item_key in content_str:
                            matching_item = item
                            break

                if matching_item:
                    disputed_items.append({
                        "item_id": str(matching_item.item_id),
                        "item_type": matching_item.item_type.value,
                        "content": matching_item.content,
                        "scores": scores_dict,
                        "disagreement_level": float(disagreement),
                        "mean_score": float(np.mean(scores)),
                        "std_score": float(np.std(scores)),
                        "n_reviewers": len(scores),
                        "threshold": threshold
                    })

        # Also check for LLM-human disagreements
        for item in review_items:
            if metric_name not in item.human_scores:
                continue

            # Check if item has LLM score
            llm_score = item.content.get("llm_scores", {}).get(metric_name)
            if llm_score is None:
                continue

            human_score = item.human_scores[metric_name]
            disagreement = abs(llm_score - human_score)

            if disagreement > threshold:
                # Check if not already in disputed list
                item_id_str = str(item.item_id)
                if not any(d["item_id"] == item_id_str for d in disputed_items):
                    disputed_items.append({
                        "item_id": item_id_str,
                        "item_type": item.item_type.value,
                        "content": item.content,
                        "scores": {
                            "llm": llm_score,
                            item.reviewer_id or "human": human_score
                        },
                        "disagreement_level": float(disagreement),
                        "mean_score": float((llm_score + human_score) / 2),
                        "std_score": float(np.std([llm_score, human_score])),
                        "n_reviewers": 2,
                        "threshold": threshold,
                        "llm_human_dispute": True
                    })

        # Sort by disagreement level (highest first)
        disputed_items.sort(key=lambda x: x["disagreement_level"], reverse=True)

        logger.info(
            f"Identified {len(disputed_items)} disputed items for {metric_name} "
            f"(threshold={threshold})"
        )

        return disputed_items


    def generate_agreement_report(
        self,
        review_items: List[ReviewItem],
        metric_name: str,
        include_llm_correlation: bool = True
    ) -> Dict[str, Any]:
        """
        Generate comprehensive agreement report.

        This method produces a complete statistical summary of agreement
        across all reviewers and between LLM and human scores. It combines
        inter-rater agreement, LLM-human correlation, and disputed item
        identification into a single comprehensive report.

        Args:
            review_items: List of review items to analyze
            metric_name: Name of the metric to analyze
            include_llm_correlation: Whether to include LLM-human correlation
                (requires items with LLM scores)

        Returns:
            Dictionary containing:
                - metric_name: Name of the analyzed metric
                - inter_rater_agreement: Inter-rater agreement metrics
                    - krippendorff_alpha: Agreement across all raters
                    - pairwise_kappa: Cohen's kappa for each pair
                    - n_items: Number of items
                    - n_raters: Number of raters
                    - mean_score: Mean score
                    - std_score: Standard deviation
                - llm_human_correlation: LLM-human correlation metrics (if enabled)
                    - pearson_correlation: Pearson coefficient
                    - spearman_correlation: Spearman coefficient
                    - mean_absolute_error: MAE
                    - agreement_rate: Percentage within threshold
                - disputed_items: List of items with low agreement
                - summary: High-level summary statistics
                    - total_items: Total items analyzed
                    - disputed_count: Number of disputed items
                    - dispute_rate: Percentage of disputed items
                    - overall_agreement_quality: "excellent", "good", "fair", "poor"

        Raises:
            InsufficientDataError: If insufficient data for analysis
            ValueError: If no valid scores found

        Examples:
            >>> # Generate full report
            >>> report = calculator.generate_agreement_report(
            ...     review_items, "answer_relevance"
            ... )
            >>> 
            >>> print(f"Inter-rater alpha: "
            ...       f"{report['inter_rater_agreement']['krippendorff_alpha']:.3f}")
            >>> print(f"LLM-human correlation: "
            ...       f"{report['llm_human_correlation']['pearson_correlation']:.3f}")
            >>> print(f"Disputed items: {report['summary']['disputed_count']}")
            >>> print(f"Overall quality: {report['summary']['overall_agreement_quality']}")

            >>> # Report without LLM correlation
            >>> report = calculator.generate_agreement_report(
            ...     review_items, "answer_relevance", include_llm_correlation=False
            ... )
        """
        report = {
            "metric_name": metric_name,
            "timestamp": np.datetime64('now').astype(str)
        }

        # Calculate inter-rater agreement
        try:
            inter_rater = self.calculate_inter_rater_agreement(
                review_items, metric_name
            )
            report["inter_rater_agreement"] = inter_rater
        except (InsufficientDataError, ValueError) as e:
            logger.warning(f"Could not calculate inter-rater agreement: {e}")
            report["inter_rater_agreement"] = None

        # Calculate LLM-human correlation if requested
        if include_llm_correlation:
            try:
                # Extract LLM and human scores
                llm_scores = []
                human_scores = []

                for item in review_items:
                    if metric_name not in item.human_scores:
                        continue

                    llm_score = item.content.get("llm_scores", {}).get(metric_name)
                    if llm_score is None:
                        continue

                    llm_scores.append(llm_score)
                    human_scores.append(item.human_scores[metric_name])

                if len(llm_scores) >= self.min_samples:
                    llm_correlation = self.calculate_llm_human_correlation(
                        llm_scores, human_scores
                    )
                    report["llm_human_correlation"] = llm_correlation
                else:
                    logger.warning(
                        f"Insufficient LLM-human pairs for correlation: "
                        f"got {len(llm_scores)}"
                    )
                    report["llm_human_correlation"] = None
            except (InsufficientDataError, ValueError) as e:
                logger.warning(f"Could not calculate LLM-human correlation: {e}")
                report["llm_human_correlation"] = None
        else:
            report["llm_human_correlation"] = None

        # Identify disputed items
        try:
            disputed = self.identify_disputed_items(review_items, metric_name)
            report["disputed_items"] = disputed
        except Exception as e:
            logger.warning(f"Could not identify disputed items: {e}")
            report["disputed_items"] = []

        # Generate summary
        total_items = len([
            item for item in review_items
            if metric_name in item.human_scores
        ])
        disputed_count = len(report["disputed_items"])
        dispute_rate = disputed_count / total_items if total_items > 0 else 0.0

        # Determine overall agreement quality
        quality = self._assess_agreement_quality(report)

        report["summary"] = {
            "total_items": total_items,
            "disputed_count": disputed_count,
            "dispute_rate": float(dispute_rate),
            "overall_agreement_quality": quality,
            "disagreement_threshold": self.disagreement_threshold
        }

        logger.info(
            f"Generated agreement report for {metric_name}: "
            f"quality={quality}, disputed={disputed_count}/{total_items}"
        )

        return report

    def _assess_agreement_quality(self, report: Dict[str, Any]) -> str:
        """
        Assess overall agreement quality based on multiple metrics.

        Returns: "excellent", "good", "fair", or "poor"
        """
        # Check inter-rater agreement
        inter_rater = report.get("inter_rater_agreement")
        if inter_rater:
            alpha = inter_rater.get("krippendorff_alpha", 0.0)
        else:
            alpha = None

        # Check LLM-human correlation
        llm_human = report.get("llm_human_correlation")
        if llm_human:
            pearson = llm_human.get("pearson_correlation", 0.0)
        else:
            pearson = None

        # Check dispute rate
        summary = report.get("summary", {})
        dispute_rate = summary.get("dispute_rate", 1.0)

        # Assess quality based on available metrics
        scores = []

        if alpha is not None:
            if alpha >= 0.8:
                scores.append(4)  # Excellent
            elif alpha >= 0.67:
                scores.append(3)  # Good
            elif alpha >= 0.5:
                scores.append(2)  # Fair
            else:
                scores.append(1)  # Poor

        if pearson is not None:
            if pearson >= 0.8:
                scores.append(4)
            elif pearson >= 0.6:
                scores.append(3)
            elif pearson >= 0.4:
                scores.append(2)
            else:
                scores.append(1)

        # Dispute rate (inverse scoring)
        if dispute_rate <= 0.1:
            scores.append(4)
        elif dispute_rate <= 0.2:
            scores.append(3)
        elif dispute_rate <= 0.3:
            scores.append(2)
        else:
            scores.append(1)

        if not scores:
            return "unknown"

        # Average score
        avg_score = np.mean(scores)

        if avg_score >= 3.5:
            return "excellent"
        elif avg_score >= 2.5:
            return "good"
        elif avg_score >= 1.5:
            return "fair"
        else:
            return "poor"

calculate_cohen_kappa(rater1_scores, rater2_scores, weights=None)

Calculate Cohen's kappa for agreement between two raters.

Cohen's kappa measures agreement between two raters, correcting for chance agreement. It's particularly useful for binary or categorical ratings but can also be used with continuous scores.

Kappa ranges from -1 to 1: - 1.0: Perfect agreement - 0.8-1.0: Almost perfect agreement - 0.6-0.8: Substantial agreement - 0.4-0.6: Moderate agreement - 0.2-0.4: Fair agreement - 0.0-0.2: Slight agreement - < 0: Less than chance agreement

Parameters:

Name	Type	Description	Default
rater1_scores	List[float]	Scores from first rater	required
rater2_scores	List[float]	Scores from second rater (must be same length)	required
weights	Optional[str]	Weighting scheme for disagreements: - None: Unweighted (default) - "linear": Linear weights (for ordinal data) - "quadratic": Quadratic weights (for interval data)	None

Returns:

Type	Description
float	Cohen's kappa coefficient (-1 to 1)

Raises:

Type	Description
InsufficientDataError	If fewer than min_samples
ValueError	If score lists have different lengths

Examples:

>>> # Perfect agreement
>>> rater1 = [0.8, 0.5, 0.9, 0.6]
>>> rater2 = [0.8, 0.5, 0.9, 0.6]
>>> kappa = calculator.calculate_cohen_kappa(rater1, rater2)
>>> print(f"Kappa: {kappa:.3f}")  # 1.0

>>> # Moderate agreement
>>> rater1 = [0.8, 0.5, 0.9, 0.6]
>>> rater2 = [0.85, 0.55, 0.88, 0.65]
>>> kappa = calculator.calculate_cohen_kappa(rater1, rater2)

>>> # With quadratic weights (for continuous scores)
>>> kappa = calculator.calculate_cohen_kappa(
...     rater1, rater2, weights="quadratic"
... )

Source code in uaef/hitl/agreement.py

def calculate_cohen_kappa(
    self,
    rater1_scores: List[float],
    rater2_scores: List[float],
    weights: Optional[str] = None
) -> float:
    """
    Calculate Cohen's kappa for agreement between two raters.

    Cohen's kappa measures agreement between two raters, correcting for
    chance agreement. It's particularly useful for binary or categorical
    ratings but can also be used with continuous scores.

    Kappa ranges from -1 to 1:
    - 1.0: Perfect agreement
    - 0.8-1.0: Almost perfect agreement
    - 0.6-0.8: Substantial agreement
    - 0.4-0.6: Moderate agreement
    - 0.2-0.4: Fair agreement
    - 0.0-0.2: Slight agreement
    - < 0: Less than chance agreement

    Args:
        rater1_scores: Scores from first rater
        rater2_scores: Scores from second rater (must be same length)
        weights: Weighting scheme for disagreements:
            - None: Unweighted (default)
            - "linear": Linear weights (for ordinal data)
            - "quadratic": Quadratic weights (for interval data)

    Returns:
        Cohen's kappa coefficient (-1 to 1)

    Raises:
        InsufficientDataError: If fewer than min_samples
        ValueError: If score lists have different lengths

    Examples:
        >>> # Perfect agreement
        >>> rater1 = [0.8, 0.5, 0.9, 0.6]
        >>> rater2 = [0.8, 0.5, 0.9, 0.6]
        >>> kappa = calculator.calculate_cohen_kappa(rater1, rater2)
        >>> print(f"Kappa: {kappa:.3f}")  # 1.0

        >>> # Moderate agreement
        >>> rater1 = [0.8, 0.5, 0.9, 0.6]
        >>> rater2 = [0.85, 0.55, 0.88, 0.65]
        >>> kappa = calculator.calculate_cohen_kappa(rater1, rater2)

        >>> # With quadratic weights (for continuous scores)
        >>> kappa = calculator.calculate_cohen_kappa(
        ...     rater1, rater2, weights="quadratic"
        ... )
    """
    if len(rater1_scores) != len(rater2_scores):
        raise ValueError(
            f"Rater score lists must have same length: "
            f"got {len(rater1_scores)} and {len(rater2_scores)}"
        )

    if len(rater1_scores) < self.min_samples:
        raise InsufficientDataError(
            f"Insufficient samples for Cohen's kappa: "
            f"need at least {self.min_samples}, got {len(rater1_scores)}"
        )

    # Convert to numpy arrays
    rater1 = np.array(rater1_scores)
    rater2 = np.array(rater2_scores)

    # For continuous scores, we need to discretize or use weighted kappa
    # We'll bin the scores into categories for standard kappa
    if weights is None:
        # Discretize continuous scores into bins
        n_bins = min(10, len(rater1_scores))
        rater1_binned = np.digitize(rater1, np.linspace(0, 1, n_bins))
        rater2_binned = np.digitize(rater2, np.linspace(0, 1, n_bins))

        # Calculate observed agreement
        observed_agreement = np.mean(rater1_binned == rater2_binned)

        # Calculate expected agreement
        unique_bins = np.unique(np.concatenate([rater1_binned, rater2_binned]))
        expected_agreement = 0.0

        for bin_val in unique_bins:
            p1 = np.mean(rater1_binned == bin_val)
            p2 = np.mean(rater2_binned == bin_val)
            expected_agreement += p1 * p2

    else:
        # Weighted kappa for continuous scores
        observed_agreement = self._calculate_weighted_agreement(
            rater1, rater2, weights
        )
        expected_agreement = self._calculate_expected_weighted_agreement(
            rater1, rater2, weights
        )

    # Avoid division by zero
    if expected_agreement >= 1.0:
        return 1.0

    # Cohen's kappa = (observed - expected) / (1 - expected)
    kappa = (observed_agreement - expected_agreement) / (1.0 - expected_agreement)

    logger.debug(
        f"Cohen's kappa: {kappa:.3f} "
        f"(observed={observed_agreement:.3f}, "
        f"expected={expected_agreement:.3f})"
    )

    return float(kappa)

calculate_inter_rater_agreement(review_items, metric_name)

Calculate inter-rater agreement between multiple human reviewers.

This method analyzes agreement between multiple human reviewers who scored the same items. It calculates both Krippendorff's alpha (for multiple raters) and Cohen's kappa (for pairwise comparisons).

Parameters:

Name	Type	Description	Default
review_items	List[ReviewItem]	List of review items with human scores	required
metric_name	str	Name of the metric to analyze	required

Returns:

Type	Description
Dict[str, Any]	Dictionary containing: - krippendorff_alpha: Agreement across all raters (0-1) - pairwise_kappa: Dict of Cohen's kappa for each rater pair - n_items: Number of items analyzed - n_raters: Number of unique raters - mean_score: Mean score across all raters - std_score: Standard deviation across all raters

Raises:

Type	Description
InsufficientDataError	If insufficient data for calculation
ValueError	If no valid scores found

Examples:

>>> # Multiple reviewers scored the same items
>>> review_items = [
...     ReviewItem(
...         item_id=uuid4(),
...         item_type=ItemType.LLM_EVALUATION,
...         content={"question": "Q1"},
...         priority=Priority.MEDIUM,
...         confidence_score=0.7,
...         reviewer_id="reviewer_1",
...         human_scores={"answer_relevance": 0.8}
...     ),
...     ReviewItem(
...         item_id=uuid4(),
...         item_type=ItemType.LLM_EVALUATION,
...         content={"question": "Q1"},
...         priority=Priority.MEDIUM,
...         confidence_score=0.7,
...         reviewer_id="reviewer_2",
...         human_scores={"answer_relevance": 0.85}
...     )
... ]
>>> agreement = calculator.calculate_inter_rater_agreement(
...     review_items, "answer_relevance"
... )
>>> print(f"Alpha: {agreement['krippendorff_alpha']:.3f}")

Source code in uaef/hitl/agreement.py

def calculate_inter_rater_agreement(
    self,
    review_items: List[ReviewItem],
    metric_name: str
) -> Dict[str, Any]:
    """
    Calculate inter-rater agreement between multiple human reviewers.

    This method analyzes agreement between multiple human reviewers who
    scored the same items. It calculates both Krippendorff's alpha (for
    multiple raters) and Cohen's kappa (for pairwise comparisons).

    Args:
        review_items: List of review items with human scores
        metric_name: Name of the metric to analyze

    Returns:
        Dictionary containing:
            - krippendorff_alpha: Agreement across all raters (0-1)
            - pairwise_kappa: Dict of Cohen's kappa for each rater pair
            - n_items: Number of items analyzed
            - n_raters: Number of unique raters
            - mean_score: Mean score across all raters
            - std_score: Standard deviation across all raters

    Raises:
        InsufficientDataError: If insufficient data for calculation
        ValueError: If no valid scores found

    Examples:
        >>> # Multiple reviewers scored the same items
        >>> review_items = [
        ...     ReviewItem(
        ...         item_id=uuid4(),
        ...         item_type=ItemType.LLM_EVALUATION,
        ...         content={"question": "Q1"},
        ...         priority=Priority.MEDIUM,
        ...         confidence_score=0.7,
        ...         reviewer_id="reviewer_1",
        ...         human_scores={"answer_relevance": 0.8}
        ...     ),
        ...     ReviewItem(
        ...         item_id=uuid4(),
        ...         item_type=ItemType.LLM_EVALUATION,
        ...         content={"question": "Q1"},
        ...         priority=Priority.MEDIUM,
        ...         confidence_score=0.7,
        ...         reviewer_id="reviewer_2",
        ...         human_scores={"answer_relevance": 0.85}
        ...     )
        ... ]
        >>> agreement = calculator.calculate_inter_rater_agreement(
        ...     review_items, "answer_relevance"
        ... )
        >>> print(f"Alpha: {agreement['krippendorff_alpha']:.3f}")
    """
    # Group items by content (same item reviewed by multiple raters)
    item_groups = self._group_items_by_content(review_items, metric_name)

    if not item_groups:
        raise ValueError(
            f"No valid scores found for metric '{metric_name}'"
        )

    # Build ratings matrix: rows = items, columns = raters
    all_reviewers = set()
    for scores_dict in item_groups.values():
        all_reviewers.update(scores_dict.keys())

    reviewer_list = sorted(all_reviewers)
    n_raters = len(reviewer_list)
    n_items = len(item_groups)

    if n_items < self.min_samples:
        raise InsufficientDataError(
            f"Insufficient items for inter-rater agreement: "
            f"need at least {self.min_samples}, got {n_items}"
        )

    if n_raters < 2:
        raise InsufficientDataError(
            f"Need at least 2 raters for inter-rater agreement, got {n_raters}"
        )

    # Create ratings matrix
    ratings = np.full((n_items, n_raters), np.nan)

    for item_idx, (item_id, scores_dict) in enumerate(item_groups.items()):
        for reviewer_idx, reviewer_id in enumerate(reviewer_list):
            if reviewer_id in scores_dict:
                ratings[item_idx, reviewer_idx] = scores_dict[reviewer_id]

    # Calculate Krippendorff's alpha
    alpha = self.calculate_krippendorff_alpha(ratings, metric="interval")

    # Calculate pairwise Cohen's kappa
    pairwise_kappa = {}

    for i in range(n_raters):
        for j in range(i + 1, n_raters):
            # Get scores for this pair (excluding missing values)
            mask = ~(np.isnan(ratings[:, i]) | np.isnan(ratings[:, j]))

            if mask.sum() >= self.min_samples:
                rater1_scores = ratings[mask, i].tolist()
                rater2_scores = ratings[mask, j].tolist()

                try:
                    kappa = self.calculate_cohen_kappa(
                        rater1_scores, rater2_scores, weights="quadratic"
                    )
                    pair_key = f"{reviewer_list[i]}_vs_{reviewer_list[j]}"
                    pairwise_kappa[pair_key] = kappa
                except (InsufficientDataError, ValueError) as e:
                    logger.warning(
                        f"Could not calculate kappa for pair "
                        f"{reviewer_list[i]}, {reviewer_list[j]}: {e}"
                    )

    # Calculate summary statistics
    all_scores = ratings[~np.isnan(ratings)]

    result = {
        "krippendorff_alpha": float(alpha),
        "pairwise_kappa": pairwise_kappa,
        "n_items": n_items,
        "n_raters": n_raters,
        "mean_score": float(np.mean(all_scores)),
        "std_score": float(np.std(all_scores)),
        "min_score": float(np.min(all_scores)),
        "max_score": float(np.max(all_scores))
    }

    logger.info(
        f"Inter-rater agreement for {metric_name}: "
        f"alpha={alpha:.3f}, n_items={n_items}, n_raters={n_raters}"
    )

    return result

calculate_krippendorff_alpha(ratings, metric='interval')

Calculate Krippendorff's alpha for inter-rater agreement.

Krippendorff's alpha is a reliability coefficient that measures agreement among multiple raters. It handles missing data, works with any number of raters, and supports different measurement levels (nominal, ordinal, interval).

Alpha ranges from 0 to 1: - 1.0: Perfect agreement - 0.8+: Good agreement (generally acceptable) - 0.67-0.8: Tentative agreement - < 0.67: Low agreement (not reliable)

Parameters:

Name	Type	Description	Default
ratings	ndarray	2D array of shape (n_items, n_raters) where each row is an item and each column is a rater. Use np.nan for missing ratings.	required
metric	str	Distance metric to use. Options: - "interval": For continuous scores (default) - "nominal": For categorical data - "ordinal": For ranked data	'interval'

Returns:

Type	Description
float	Krippendorff's alpha coefficient (0-1)

Raises:

Type	Description
InsufficientDataError	If fewer than min_samples items
ValueError	If ratings array is invalid

Examples:

>>> # Perfect agreement
>>> ratings = np.array([[0.8, 0.8, 0.8], [0.5, 0.5, 0.5]])
>>> alpha = calculator.calculate_krippendorff_alpha(ratings)
>>> print(f"Alpha: {alpha:.3f}")  # 1.0

>>> # With missing data
>>> ratings = np.array([[0.8, 0.85, np.nan], [0.5, np.nan, 0.55]])
>>> alpha = calculator.calculate_krippendorff_alpha(ratings)

>>> # Binary data (nominal)
>>> ratings = np.array([[1, 1, 0], [0, 0, 0], [1, 1, 1]])
>>> alpha = calculator.calculate_krippendorff_alpha(
...     ratings, metric="nominal"
... )

Source code in uaef/hitl/agreement.py

def calculate_krippendorff_alpha(
    self,
    ratings: np.ndarray,
    metric: str = "interval"
) -> float:
    """
    Calculate Krippendorff's alpha for inter-rater agreement.

    Krippendorff's alpha is a reliability coefficient that measures agreement
    among multiple raters. It handles missing data, works with any number of
    raters, and supports different measurement levels (nominal, ordinal, interval).

    Alpha ranges from 0 to 1:
    - 1.0: Perfect agreement
    - 0.8+: Good agreement (generally acceptable)
    - 0.67-0.8: Tentative agreement
    - < 0.67: Low agreement (not reliable)

    Args:
        ratings: 2D array of shape (n_items, n_raters) where each row is an item
            and each column is a rater. Use np.nan for missing ratings.
        metric: Distance metric to use. Options:
            - "interval": For continuous scores (default)
            - "nominal": For categorical data
            - "ordinal": For ranked data

    Returns:
        Krippendorff's alpha coefficient (0-1)

    Raises:
        InsufficientDataError: If fewer than min_samples items
        ValueError: If ratings array is invalid

    Examples:
        >>> # Perfect agreement
        >>> ratings = np.array([[0.8, 0.8, 0.8], [0.5, 0.5, 0.5]])
        >>> alpha = calculator.calculate_krippendorff_alpha(ratings)
        >>> print(f"Alpha: {alpha:.3f}")  # 1.0

        >>> # With missing data
        >>> ratings = np.array([[0.8, 0.85, np.nan], [0.5, np.nan, 0.55]])
        >>> alpha = calculator.calculate_krippendorff_alpha(ratings)

        >>> # Binary data (nominal)
        >>> ratings = np.array([[1, 1, 0], [0, 0, 0], [1, 1, 1]])
        >>> alpha = calculator.calculate_krippendorff_alpha(
        ...     ratings, metric="nominal"
        ... )
    """
    if ratings.ndim != 2:
        raise ValueError(
            f"ratings must be 2D array, got shape {ratings.shape}"
        )

    n_items, n_raters = ratings.shape

    if n_items < self.min_samples:
        raise InsufficientDataError(
            f"Insufficient items for Krippendorff's alpha: "
            f"need at least {self.min_samples}, got {n_items}"
        )

    if n_raters < 2:
        raise ValueError(
            f"Need at least 2 raters, got {n_raters}"
        )

    # Calculate observed disagreement
    observed_disagreement = self._calculate_disagreement(ratings, metric)

    # Calculate expected disagreement
    expected_disagreement = self._calculate_expected_disagreement(ratings, metric)

    # Avoid division by zero
    if expected_disagreement == 0:
        # If expected disagreement is 0, all values are identical
        return 1.0

    # Krippendorff's alpha = 1 - (observed / expected)
    alpha = 1.0 - (observed_disagreement / expected_disagreement)

    logger.debug(
        f"Krippendorff's alpha: {alpha:.3f} "
        f"(observed={observed_disagreement:.3f}, "
        f"expected={expected_disagreement:.3f})"
    )

    return float(alpha)

calculate_llm_human_correlation(llm_scores, human_scores)

Calculate correlation between LLM judge and human reviewer scores.

This method computes multiple correlation metrics to measure how well LLM scores align with human judgments. It includes both parametric (Pearson) and non-parametric (Spearman) correlations, plus error metrics.

Parameters:

Name	Type	Description	Default
llm_scores	List[float]	Scores from LLM judge	required
human_scores	List[float]	Scores from human reviewers (must be same length)	required

Returns:

Type

Description

Dict[str, Any]

Dictionary containing: - pearson_correlation: Pearson correlation coefficient (-1 to 1) - pearson_p_value: Statistical significance of Pearson correlation - spearman_correlation: Spearman rank correlation (-1 to 1) - spearman_p_value: Statistical significance of Spearman correlation - mean_absolute_error: Mean absolute difference between scores - root_mean_squared_error: RMSE between scores - agreement_rate: Percentage within disagreement threshold - mean_delta: Mean signed difference (LLM - human) - max_delta: Maximum absolute difference - std_delta: Standard deviation of differences - n_samples: Number of score pairs

Raises:

Type	Description
InsufficientDataError	If fewer than min_samples
ValueError	If score lists have different lengths

Examples:

>>> # High correlation
>>> llm_scores = [0.8, 0.7, 0.9, 0.6, 0.85]
>>> human_scores = [0.85, 0.75, 0.88, 0.65, 0.82]
>>> correlation = calculator.calculate_llm_human_correlation(
...     llm_scores, human_scores
... )
>>> print(f"Pearson: {correlation['pearson_correlation']:.3f}")
>>> print(f"MAE: {correlation['mean_absolute_error']:.3f}")

>>> # Check if correlation is significant
>>> if correlation['pearson_p_value'] < 0.05:
...     print("Correlation is statistically significant")

Source code in uaef/hitl/agreement.py

def calculate_llm_human_correlation(
    self,
    llm_scores: List[float],
    human_scores: List[float]
) -> Dict[str, Any]:
    """
    Calculate correlation between LLM judge and human reviewer scores.

    This method computes multiple correlation metrics to measure how well
    LLM scores align with human judgments. It includes both parametric
    (Pearson) and non-parametric (Spearman) correlations, plus error metrics.

    Args:
        llm_scores: Scores from LLM judge
        human_scores: Scores from human reviewers (must be same length)

    Returns:
        Dictionary containing:
            - pearson_correlation: Pearson correlation coefficient (-1 to 1)
            - pearson_p_value: Statistical significance of Pearson correlation
            - spearman_correlation: Spearman rank correlation (-1 to 1)
            - spearman_p_value: Statistical significance of Spearman correlation
            - mean_absolute_error: Mean absolute difference between scores
            - root_mean_squared_error: RMSE between scores
            - agreement_rate: Percentage within disagreement threshold
            - mean_delta: Mean signed difference (LLM - human)
            - max_delta: Maximum absolute difference
            - std_delta: Standard deviation of differences
            - n_samples: Number of score pairs

    Raises:
        InsufficientDataError: If fewer than min_samples
        ValueError: If score lists have different lengths

    Examples:
        >>> # High correlation
        >>> llm_scores = [0.8, 0.7, 0.9, 0.6, 0.85]
        >>> human_scores = [0.85, 0.75, 0.88, 0.65, 0.82]
        >>> correlation = calculator.calculate_llm_human_correlation(
        ...     llm_scores, human_scores
        ... )
        >>> print(f"Pearson: {correlation['pearson_correlation']:.3f}")
        >>> print(f"MAE: {correlation['mean_absolute_error']:.3f}")

        >>> # Check if correlation is significant
        >>> if correlation['pearson_p_value'] < 0.05:
        ...     print("Correlation is statistically significant")
    """
    if len(llm_scores) != len(human_scores):
        raise ValueError(
            f"Score lists must have same length: "
            f"got {len(llm_scores)} and {len(human_scores)}"
        )

    if len(llm_scores) < self.min_samples:
        raise InsufficientDataError(
            f"Insufficient samples for correlation: "
            f"need at least {self.min_samples}, got {len(llm_scores)}"
        )

    # Convert to numpy arrays
    llm_arr = np.array(llm_scores)
    human_arr = np.array(human_scores)

    # Calculate Pearson correlation
    pearson_corr, pearson_p = stats.pearsonr(llm_arr, human_arr)

    # Calculate Spearman correlation
    spearman_corr, spearman_p = stats.spearmanr(llm_arr, human_arr)

    # Calculate error metrics
    score_diffs = np.abs(llm_arr - human_arr)
    mae = np.mean(score_diffs)
    rmse = np.sqrt(np.mean((llm_arr - human_arr) ** 2))

    # Calculate agreement rate (within threshold)
    agreements = score_diffs <= self.disagreement_threshold
    agreement_rate = np.mean(agreements)

    # Calculate signed differences
    signed_diffs = llm_arr - human_arr
    mean_delta = np.mean(signed_diffs)
    max_delta = np.max(score_diffs)
    std_delta = np.std(signed_diffs)

    result = {
        "pearson_correlation": float(pearson_corr),
        "pearson_p_value": float(pearson_p),
        "spearman_correlation": float(spearman_corr),
        "spearman_p_value": float(spearman_p),
        "mean_absolute_error": float(mae),
        "root_mean_squared_error": float(rmse),
        "agreement_rate": float(agreement_rate),
        "mean_delta": float(mean_delta),
        "max_delta": float(max_delta),
        "std_delta": float(std_delta),
        "n_samples": len(llm_scores)
    }

    logger.info(
        f"LLM-human correlation: "
        f"pearson={pearson_corr:.3f} (p={pearson_p:.4f}), "
        f"spearman={spearman_corr:.3f} (p={spearman_p:.4f}), "
        f"mae={mae:.3f}"
    )

    return result

generate_agreement_report(review_items, metric_name, include_llm_correlation=True)

Generate comprehensive agreement report.

This method produces a complete statistical summary of agreement across all reviewers and between LLM and human scores. It combines inter-rater agreement, LLM-human correlation, and disputed item identification into a single comprehensive report.

Parameters:

Name	Type	Description	Default
review_items	List[ReviewItem]	List of review items to analyze	required
metric_name	str	Name of the metric to analyze	required
include_llm_correlation	bool	Whether to include LLM-human correlation (requires items with LLM scores)	True

Returns:

Type

Description

Dict[str, Any]

Dictionary containing: - metric_name: Name of the analyzed metric - inter_rater_agreement: Inter-rater agreement metrics - krippendorff_alpha: Agreement across all raters - pairwise_kappa: Cohen's kappa for each pair - n_items: Number of items - n_raters: Number of raters - mean_score: Mean score - std_score: Standard deviation - llm_human_correlation: LLM-human correlation metrics (if enabled) - pearson_correlation: Pearson coefficient - spearman_correlation: Spearman coefficient - mean_absolute_error: MAE - agreement_rate: Percentage within threshold - disputed_items: List of items with low agreement - summary: High-level summary statistics - total_items: Total items analyzed - disputed_count: Number of disputed items - dispute_rate: Percentage of disputed items - overall_agreement_quality: "excellent", "good", "fair", "poor"

Raises:

Type	Description
InsufficientDataError	If insufficient data for analysis
ValueError	If no valid scores found

Examples:

>>> # Generate full report
>>> report = calculator.generate_agreement_report(
...     review_items, "answer_relevance"
... )
>>> 
>>> print(f"Inter-rater alpha: "
...       f"{report['inter_rater_agreement']['krippendorff_alpha']:.3f}")
>>> print(f"LLM-human correlation: "
...       f"{report['llm_human_correlation']['pearson_correlation']:.3f}")
>>> print(f"Disputed items: {report['summary']['disputed_count']}")
>>> print(f"Overall quality: {report['summary']['overall_agreement_quality']}")

>>> # Report without LLM correlation
>>> report = calculator.generate_agreement_report(
...     review_items, "answer_relevance", include_llm_correlation=False
... )

Source code in uaef/hitl/agreement.py

def generate_agreement_report(
    self,
    review_items: List[ReviewItem],
    metric_name: str,
    include_llm_correlation: bool = True
) -> Dict[str, Any]:
    """
    Generate comprehensive agreement report.

    This method produces a complete statistical summary of agreement
    across all reviewers and between LLM and human scores. It combines
    inter-rater agreement, LLM-human correlation, and disputed item
    identification into a single comprehensive report.

    Args:
        review_items: List of review items to analyze
        metric_name: Name of the metric to analyze
        include_llm_correlation: Whether to include LLM-human correlation
            (requires items with LLM scores)

    Returns:
        Dictionary containing:
            - metric_name: Name of the analyzed metric
            - inter_rater_agreement: Inter-rater agreement metrics
                - krippendorff_alpha: Agreement across all raters
                - pairwise_kappa: Cohen's kappa for each pair
                - n_items: Number of items
                - n_raters: Number of raters
                - mean_score: Mean score
                - std_score: Standard deviation
            - llm_human_correlation: LLM-human correlation metrics (if enabled)
                - pearson_correlation: Pearson coefficient
                - spearman_correlation: Spearman coefficient
                - mean_absolute_error: MAE
                - agreement_rate: Percentage within threshold
            - disputed_items: List of items with low agreement
            - summary: High-level summary statistics
                - total_items: Total items analyzed
                - disputed_count: Number of disputed items
                - dispute_rate: Percentage of disputed items
                - overall_agreement_quality: "excellent", "good", "fair", "poor"

    Raises:
        InsufficientDataError: If insufficient data for analysis
        ValueError: If no valid scores found

    Examples:
        >>> # Generate full report
        >>> report = calculator.generate_agreement_report(
        ...     review_items, "answer_relevance"
        ... )
        >>> 
        >>> print(f"Inter-rater alpha: "
        ...       f"{report['inter_rater_agreement']['krippendorff_alpha']:.3f}")
        >>> print(f"LLM-human correlation: "
        ...       f"{report['llm_human_correlation']['pearson_correlation']:.3f}")
        >>> print(f"Disputed items: {report['summary']['disputed_count']}")
        >>> print(f"Overall quality: {report['summary']['overall_agreement_quality']}")

        >>> # Report without LLM correlation
        >>> report = calculator.generate_agreement_report(
        ...     review_items, "answer_relevance", include_llm_correlation=False
        ... )
    """
    report = {
        "metric_name": metric_name,
        "timestamp": np.datetime64('now').astype(str)
    }

    # Calculate inter-rater agreement
    try:
        inter_rater = self.calculate_inter_rater_agreement(
            review_items, metric_name
        )
        report["inter_rater_agreement"] = inter_rater
    except (InsufficientDataError, ValueError) as e:
        logger.warning(f"Could not calculate inter-rater agreement: {e}")
        report["inter_rater_agreement"] = None

    # Calculate LLM-human correlation if requested
    if include_llm_correlation:
        try:
            # Extract LLM and human scores
            llm_scores = []
            human_scores = []

            for item in review_items:
                if metric_name not in item.human_scores:
                    continue

                llm_score = item.content.get("llm_scores", {}).get(metric_name)
                if llm_score is None:
                    continue

                llm_scores.append(llm_score)
                human_scores.append(item.human_scores[metric_name])

            if len(llm_scores) >= self.min_samples:
                llm_correlation = self.calculate_llm_human_correlation(
                    llm_scores, human_scores
                )
                report["llm_human_correlation"] = llm_correlation
            else:
                logger.warning(
                    f"Insufficient LLM-human pairs for correlation: "
                    f"got {len(llm_scores)}"
                )
                report["llm_human_correlation"] = None
        except (InsufficientDataError, ValueError) as e:
            logger.warning(f"Could not calculate LLM-human correlation: {e}")
            report["llm_human_correlation"] = None
    else:
        report["llm_human_correlation"] = None

    # Identify disputed items
    try:
        disputed = self.identify_disputed_items(review_items, metric_name)
        report["disputed_items"] = disputed
    except Exception as e:
        logger.warning(f"Could not identify disputed items: {e}")
        report["disputed_items"] = []

    # Generate summary
    total_items = len([
        item for item in review_items
        if metric_name in item.human_scores
    ])
    disputed_count = len(report["disputed_items"])
    dispute_rate = disputed_count / total_items if total_items > 0 else 0.0

    # Determine overall agreement quality
    quality = self._assess_agreement_quality(report)

    report["summary"] = {
        "total_items": total_items,
        "disputed_count": disputed_count,
        "dispute_rate": float(dispute_rate),
        "overall_agreement_quality": quality,
        "disagreement_threshold": self.disagreement_threshold
    }

    logger.info(
        f"Generated agreement report for {metric_name}: "
        f"quality={quality}, disputed={disputed_count}/{total_items}"
    )

    return report

identify_disputed_items(review_items, metric_name, threshold=None)

Identify items with low agreement (disputed items).

This method finds review items where reviewers significantly disagree, either between multiple human reviewers or between LLM and human scores. Disputed items may need additional review or dispute resolution.

Parameters:

Name	Type	Description	Default
review_items	List[ReviewItem]	List of review items to analyze	required
metric_name	str	Name of the metric to check for disputes	required
threshold	Optional[float]	Disagreement threshold (uses instance default if None)	None

Returns:

Type	Description
List[Dict[str, Any]]	List of disputed items, each containing: - item_id: UUID of the disputed item - item_type: Type of review item - content: Item content - scores: Dict of all scores for this item - disagreement_level: Maximum score difference - mean_score: Mean of all scores - std_score: Standard deviation of scores - n_reviewers: Number of reviewers

Examples:

>>> # Find disputed items
>>> disputed = calculator.identify_disputed_items(
...     review_items, "answer_relevance"
... )
>>> 
>>> for item in disputed:
...     print(f"Item {item['item_id']}: "
...           f"disagreement={item['disagreement_level']:.3f}")
...     print(f"  Scores: {item['scores']}")

>>> # Use custom threshold
>>> disputed = calculator.identify_disputed_items(
...     review_items, "answer_relevance", threshold=0.15
... )

Source code in uaef/hitl/agreement.py

def identify_disputed_items(
    self,
    review_items: List[ReviewItem],
    metric_name: str,
    threshold: Optional[float] = None
) -> List[Dict[str, Any]]:
    """
    Identify items with low agreement (disputed items).

    This method finds review items where reviewers significantly disagree,
    either between multiple human reviewers or between LLM and human scores.
    Disputed items may need additional review or dispute resolution.

    Args:
        review_items: List of review items to analyze
        metric_name: Name of the metric to check for disputes
        threshold: Disagreement threshold (uses instance default if None)

    Returns:
        List of disputed items, each containing:
            - item_id: UUID of the disputed item
            - item_type: Type of review item
            - content: Item content
            - scores: Dict of all scores for this item
            - disagreement_level: Maximum score difference
            - mean_score: Mean of all scores
            - std_score: Standard deviation of scores
            - n_reviewers: Number of reviewers

    Examples:
        >>> # Find disputed items
        >>> disputed = calculator.identify_disputed_items(
        ...     review_items, "answer_relevance"
        ... )
        >>> 
        >>> for item in disputed:
        ...     print(f"Item {item['item_id']}: "
        ...           f"disagreement={item['disagreement_level']:.3f}")
        ...     print(f"  Scores: {item['scores']}")

        >>> # Use custom threshold
        >>> disputed = calculator.identify_disputed_items(
        ...     review_items, "answer_relevance", threshold=0.15
        ... )
    """
    if threshold is None:
        threshold = self.disagreement_threshold

    disputed_items = []

    # Group items by content to find items with multiple reviews
    item_groups = self._group_items_by_content(review_items, metric_name)

    for item_key, scores_dict in item_groups.items():
        scores = list(scores_dict.values())

        if len(scores) < 2:
            continue

        # Calculate disagreement level (max difference)
        min_score = min(scores)
        max_score = max(scores)
        disagreement = max_score - min_score

        # Check if this is a disputed item
        if disagreement > threshold:
            # Find the original review item for metadata
            matching_item = None
            for item in review_items:
                if metric_name in item.human_scores:
                    content_str = str(sorted(item.content.items()))
                    if content_str in item_key or item_key in content_str:
                        matching_item = item
                        break

            if matching_item:
                disputed_items.append({
                    "item_id": str(matching_item.item_id),
                    "item_type": matching_item.item_type.value,
                    "content": matching_item.content,
                    "scores": scores_dict,
                    "disagreement_level": float(disagreement),
                    "mean_score": float(np.mean(scores)),
                    "std_score": float(np.std(scores)),
                    "n_reviewers": len(scores),
                    "threshold": threshold
                })

    # Also check for LLM-human disagreements
    for item in review_items:
        if metric_name not in item.human_scores:
            continue

        # Check if item has LLM score
        llm_score = item.content.get("llm_scores", {}).get(metric_name)
        if llm_score is None:
            continue

        human_score = item.human_scores[metric_name]
        disagreement = abs(llm_score - human_score)

        if disagreement > threshold:
            # Check if not already in disputed list
            item_id_str = str(item.item_id)
            if not any(d["item_id"] == item_id_str for d in disputed_items):
                disputed_items.append({
                    "item_id": item_id_str,
                    "item_type": item.item_type.value,
                    "content": item.content,
                    "scores": {
                        "llm": llm_score,
                        item.reviewer_id or "human": human_score
                    },
                    "disagreement_level": float(disagreement),
                    "mean_score": float((llm_score + human_score) / 2),
                    "std_score": float(np.std([llm_score, human_score])),
                    "n_reviewers": 2,
                    "threshold": threshold,
                    "llm_human_dispute": True
                })

    # Sort by disagreement level (highest first)
    disputed_items.sort(key=lambda x: x["disagreement_level"], reverse=True)

    logger.info(
        f"Identified {len(disputed_items)} disputed items for {metric_name} "
        f"(threshold={threshold})"
    )

    return disputed_items

InsufficientDataError

Bases: AgreementCalculationError

Raised when there is insufficient data for agreement calculation.

Source code in uaef/hitl/agreement.py

class InsufficientDataError(AgreementCalculationError):
    """Raised when there is insufficient data for agreement calculation."""
    pass

uaef.hitl.online_feedback

Online evaluation feedback workflow for HITL validation.

This module implements the online evaluation feedback workflow, one of three HITL workflows in UAEF. It collects human feedback on agent responses in real-time during production use.

The workflow: 1. Collects agent responses from online evaluation mode 2. Queues responses for human feedback (thumbs up/down or detailed scores) 3. Aggregates feedback across multiple reviewers 4. Tracks feedback trends over time to identify systematic issues

FeedbackTrend

Represents a trend in feedback over time.

Attributes:

Name	Type	Description
metric_name		Name of the metric being tracked
time_period		Time period for the trend (e.g., "last_7_days")
average_score		Average score over the period
score_change		Change in score compared to previous period
feedback_count		Number of feedback items in the period
trend_direction		Direction of trend ("improving", "declining", "stable")
issues_detected		List of detected issues or patterns

Source code in uaef/hitl/online_feedback.py

class FeedbackTrend:
    """
    Represents a trend in feedback over time.

    Attributes:
        metric_name: Name of the metric being tracked
        time_period: Time period for the trend (e.g., "last_7_days")
        average_score: Average score over the period
        score_change: Change in score compared to previous period
        feedback_count: Number of feedback items in the period
        trend_direction: Direction of trend ("improving", "declining", "stable")
        issues_detected: List of detected issues or patterns
    """

    def __init__(
        self,
        metric_name: str,
        time_period: str,
        average_score: float,
        score_change: float,
        feedback_count: int,
        trend_direction: str,
        issues_detected: List[str]
    ):
        """Initialize feedback trend."""
        self.metric_name = metric_name
        self.time_period = time_period
        self.average_score = average_score
        self.score_change = score_change
        self.feedback_count = feedback_count
        self.trend_direction = trend_direction
        self.issues_detected = issues_detected

    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary representation."""
        return {
            "metric_name": self.metric_name,
            "time_period": self.time_period,
            "average_score": self.average_score,
            "score_change": self.score_change,
            "feedback_count": self.feedback_count,
            "trend_direction": self.trend_direction,
            "issues_detected": self.issues_detected
        }

to_dict()

Convert to dictionary representation.

Source code in uaef/hitl/online_feedback.py

def to_dict(self) -> Dict[str, Any]:
    """Convert to dictionary representation."""
    return {
        "metric_name": self.metric_name,
        "time_period": self.time_period,
        "average_score": self.average_score,
        "score_change": self.score_change,
        "feedback_count": self.feedback_count,
        "trend_direction": self.trend_direction,
        "issues_detected": self.issues_detected
    }

InsufficientFeedbackError

Bases: OnlineFeedbackError

Raised when insufficient feedback is available for analysis.

Source code in uaef/hitl/online_feedback.py

class InsufficientFeedbackError(OnlineFeedbackError):
    """Raised when insufficient feedback is available for analysis."""
    pass

OnlineFeedbackError

Bases: Exception

Base exception for online feedback errors.

Source code in uaef/hitl/online_feedback.py

class OnlineFeedbackError(Exception):
    """Base exception for online feedback errors."""
    pass

OnlineFeedbackWorkflow

Workflow for collecting human feedback on agent responses in real-time.

This workflow collects and analyzes human feedback on agent responses during online evaluation mode. It supports: 1. Simple feedback (thumbs up/down) 2. Detailed feedback (metric scores 0-1) 3. Aggregation across multiple reviewers 4. Trend tracking over time

The workflow integrates with: - ReviewQueue: Manages items needing feedback - OnlineEvaluator: Provides agent responses for feedback

Attributes:

Name	Type	Description
review_queue		ReviewQueue instance for managing review items
simple_feedback_mapping		Mapping from thumbs up/down to scores
consensus_threshold		Threshold for reviewer agreement (default: 0.7)

Examples:

>>> # Initialize workflow
>>> workflow = OnlineFeedbackWorkflow()

>>> # Collect feedback on an agent response
>>> response_data = {
...     "trace_id": "trace_001",
...     "user_query": "Book a flight to NYC",
...     "agent_response": "I found 3 flights...",
...     "tools_called": ["search_flights", "get_prices"],
...     "evaluation_scores": {"answer_relevance": 0.85}
... }
>>> item = workflow.collect_feedback(response_data)

>>> # Submit simple feedback (thumbs up/down)
>>> workflow.submit_feedback(
...     item_id=item.item_id,
...     feedback_type="simple",
...     simple_rating="thumbs_up",
...     reviewer_id="user_123"
... )

>>> # Submit detailed feedback (metric scores)
>>> workflow.submit_feedback(
...     item_id=item.item_id,
...     feedback_type="detailed",
...     detailed_scores={
...         "answer_relevance": 0.90,
...         "answer_correctness": 0.85,
...         "helpfulness": 0.95
...     },
...     reviewer_id="reviewer_456",
...     feedback_text="Good response but could include pricing details"
... )

>>> # Aggregate feedback across reviewers
>>> aggregated = workflow.aggregate_feedback(item.item_id)
>>> aggregated["consensus_scores"]
{'answer_relevance': 0.875, 'answer_correctness': 0.85, 'helpfulness': 0.95}

>>> # Get feedback trends
>>> trends = workflow.get_feedback_trends(
...     time_period_days=7,
...     metric_name="answer_relevance"
... )
>>> trends[0].trend_direction
'improving'

Source code in uaef/hitl/online_feedback.py

class OnlineFeedbackWorkflow:
    """
    Workflow for collecting human feedback on agent responses in real-time.

    This workflow collects and analyzes human feedback on agent responses during
    online evaluation mode. It supports:
    1. Simple feedback (thumbs up/down)
    2. Detailed feedback (metric scores 0-1)
    3. Aggregation across multiple reviewers
    4. Trend tracking over time

    The workflow integrates with:
    - ReviewQueue: Manages items needing feedback
    - OnlineEvaluator: Provides agent responses for feedback

    Attributes:
        review_queue: ReviewQueue instance for managing review items
        simple_feedback_mapping: Mapping from thumbs up/down to scores
        consensus_threshold: Threshold for reviewer agreement (default: 0.7)

    Examples:
        >>> # Initialize workflow
        >>> workflow = OnlineFeedbackWorkflow()

        >>> # Collect feedback on an agent response
        >>> response_data = {
        ...     "trace_id": "trace_001",
        ...     "user_query": "Book a flight to NYC",
        ...     "agent_response": "I found 3 flights...",
        ...     "tools_called": ["search_flights", "get_prices"],
        ...     "evaluation_scores": {"answer_relevance": 0.85}
        ... }
        >>> item = workflow.collect_feedback(response_data)

        >>> # Submit simple feedback (thumbs up/down)
        >>> workflow.submit_feedback(
        ...     item_id=item.item_id,
        ...     feedback_type="simple",
        ...     simple_rating="thumbs_up",
        ...     reviewer_id="user_123"
        ... )

        >>> # Submit detailed feedback (metric scores)
        >>> workflow.submit_feedback(
        ...     item_id=item.item_id,
        ...     feedback_type="detailed",
        ...     detailed_scores={
        ...         "answer_relevance": 0.90,
        ...         "answer_correctness": 0.85,
        ...         "helpfulness": 0.95
        ...     },
        ...     reviewer_id="reviewer_456",
        ...     feedback_text="Good response but could include pricing details"
        ... )

        >>> # Aggregate feedback across reviewers
        >>> aggregated = workflow.aggregate_feedback(item.item_id)
        >>> aggregated["consensus_scores"]
        {'answer_relevance': 0.875, 'answer_correctness': 0.85, 'helpfulness': 0.95}

        >>> # Get feedback trends
        >>> trends = workflow.get_feedback_trends(
        ...     time_period_days=7,
        ...     metric_name="answer_relevance"
        ... )
        >>> trends[0].trend_direction
        'improving'
    """

    def __init__(
        self,
        review_queue: Optional[ReviewQueue] = None,
        simple_feedback_mapping: Optional[Dict[str, float]] = None,
        consensus_threshold: float = 0.7
    ):
        """
        Initialize the online feedback workflow.

        Args:
            review_queue: ReviewQueue instance (creates new if None)
            simple_feedback_mapping: Mapping from simple ratings to scores
                Default: {"thumbs_up": 1.0, "thumbs_down": 0.0}
            consensus_threshold: Threshold for reviewer agreement (0-1)

        Raises:
            ValueError: If consensus_threshold is not between 0 and 1
        """
        if not 0.0 <= consensus_threshold <= 1.0:
            raise ValueError(
                f"consensus_threshold must be between 0 and 1, got {consensus_threshold}"
            )

        self.review_queue = review_queue or ReviewQueue()
        self.simple_feedback_mapping = simple_feedback_mapping or {
            "thumbs_up": 1.0,
            "thumbs_down": 0.0
        }
        self.consensus_threshold = consensus_threshold

        # Storage for multiple reviewer feedback (item_id -> list of feedback)
        self._feedback_history: Dict[UUID, List[Dict[str, Any]]] = {}

        logger.info(
            f"Initialized OnlineFeedbackWorkflow with "
            f"consensus_threshold={consensus_threshold}"
        )

    def collect_feedback(
        self,
        response_data: Dict[str, Any],
        auto_priority: bool = True
    ) -> ReviewItem:
        """
        Collect an agent response for human feedback.

        Adds an agent response to the review queue for human feedback collection.
        This is typically called after online evaluation completes.

        Args:
            response_data: Dictionary containing:
                - trace_id: Unique identifier for the trace
                - user_query: The user's input/question
                - agent_response: The agent's response
                - tools_called: List of tools the agent called (optional)
                - evaluation_scores: Dict of automated evaluation scores (optional)
                - context: Additional context information (optional)
                - confidence_score: Confidence in the response (optional, default: 0.5)
            auto_priority: If True, automatically assign priority based on confidence

        Returns:
            ReviewItem that was added to the queue

        Raises:
            ValueError: If response_data format is invalid

        Examples:
            >>> workflow = OnlineFeedbackWorkflow()
            >>> response = {
            ...     "trace_id": "trace_001",
            ...     "user_query": "What is AI?",
            ...     "agent_response": "AI is artificial intelligence...",
            ...     "evaluation_scores": {"answer_relevance": 0.85},
            ...     "confidence_score": 0.75
            ... }
            >>> item = workflow.collect_feedback(response)
            >>> item.item_type
            <ItemType.AGENT_RESPONSE: 'agent_response'>
        """
        # Validate required fields
        if "trace_id" not in response_data:
            raise ValueError("response_data must contain 'trace_id'")
        if "user_query" not in response_data:
            raise ValueError("response_data must contain 'user_query'")
        if "agent_response" not in response_data:
            raise ValueError("response_data must contain 'agent_response'")

        trace_id = response_data["trace_id"]
        confidence_score = response_data.get("confidence_score", 0.5)

        # Create review item
        review_item = ReviewItem(
            item_type=ItemType.AGENT_RESPONSE,
            content={
                "user_query": response_data["user_query"],
                "agent_response": response_data["agent_response"],
                "tools_called": response_data.get("tools_called", []),
                "evaluation_scores": response_data.get("evaluation_scores", {}),
                "context": response_data.get("context", "")
            },
            metadata={
                "trace_id": str(trace_id),
                "timestamp": datetime.utcnow().isoformat()
            },
            priority=Priority.MEDIUM,  # Will be overridden if auto_priority=True
            confidence_score=confidence_score
        )

        # Add to queue
        added_item = self.review_queue.add_to_queue(
            review_item,
            auto_priority=auto_priority
        )

        # Initialize feedback history for this item
        self._feedback_history[added_item.item_id] = []

        logger.debug(
            f"Collected agent response for feedback: trace_id={trace_id}, "
            f"confidence={confidence_score:.3f}"
        )

        return added_item

    def submit_feedback(
        self,
        item_id: UUID,
        feedback_type: str,
        reviewer_id: str,
        simple_rating: Optional[str] = None,
        detailed_scores: Optional[Dict[str, float]] = None,
        feedback_text: Optional[str] = None
    ) -> ReviewItem:
        """
        Submit human feedback for an agent response.

        Supports two types of feedback:
        1. Simple: thumbs up/down rating
        2. Detailed: metric scores (0-1) for multiple dimensions

        Args:
            item_id: UUID of the item to provide feedback on
            feedback_type: Type of feedback ("simple" or "detailed")
            reviewer_id: ID of the human reviewer
            simple_rating: Simple rating ("thumbs_up" or "thumbs_down")
                Required if feedback_type="simple"
            detailed_scores: Dictionary of metric_name -> score (0-1)
                Required if feedback_type="detailed"
            feedback_text: Optional textual feedback

        Returns:
            Updated ReviewItem

        Raises:
            ValueError: If feedback format is invalid
            OnlineFeedbackError: If item not found or invalid operation

        Examples:
            >>> workflow = OnlineFeedbackWorkflow()
            >>> # Simple feedback
            >>> item = workflow.submit_feedback(
            ...     item_id=some_uuid,
            ...     feedback_type="simple",
            ...     simple_rating="thumbs_up",
            ...     reviewer_id="user_123"
            ... )

            >>> # Detailed feedback
            >>> item = workflow.submit_feedback(
            ...     item_id=some_uuid,
            ...     feedback_type="detailed",
            ...     detailed_scores={
            ...         "answer_relevance": 0.90,
            ...         "helpfulness": 0.85
            ...     },
            ...     reviewer_id="reviewer_456",
            ...     feedback_text="Good response"
            ... )
        """
        # Validate feedback_type
        if feedback_type not in ["simple", "detailed"]:
            raise ValueError(
                f"feedback_type must be 'simple' or 'detailed', got '{feedback_type}'"
            )

        # Validate simple feedback
        if feedback_type == "simple":
            if simple_rating is None:
                raise ValueError("simple_rating is required for simple feedback")
            if simple_rating not in self.simple_feedback_mapping:
                raise ValueError(
                    f"simple_rating must be one of {list(self.simple_feedback_mapping.keys())}, "
                    f"got '{simple_rating}'"
                )

        # Validate detailed feedback
        if feedback_type == "detailed":
            if detailed_scores is None or not detailed_scores:
                raise ValueError("detailed_scores is required for detailed feedback")
            for metric_name, score in detailed_scores.items():
                if not 0.0 <= score <= 1.0:
                    raise ValueError(
                        f"Score for '{metric_name}' must be between 0 and 1, got {score}"
                    )

        # Get the item
        try:
            item = self.review_queue.get_item(item_id)
        except Exception as e:
            raise OnlineFeedbackError(f"Failed to get item {item_id}: {e}")

        # Convert simple rating to scores
        if feedback_type == "simple":
            human_scores = {
                "overall_rating": self.simple_feedback_mapping[simple_rating]
            }
        else:
            human_scores = detailed_scores

        # Store feedback in history (for aggregation across multiple reviewers)
        feedback_entry = {
            "reviewer_id": reviewer_id,
            "timestamp": datetime.utcnow().isoformat(),
            "feedback_type": feedback_type,
            "scores": human_scores,
            "feedback_text": feedback_text
        }

        if item_id not in self._feedback_history:
            self._feedback_history[item_id] = []
        self._feedback_history[item_id].append(feedback_entry)

        # Update the review item with the latest feedback
        # Note: For multiple reviewers, we'll aggregate in aggregate_feedback()
        try:
            updated_item = self.review_queue.submit_review(
                item_id=item_id,
                human_scores=human_scores,
                reviewer_id=reviewer_id,
                feedback=feedback_text
            )
        except Exception as e:
            raise OnlineFeedbackError(f"Failed to submit review: {e}")

        logger.info(
            f"Submitted {feedback_type} feedback for item {item_id} "
            f"from reviewer {reviewer_id}"
        )

        return updated_item

    def aggregate_feedback(
        self,
        item_id: UUID,
        aggregation_method: str = "mean"
    ) -> Dict[str, Any]:
        """
        Aggregate feedback across multiple reviewers.

        Combines feedback from multiple reviewers to produce consensus scores
        and identify areas of agreement/disagreement.

        Args:
            item_id: UUID of the item to aggregate feedback for
            aggregation_method: Method for aggregation ("mean", "median", "weighted")

        Returns:
            Dictionary containing:
                - consensus_scores: Aggregated scores for each metric
                - reviewer_count: Number of reviewers
                - score_variance: Variance in scores across reviewers
                - agreement_level: Level of agreement ("high", "medium", "low")
                - individual_feedback: List of individual feedback entries

        Raises:
            OnlineFeedbackError: If item not found
            InsufficientFeedbackError: If no feedback available

        Examples:
            >>> workflow = OnlineFeedbackWorkflow()
            >>> # After multiple reviewers submit feedback...
            >>> aggregated = workflow.aggregate_feedback(item_id)
            >>> aggregated["consensus_scores"]
            {'answer_relevance': 0.875, 'helpfulness': 0.90}
            >>> aggregated["agreement_level"]
            'high'
        """
        # Validate aggregation method
        if aggregation_method not in ["mean", "median", "weighted"]:
            raise ValueError(
                f"aggregation_method must be 'mean', 'median', or 'weighted', "
                f"got '{aggregation_method}'"
            )

        # Get feedback history
        if item_id not in self._feedback_history:
            raise OnlineFeedbackError(f"No feedback history found for item {item_id}")

        feedback_entries = self._feedback_history[item_id]

        if not feedback_entries:
            raise InsufficientFeedbackError(
                f"No feedback available for item {item_id}"
            )

        # Collect all scores by metric
        scores_by_metric: Dict[str, List[float]] = {}

        for entry in feedback_entries:
            for metric_name, score in entry["scores"].items():
                if metric_name not in scores_by_metric:
                    scores_by_metric[metric_name] = []
                scores_by_metric[metric_name].append(score)

        # Aggregate scores
        consensus_scores = {}
        score_variance = {}

        for metric_name, scores in scores_by_metric.items():
            scores_arr = np.array(scores)

            if aggregation_method == "mean":
                consensus_scores[metric_name] = float(np.mean(scores_arr))
            elif aggregation_method == "median":
                consensus_scores[metric_name] = float(np.median(scores_arr))
            elif aggregation_method == "weighted":
                # For weighted, use recency weighting (more recent = higher weight)
                weights = np.linspace(0.5, 1.0, len(scores))
                consensus_scores[metric_name] = float(
                    np.average(scores_arr, weights=weights)
                )

            # Calculate variance
            score_variance[metric_name] = float(np.var(scores_arr))

        # Calculate overall agreement level
        avg_variance = np.mean(list(score_variance.values())) if score_variance else 0.0

        if avg_variance < 0.01:
            agreement_level = "high"
        elif avg_variance < 0.05:
            agreement_level = "medium"
        else:
            agreement_level = "low"

        # Build result
        result = {
            "item_id": str(item_id),
            "consensus_scores": consensus_scores,
            "reviewer_count": len(feedback_entries),
            "score_variance": score_variance,
            "average_variance": float(avg_variance),
            "agreement_level": agreement_level,
            "aggregation_method": aggregation_method,
            "individual_feedback": feedback_entries,
            "timestamp": datetime.utcnow().isoformat()
        }

        logger.info(
            f"Aggregated feedback for item {item_id}: "
            f"{len(feedback_entries)} reviewers, "
            f"agreement={agreement_level}"
        )

        return result

    def get_feedback_trends(
        self,
        time_period_days: int = 7,
        metric_name: Optional[str] = None,
        min_feedback_count: int = 3
    ) -> List[FeedbackTrend]:
        """
        Analyze feedback trends over time.

        Tracks how feedback scores change over time to identify:
        - Improving or declining quality
        - Systematic issues
        - Patterns in user satisfaction

        Args:
            time_period_days: Number of days to analyze
            metric_name: Optional filter by specific metric
            min_feedback_count: Minimum feedback items required for trend

        Returns:
            List of FeedbackTrend objects

        Raises:
            InsufficientFeedbackError: If insufficient feedback for analysis

        Examples:
            >>> workflow = OnlineFeedbackWorkflow()
            >>> trends = workflow.get_feedback_trends(
            ...     time_period_days=7,
            ...     metric_name="answer_relevance"
            ... )
            >>> trends[0].trend_direction
            'improving'
            >>> trends[0].average_score
            0.85
            >>> trends[0].score_change
            0.05
        """
        # Get all agent response items
        all_items = self.review_queue.get_all_items(
            item_type=ItemType.AGENT_RESPONSE
        )

        # Filter by time period
        cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
        recent_items = [
            item for item in all_items
            if item.created_at >= cutoff_date and item.human_scores
        ]

        if len(recent_items) < min_feedback_count:
            raise InsufficientFeedbackError(
                f"Insufficient feedback for trend analysis: "
                f"need at least {min_feedback_count}, got {len(recent_items)}"
            )

        # Collect scores by metric
        scores_by_metric: Dict[str, List[Tuple[datetime, float]]] = {}

        for item in recent_items:
            for m_name, score in item.human_scores.items():
                # Filter by metric_name if specified
                if metric_name and m_name != metric_name:
                    continue

                if m_name not in scores_by_metric:
                    scores_by_metric[m_name] = []
                scores_by_metric[m_name].append((item.created_at, score))

        # Analyze trends for each metric
        trends = []

        for m_name, score_data in scores_by_metric.items():
            if len(score_data) < min_feedback_count:
                continue

            # Sort by timestamp
            score_data.sort(key=lambda x: x[0])

            # Extract scores
            scores = [score for _, score in score_data]
            scores_arr = np.array(scores)

            # Calculate average score
            avg_score = float(np.mean(scores_arr))

            # Calculate trend (compare first half vs second half)
            mid_point = len(scores) // 2
            first_half_avg = float(np.mean(scores_arr[:mid_point]))
            second_half_avg = float(np.mean(scores_arr[mid_point:]))
            score_change = second_half_avg - first_half_avg

            # Determine trend direction
            if score_change > 0.05:
                trend_direction = "improving"
            elif score_change < -0.05:
                trend_direction = "declining"
            else:
                trend_direction = "stable"

            # Detect issues
            issues_detected = []

            # Check for low scores
            low_score_count = np.sum(scores_arr < 0.5)
            if low_score_count > len(scores) * 0.3:
                issues_detected.append(
                    f"High rate of low scores: {low_score_count}/{len(scores)} "
                    f"({low_score_count/len(scores):.1%})"
                )

            # Check for declining trend
            if trend_direction == "declining":
                issues_detected.append(
                    f"Declining trend detected: score dropped by {abs(score_change):.3f}"
                )

            # Check for high variance
            score_variance = float(np.var(scores_arr))
            if score_variance > 0.1:
                issues_detected.append(
                    f"High variance in scores: {score_variance:.3f} "
                    "(inconsistent quality)"
                )

            # Create trend object
            trend = FeedbackTrend(
                metric_name=m_name,
                time_period=f"last_{time_period_days}_days",
                average_score=avg_score,
                score_change=score_change,
                feedback_count=len(scores),
                trend_direction=trend_direction,
                issues_detected=issues_detected
            )

            trends.append(trend)

            logger.debug(
                f"Trend for {m_name}: avg={avg_score:.3f}, "
                f"change={score_change:+.3f}, direction={trend_direction}"
            )

        if not trends:
            raise InsufficientFeedbackError(
                f"No trends available for the specified criteria"
            )

        logger.info(
            f"Analyzed feedback trends over {time_period_days} days: "
            f"{len(trends)} metrics analyzed"
        )

        return trends

    def get_feedback_summary(
        self,
        time_period_days: Optional[int] = None
    ) -> Dict[str, Any]:
        """
        Get a summary of feedback collection status.

        Provides an overview of:
        - Total feedback collected
        - Feedback by type (simple vs detailed)
        - Average scores by metric
        - Recent trends

        Args:
            time_period_days: Optional time period for filtering (default: all time)

        Returns:
            Dictionary with feedback summary

        Examples:
            >>> workflow = OnlineFeedbackWorkflow()
            >>> summary = workflow.get_feedback_summary(time_period_days=7)
            >>> summary["total_feedback_items"]
            42
            >>> summary["average_scores"]["answer_relevance"]
            0.85
        """
        # Get all agent response items
        all_items = self.review_queue.get_all_items(
            item_type=ItemType.AGENT_RESPONSE
        )

        # Filter by time period if specified
        if time_period_days:
            cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
            items = [item for item in all_items if item.created_at >= cutoff_date]
        else:
            items = all_items

        # Count feedback by type
        simple_feedback_count = 0
        detailed_feedback_count = 0

        for item_id, feedback_entries in self._feedback_history.items():
            for entry in feedback_entries:
                if entry["feedback_type"] == "simple":
                    simple_feedback_count += 1
                else:
                    detailed_feedback_count += 1

        # Calculate average scores by metric
        scores_by_metric: Dict[str, List[float]] = {}

        for item in items:
            if item.human_scores:
                for metric_name, score in item.human_scores.items():
                    if metric_name not in scores_by_metric:
                        scores_by_metric[metric_name] = []
                    scores_by_metric[metric_name].append(score)

        average_scores = {
            metric_name: float(np.mean(scores))
            for metric_name, scores in scores_by_metric.items()
        }

        # Get queue stats
        queue_stats = self.review_queue.get_queue_stats()

        return {
            "timestamp": datetime.utcnow().isoformat(),
            "time_period": f"last_{time_period_days}_days" if time_period_days else "all_time",
            "total_feedback_items": len(items),
            "pending_feedback": queue_stats["by_type"]["agent_response"],
            "completed_feedback": len([item for item in items if item.human_scores]),
            "feedback_by_type": {
                "simple": simple_feedback_count,
                "detailed": detailed_feedback_count
            },
            "average_scores": average_scores,
            "metrics_tracked": list(scores_by_metric.keys()),
            "total_reviewers": len(set(
                item.reviewer_id for item in items if item.reviewer_id
            ))
        }

aggregate_feedback(item_id, aggregation_method='mean')

Aggregate feedback across multiple reviewers.

Combines feedback from multiple reviewers to produce consensus scores and identify areas of agreement/disagreement.

Parameters:

Name	Type	Description	Default
item_id	UUID	UUID of the item to aggregate feedback for	required
aggregation_method	str	Method for aggregation ("mean", "median", "weighted")	'mean'

Returns:

Type	Description
Dict[str, Any]	Dictionary containing: - consensus_scores: Aggregated scores for each metric - reviewer_count: Number of reviewers - score_variance: Variance in scores across reviewers - agreement_level: Level of agreement ("high", "medium", "low") - individual_feedback: List of individual feedback entries

Raises:

Type	Description
OnlineFeedbackError	If item not found
InsufficientFeedbackError	If no feedback available

Examples:

>>> workflow = OnlineFeedbackWorkflow()
>>> # After multiple reviewers submit feedback...
>>> aggregated = workflow.aggregate_feedback(item_id)
>>> aggregated["consensus_scores"]
{'answer_relevance': 0.875, 'helpfulness': 0.90}
>>> aggregated["agreement_level"]
'high'

Source code in uaef/hitl/online_feedback.py

def aggregate_feedback(
    self,
    item_id: UUID,
    aggregation_method: str = "mean"
) -> Dict[str, Any]:
    """
    Aggregate feedback across multiple reviewers.

    Combines feedback from multiple reviewers to produce consensus scores
    and identify areas of agreement/disagreement.

    Args:
        item_id: UUID of the item to aggregate feedback for
        aggregation_method: Method for aggregation ("mean", "median", "weighted")

    Returns:
        Dictionary containing:
            - consensus_scores: Aggregated scores for each metric
            - reviewer_count: Number of reviewers
            - score_variance: Variance in scores across reviewers
            - agreement_level: Level of agreement ("high", "medium", "low")
            - individual_feedback: List of individual feedback entries

    Raises:
        OnlineFeedbackError: If item not found
        InsufficientFeedbackError: If no feedback available

    Examples:
        >>> workflow = OnlineFeedbackWorkflow()
        >>> # After multiple reviewers submit feedback...
        >>> aggregated = workflow.aggregate_feedback(item_id)
        >>> aggregated["consensus_scores"]
        {'answer_relevance': 0.875, 'helpfulness': 0.90}
        >>> aggregated["agreement_level"]
        'high'
    """
    # Validate aggregation method
    if aggregation_method not in ["mean", "median", "weighted"]:
        raise ValueError(
            f"aggregation_method must be 'mean', 'median', or 'weighted', "
            f"got '{aggregation_method}'"
        )

    # Get feedback history
    if item_id not in self._feedback_history:
        raise OnlineFeedbackError(f"No feedback history found for item {item_id}")

    feedback_entries = self._feedback_history[item_id]

    if not feedback_entries:
        raise InsufficientFeedbackError(
            f"No feedback available for item {item_id}"
        )

    # Collect all scores by metric
    scores_by_metric: Dict[str, List[float]] = {}

    for entry in feedback_entries:
        for metric_name, score in entry["scores"].items():
            if metric_name not in scores_by_metric:
                scores_by_metric[metric_name] = []
            scores_by_metric[metric_name].append(score)

    # Aggregate scores
    consensus_scores = {}
    score_variance = {}

    for metric_name, scores in scores_by_metric.items():
        scores_arr = np.array(scores)

        if aggregation_method == "mean":
            consensus_scores[metric_name] = float(np.mean(scores_arr))
        elif aggregation_method == "median":
            consensus_scores[metric_name] = float(np.median(scores_arr))
        elif aggregation_method == "weighted":
            # For weighted, use recency weighting (more recent = higher weight)
            weights = np.linspace(0.5, 1.0, len(scores))
            consensus_scores[metric_name] = float(
                np.average(scores_arr, weights=weights)
            )

        # Calculate variance
        score_variance[metric_name] = float(np.var(scores_arr))

    # Calculate overall agreement level
    avg_variance = np.mean(list(score_variance.values())) if score_variance else 0.0

    if avg_variance < 0.01:
        agreement_level = "high"
    elif avg_variance < 0.05:
        agreement_level = "medium"
    else:
        agreement_level = "low"

    # Build result
    result = {
        "item_id": str(item_id),
        "consensus_scores": consensus_scores,
        "reviewer_count": len(feedback_entries),
        "score_variance": score_variance,
        "average_variance": float(avg_variance),
        "agreement_level": agreement_level,
        "aggregation_method": aggregation_method,
        "individual_feedback": feedback_entries,
        "timestamp": datetime.utcnow().isoformat()
    }

    logger.info(
        f"Aggregated feedback for item {item_id}: "
        f"{len(feedback_entries)} reviewers, "
        f"agreement={agreement_level}"
    )

    return result

collect_feedback(response_data, auto_priority=True)

Collect an agent response for human feedback.

Adds an agent response to the review queue for human feedback collection. This is typically called after online evaluation completes.

Parameters:

Name	Type	Description	Default
response_data	Dict[str, Any]	Dictionary containing: - trace_id: Unique identifier for the trace - user_query: The user's input/question - agent_response: The agent's response - tools_called: List of tools the agent called (optional) - evaluation_scores: Dict of automated evaluation scores (optional) - context: Additional context information (optional) - confidence_score: Confidence in the response (optional, default: 0.5)	required
auto_priority	bool	If True, automatically assign priority based on confidence	True

Returns:

Type	Description
ReviewItem	ReviewItem that was added to the queue

Raises:

Type	Description
ValueError	If response_data format is invalid

Examples:

>>> workflow = OnlineFeedbackWorkflow()
>>> response = {
...     "trace_id": "trace_001",
...     "user_query": "What is AI?",
...     "agent_response": "AI is artificial intelligence...",
...     "evaluation_scores": {"answer_relevance": 0.85},
...     "confidence_score": 0.75
... }
>>> item = workflow.collect_feedback(response)
>>> item.item_type
<ItemType.AGENT_RESPONSE: 'agent_response'>

Source code in uaef/hitl/online_feedback.py

def collect_feedback(
    self,
    response_data: Dict[str, Any],
    auto_priority: bool = True
) -> ReviewItem:
    """
    Collect an agent response for human feedback.

    Adds an agent response to the review queue for human feedback collection.
    This is typically called after online evaluation completes.

    Args:
        response_data: Dictionary containing:
            - trace_id: Unique identifier for the trace
            - user_query: The user's input/question
            - agent_response: The agent's response
            - tools_called: List of tools the agent called (optional)
            - evaluation_scores: Dict of automated evaluation scores (optional)
            - context: Additional context information (optional)
            - confidence_score: Confidence in the response (optional, default: 0.5)
        auto_priority: If True, automatically assign priority based on confidence

    Returns:
        ReviewItem that was added to the queue

    Raises:
        ValueError: If response_data format is invalid

    Examples:
        >>> workflow = OnlineFeedbackWorkflow()
        >>> response = {
        ...     "trace_id": "trace_001",
        ...     "user_query": "What is AI?",
        ...     "agent_response": "AI is artificial intelligence...",
        ...     "evaluation_scores": {"answer_relevance": 0.85},
        ...     "confidence_score": 0.75
        ... }
        >>> item = workflow.collect_feedback(response)
        >>> item.item_type
        <ItemType.AGENT_RESPONSE: 'agent_response'>
    """
    # Validate required fields
    if "trace_id" not in response_data:
        raise ValueError("response_data must contain 'trace_id'")
    if "user_query" not in response_data:
        raise ValueError("response_data must contain 'user_query'")
    if "agent_response" not in response_data:
        raise ValueError("response_data must contain 'agent_response'")

    trace_id = response_data["trace_id"]
    confidence_score = response_data.get("confidence_score", 0.5)

    # Create review item
    review_item = ReviewItem(
        item_type=ItemType.AGENT_RESPONSE,
        content={
            "user_query": response_data["user_query"],
            "agent_response": response_data["agent_response"],
            "tools_called": response_data.get("tools_called", []),
            "evaluation_scores": response_data.get("evaluation_scores", {}),
            "context": response_data.get("context", "")
        },
        metadata={
            "trace_id": str(trace_id),
            "timestamp": datetime.utcnow().isoformat()
        },
        priority=Priority.MEDIUM,  # Will be overridden if auto_priority=True
        confidence_score=confidence_score
    )

    # Add to queue
    added_item = self.review_queue.add_to_queue(
        review_item,
        auto_priority=auto_priority
    )

    # Initialize feedback history for this item
    self._feedback_history[added_item.item_id] = []

    logger.debug(
        f"Collected agent response for feedback: trace_id={trace_id}, "
        f"confidence={confidence_score:.3f}"
    )

    return added_item

get_feedback_summary(time_period_days=None)

Get a summary of feedback collection status.

Provides an overview of: - Total feedback collected - Feedback by type (simple vs detailed) - Average scores by metric - Recent trends

Parameters:

Name	Type	Description	Default
time_period_days	Optional[int]	Optional time period for filtering (default: all time)	None

Returns:

Type	Description
Dict[str, Any]	Dictionary with feedback summary

Examples:

>>> workflow = OnlineFeedbackWorkflow()
>>> summary = workflow.get_feedback_summary(time_period_days=7)
>>> summary["total_feedback_items"]
42
>>> summary["average_scores"]["answer_relevance"]
0.85

Source code in uaef/hitl/online_feedback.py

def get_feedback_summary(
    self,
    time_period_days: Optional[int] = None
) -> Dict[str, Any]:
    """
    Get a summary of feedback collection status.

    Provides an overview of:
    - Total feedback collected
    - Feedback by type (simple vs detailed)
    - Average scores by metric
    - Recent trends

    Args:
        time_period_days: Optional time period for filtering (default: all time)

    Returns:
        Dictionary with feedback summary

    Examples:
        >>> workflow = OnlineFeedbackWorkflow()
        >>> summary = workflow.get_feedback_summary(time_period_days=7)
        >>> summary["total_feedback_items"]
        42
        >>> summary["average_scores"]["answer_relevance"]
        0.85
    """
    # Get all agent response items
    all_items = self.review_queue.get_all_items(
        item_type=ItemType.AGENT_RESPONSE
    )

    # Filter by time period if specified
    if time_period_days:
        cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
        items = [item for item in all_items if item.created_at >= cutoff_date]
    else:
        items = all_items

    # Count feedback by type
    simple_feedback_count = 0
    detailed_feedback_count = 0

    for item_id, feedback_entries in self._feedback_history.items():
        for entry in feedback_entries:
            if entry["feedback_type"] == "simple":
                simple_feedback_count += 1
            else:
                detailed_feedback_count += 1

    # Calculate average scores by metric
    scores_by_metric: Dict[str, List[float]] = {}

    for item in items:
        if item.human_scores:
            for metric_name, score in item.human_scores.items():
                if metric_name not in scores_by_metric:
                    scores_by_metric[metric_name] = []
                scores_by_metric[metric_name].append(score)

    average_scores = {
        metric_name: float(np.mean(scores))
        for metric_name, scores in scores_by_metric.items()
    }

    # Get queue stats
    queue_stats = self.review_queue.get_queue_stats()

    return {
        "timestamp": datetime.utcnow().isoformat(),
        "time_period": f"last_{time_period_days}_days" if time_period_days else "all_time",
        "total_feedback_items": len(items),
        "pending_feedback": queue_stats["by_type"]["agent_response"],
        "completed_feedback": len([item for item in items if item.human_scores]),
        "feedback_by_type": {
            "simple": simple_feedback_count,
            "detailed": detailed_feedback_count
        },
        "average_scores": average_scores,
        "metrics_tracked": list(scores_by_metric.keys()),
        "total_reviewers": len(set(
            item.reviewer_id for item in items if item.reviewer_id
        ))
    }

get_feedback_trends(time_period_days=7, metric_name=None, min_feedback_count=3)

Analyze feedback trends over time.

Tracks how feedback scores change over time to identify: - Improving or declining quality - Systematic issues - Patterns in user satisfaction

Parameters:

Name	Type	Description	Default
time_period_days	int	Number of days to analyze	7
metric_name	Optional[str]	Optional filter by specific metric	None
min_feedback_count	int	Minimum feedback items required for trend	3

Returns:

Type	Description
List[FeedbackTrend]	List of FeedbackTrend objects

Raises:

Type	Description
InsufficientFeedbackError	If insufficient feedback for analysis

Examples:

>>> workflow = OnlineFeedbackWorkflow()
>>> trends = workflow.get_feedback_trends(
...     time_period_days=7,
...     metric_name="answer_relevance"
... )
>>> trends[0].trend_direction
'improving'
>>> trends[0].average_score
0.85
>>> trends[0].score_change
0.05

Source code in uaef/hitl/online_feedback.py

def get_feedback_trends(
    self,
    time_period_days: int = 7,
    metric_name: Optional[str] = None,
    min_feedback_count: int = 3
) -> List[FeedbackTrend]:
    """
    Analyze feedback trends over time.

    Tracks how feedback scores change over time to identify:
    - Improving or declining quality
    - Systematic issues
    - Patterns in user satisfaction

    Args:
        time_period_days: Number of days to analyze
        metric_name: Optional filter by specific metric
        min_feedback_count: Minimum feedback items required for trend

    Returns:
        List of FeedbackTrend objects

    Raises:
        InsufficientFeedbackError: If insufficient feedback for analysis

    Examples:
        >>> workflow = OnlineFeedbackWorkflow()
        >>> trends = workflow.get_feedback_trends(
        ...     time_period_days=7,
        ...     metric_name="answer_relevance"
        ... )
        >>> trends[0].trend_direction
        'improving'
        >>> trends[0].average_score
        0.85
        >>> trends[0].score_change
        0.05
    """
    # Get all agent response items
    all_items = self.review_queue.get_all_items(
        item_type=ItemType.AGENT_RESPONSE
    )

    # Filter by time period
    cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
    recent_items = [
        item for item in all_items
        if item.created_at >= cutoff_date and item.human_scores
    ]

    if len(recent_items) < min_feedback_count:
        raise InsufficientFeedbackError(
            f"Insufficient feedback for trend analysis: "
            f"need at least {min_feedback_count}, got {len(recent_items)}"
        )

    # Collect scores by metric
    scores_by_metric: Dict[str, List[Tuple[datetime, float]]] = {}

    for item in recent_items:
        for m_name, score in item.human_scores.items():
            # Filter by metric_name if specified
            if metric_name and m_name != metric_name:
                continue

            if m_name not in scores_by_metric:
                scores_by_metric[m_name] = []
            scores_by_metric[m_name].append((item.created_at, score))

    # Analyze trends for each metric
    trends = []

    for m_name, score_data in scores_by_metric.items():
        if len(score_data) < min_feedback_count:
            continue

        # Sort by timestamp
        score_data.sort(key=lambda x: x[0])

        # Extract scores
        scores = [score for _, score in score_data]
        scores_arr = np.array(scores)

        # Calculate average score
        avg_score = float(np.mean(scores_arr))

        # Calculate trend (compare first half vs second half)
        mid_point = len(scores) // 2
        first_half_avg = float(np.mean(scores_arr[:mid_point]))
        second_half_avg = float(np.mean(scores_arr[mid_point:]))
        score_change = second_half_avg - first_half_avg

        # Determine trend direction
        if score_change > 0.05:
            trend_direction = "improving"
        elif score_change < -0.05:
            trend_direction = "declining"
        else:
            trend_direction = "stable"

        # Detect issues
        issues_detected = []

        # Check for low scores
        low_score_count = np.sum(scores_arr < 0.5)
        if low_score_count > len(scores) * 0.3:
            issues_detected.append(
                f"High rate of low scores: {low_score_count}/{len(scores)} "
                f"({low_score_count/len(scores):.1%})"
            )

        # Check for declining trend
        if trend_direction == "declining":
            issues_detected.append(
                f"Declining trend detected: score dropped by {abs(score_change):.3f}"
            )

        # Check for high variance
        score_variance = float(np.var(scores_arr))
        if score_variance > 0.1:
            issues_detected.append(
                f"High variance in scores: {score_variance:.3f} "
                "(inconsistent quality)"
            )

        # Create trend object
        trend = FeedbackTrend(
            metric_name=m_name,
            time_period=f"last_{time_period_days}_days",
            average_score=avg_score,
            score_change=score_change,
            feedback_count=len(scores),
            trend_direction=trend_direction,
            issues_detected=issues_detected
        )

        trends.append(trend)

        logger.debug(
            f"Trend for {m_name}: avg={avg_score:.3f}, "
            f"change={score_change:+.3f}, direction={trend_direction}"
        )

    if not trends:
        raise InsufficientFeedbackError(
            f"No trends available for the specified criteria"
        )

    logger.info(
        f"Analyzed feedback trends over {time_period_days} days: "
        f"{len(trends)} metrics analyzed"
    )

    return trends

submit_feedback(item_id, feedback_type, reviewer_id, simple_rating=None, detailed_scores=None, feedback_text=None)

Submit human feedback for an agent response.

Supports two types of feedback: 1. Simple: thumbs up/down rating 2. Detailed: metric scores (0-1) for multiple dimensions

Parameters:

Name	Type	Description	Default
item_id	UUID	UUID of the item to provide feedback on	required
feedback_type	str	Type of feedback ("simple" or "detailed")	required
reviewer_id	str	ID of the human reviewer	required
simple_rating	Optional[str]	Simple rating ("thumbs_up" or "thumbs_down") Required if feedback_type="simple"	None
detailed_scores	Optional[Dict[str, float]]	Dictionary of metric_name -> score (0-1) Required if feedback_type="detailed"	None
feedback_text	Optional[str]	Optional textual feedback	None

Returns:

Type	Description
ReviewItem	Updated ReviewItem

Raises:

Type	Description
ValueError	If feedback format is invalid
OnlineFeedbackError	If item not found or invalid operation

Examples:

>>> workflow = OnlineFeedbackWorkflow()
>>> # Simple feedback
>>> item = workflow.submit_feedback(
...     item_id=some_uuid,
...     feedback_type="simple",
...     simple_rating="thumbs_up",
...     reviewer_id="user_123"
... )

>>> # Detailed feedback
>>> item = workflow.submit_feedback(
...     item_id=some_uuid,
...     feedback_type="detailed",
...     detailed_scores={
...         "answer_relevance": 0.90,
...         "helpfulness": 0.85
...     },
...     reviewer_id="reviewer_456",
...     feedback_text="Good response"
... )

Source code in uaef/hitl/online_feedback.py

def submit_feedback(
    self,
    item_id: UUID,
    feedback_type: str,
    reviewer_id: str,
    simple_rating: Optional[str] = None,
    detailed_scores: Optional[Dict[str, float]] = None,
    feedback_text: Optional[str] = None
) -> ReviewItem:
    """
    Submit human feedback for an agent response.

    Supports two types of feedback:
    1. Simple: thumbs up/down rating
    2. Detailed: metric scores (0-1) for multiple dimensions

    Args:
        item_id: UUID of the item to provide feedback on
        feedback_type: Type of feedback ("simple" or "detailed")
        reviewer_id: ID of the human reviewer
        simple_rating: Simple rating ("thumbs_up" or "thumbs_down")
            Required if feedback_type="simple"
        detailed_scores: Dictionary of metric_name -> score (0-1)
            Required if feedback_type="detailed"
        feedback_text: Optional textual feedback

    Returns:
        Updated ReviewItem

    Raises:
        ValueError: If feedback format is invalid
        OnlineFeedbackError: If item not found or invalid operation

    Examples:
        >>> workflow = OnlineFeedbackWorkflow()
        >>> # Simple feedback
        >>> item = workflow.submit_feedback(
        ...     item_id=some_uuid,
        ...     feedback_type="simple",
        ...     simple_rating="thumbs_up",
        ...     reviewer_id="user_123"
        ... )

        >>> # Detailed feedback
        >>> item = workflow.submit_feedback(
        ...     item_id=some_uuid,
        ...     feedback_type="detailed",
        ...     detailed_scores={
        ...         "answer_relevance": 0.90,
        ...         "helpfulness": 0.85
        ...     },
        ...     reviewer_id="reviewer_456",
        ...     feedback_text="Good response"
        ... )
    """
    # Validate feedback_type
    if feedback_type not in ["simple", "detailed"]:
        raise ValueError(
            f"feedback_type must be 'simple' or 'detailed', got '{feedback_type}'"
        )

    # Validate simple feedback
    if feedback_type == "simple":
        if simple_rating is None:
            raise ValueError("simple_rating is required for simple feedback")
        if simple_rating not in self.simple_feedback_mapping:
            raise ValueError(
                f"simple_rating must be one of {list(self.simple_feedback_mapping.keys())}, "
                f"got '{simple_rating}'"
            )

    # Validate detailed feedback
    if feedback_type == "detailed":
        if detailed_scores is None or not detailed_scores:
            raise ValueError("detailed_scores is required for detailed feedback")
        for metric_name, score in detailed_scores.items():
            if not 0.0 <= score <= 1.0:
                raise ValueError(
                    f"Score for '{metric_name}' must be between 0 and 1, got {score}"
                )

    # Get the item
    try:
        item = self.review_queue.get_item(item_id)
    except Exception as e:
        raise OnlineFeedbackError(f"Failed to get item {item_id}: {e}")

    # Convert simple rating to scores
    if feedback_type == "simple":
        human_scores = {
            "overall_rating": self.simple_feedback_mapping[simple_rating]
        }
    else:
        human_scores = detailed_scores

    # Store feedback in history (for aggregation across multiple reviewers)
    feedback_entry = {
        "reviewer_id": reviewer_id,
        "timestamp": datetime.utcnow().isoformat(),
        "feedback_type": feedback_type,
        "scores": human_scores,
        "feedback_text": feedback_text
    }

    if item_id not in self._feedback_history:
        self._feedback_history[item_id] = []
    self._feedback_history[item_id].append(feedback_entry)

    # Update the review item with the latest feedback
    # Note: For multiple reviewers, we'll aggregate in aggregate_feedback()
    try:
        updated_item = self.review_queue.submit_review(
            item_id=item_id,
            human_scores=human_scores,
            reviewer_id=reviewer_id,
            feedback=feedback_text
        )
    except Exception as e:
        raise OnlineFeedbackError(f"Failed to submit review: {e}")

    logger.info(
        f"Submitted {feedback_type} feedback for item {item_id} "
        f"from reviewer {reviewer_id}"
    )

    return updated_item

uaef.hitl.conversation_scoring

Conversation scoring workflow for HITL validation.

This module implements the conversation scoring workflow, one of three HITL workflows in UAEF. It collects human feedback on simulated conversations to validate and improve conversation simulation quality.

The workflow: 1. Queues simulated conversations for human scoring 2. Collects quality ratings across multiple dimensions 3. Aggregates scores across multiple reviewers 4. Analyzes simulation quality trends over time 5. Provides improvement suggestions for the conversation simulator

ConversationScoringError

Bases: Exception

Base exception for conversation scoring errors.

Source code in uaef/hitl/conversation_scoring.py

class ConversationScoringError(Exception):
    """Base exception for conversation scoring errors."""
    pass

ConversationScoringWorkflow

Workflow for scoring simulated conversations with human feedback.

This workflow validates and improves conversation simulation quality by: 1. Queuing simulated conversations for human scoring 2. Collecting quality ratings across dimensions (realism, coherence, etc.) 3. Aggregating scores across multiple reviewers 4. Analyzing quality trends over time 5. Providing improvement suggestions for the simulator

The workflow integrates with: - ReviewQueue: Manages conversations needing scoring - ConversationSimulator: Provides simulated conversations (Phase 7)

Scoring dimensions: - realism: How realistic the conversation feels - coherence: Logical flow and consistency - goal_achievement: Whether the conversation achieved its goal - naturalness: How natural the language and interactions are

Attributes:

Name	Type	Description
review_queue		ReviewQueue instance for managing review items
scoring_dimensions		List of quality dimensions to score
consensus_threshold		Threshold for reviewer agreement (default: 0.7)

Examples:

>>> # Initialize workflow
>>> workflow = ConversationScoringWorkflow()

>>> # Queue a simulated conversation for scoring
>>> conversation_data = {
...     "conversation_id": "conv_001",
...     "scenario": "Customer support - refund request",
...     "conversation": [
...         {"role": "user", "content": "I want a refund"},
...         {"role": "agent", "content": "I can help with that..."}
...     ],
...     "goal_achieved": True,
...     "turn_count": 5
... }
>>> item = workflow.queue_conversation(conversation_data)

>>> # Submit scores from a reviewer
>>> workflow.submit_scores(
...     item_id=item.item_id,
...     scores={
...         "realism": 0.85,
...         "coherence": 0.90,
...         "goal_achievement": 0.95,
...         "naturalness": 0.80
...     },
...     reviewer_id="reviewer_123",
...     feedback="Good conversation but agent could be more empathetic"
... )

>>> # Analyze simulation quality trends
>>> trends = workflow.analyze_simulation_quality(time_period_days=7)
>>> trends[0].trend_direction
'improving'

>>> # Get improvement suggestions
>>> suggestions = workflow.get_improvement_suggestions()
>>> suggestions["realism"]
['Add more varied user responses', 'Include edge cases']

Source code in uaef/hitl/conversation_scoring.py

class ConversationScoringWorkflow:
    """
    Workflow for scoring simulated conversations with human feedback.

    This workflow validates and improves conversation simulation quality by:
    1. Queuing simulated conversations for human scoring
    2. Collecting quality ratings across dimensions (realism, coherence, etc.)
    3. Aggregating scores across multiple reviewers
    4. Analyzing quality trends over time
    5. Providing improvement suggestions for the simulator

    The workflow integrates with:
    - ReviewQueue: Manages conversations needing scoring
    - ConversationSimulator: Provides simulated conversations (Phase 7)

    Scoring dimensions:
    - realism: How realistic the conversation feels
    - coherence: Logical flow and consistency
    - goal_achievement: Whether the conversation achieved its goal
    - naturalness: How natural the language and interactions are


    Attributes:
        review_queue: ReviewQueue instance for managing review items
        scoring_dimensions: List of quality dimensions to score
        consensus_threshold: Threshold for reviewer agreement (default: 0.7)

    Examples:
        >>> # Initialize workflow
        >>> workflow = ConversationScoringWorkflow()

        >>> # Queue a simulated conversation for scoring
        >>> conversation_data = {
        ...     "conversation_id": "conv_001",
        ...     "scenario": "Customer support - refund request",
        ...     "conversation": [
        ...         {"role": "user", "content": "I want a refund"},
        ...         {"role": "agent", "content": "I can help with that..."}
        ...     ],
        ...     "goal_achieved": True,
        ...     "turn_count": 5
        ... }
        >>> item = workflow.queue_conversation(conversation_data)

        >>> # Submit scores from a reviewer
        >>> workflow.submit_scores(
        ...     item_id=item.item_id,
        ...     scores={
        ...         "realism": 0.85,
        ...         "coherence": 0.90,
        ...         "goal_achievement": 0.95,
        ...         "naturalness": 0.80
        ...     },
        ...     reviewer_id="reviewer_123",
        ...     feedback="Good conversation but agent could be more empathetic"
        ... )

        >>> # Analyze simulation quality trends
        >>> trends = workflow.analyze_simulation_quality(time_period_days=7)
        >>> trends[0].trend_direction
        'improving'

        >>> # Get improvement suggestions
        >>> suggestions = workflow.get_improvement_suggestions()
        >>> suggestions["realism"]
        ['Add more varied user responses', 'Include edge cases']
    """


    def __init__(
        self,
        review_queue: Optional[ReviewQueue] = None,
        scoring_dimensions: Optional[List[str]] = None,
        consensus_threshold: float = 0.7
    ):
        """
        Initialize the conversation scoring workflow.

        Args:
            review_queue: ReviewQueue instance (creates new if None)
            scoring_dimensions: List of quality dimensions to score
                Default: ["realism", "coherence", "goal_achievement", "naturalness"]
            consensus_threshold: Threshold for reviewer agreement (0-1)

        Raises:
            ValueError: If consensus_threshold is not between 0 and 1
        """
        if not 0.0 <= consensus_threshold <= 1.0:
            raise ValueError(
                f"consensus_threshold must be between 0 and 1, got {consensus_threshold}"
            )

        self.review_queue = review_queue or ReviewQueue()
        self.scoring_dimensions = scoring_dimensions or [
            "realism",
            "coherence",
            "goal_achievement",
            "naturalness"
        ]
        self.consensus_threshold = consensus_threshold

        # Storage for multiple reviewer scores (item_id -> list of score entries)
        self._score_history: Dict[UUID, List[Dict[str, Any]]] = {}

        logger.info(
            f"Initialized ConversationScoringWorkflow with "
            f"dimensions={self.scoring_dimensions}, "
            f"consensus_threshold={consensus_threshold}"
        )


    def queue_conversation(
        self,
        conversation_data: Dict[str, Any],
        auto_priority: bool = True
    ) -> ReviewItem:
        """
        Queue a simulated conversation for human scoring.

        Adds a simulated conversation to the review queue for quality assessment.
        This is typically called after the conversation simulator generates a conversation.

        Args:
            conversation_data: Dictionary containing:
                - conversation_id: Unique identifier for the conversation
                - scenario: Description of the scenario
                - conversation: List of message dictionaries with role and content
                - goal_achieved: Whether the conversation achieved its goal (optional)
                - turn_count: Number of turns in the conversation (optional)
                - user_model: User behavior model used (optional)
                - metadata: Additional metadata (optional)
                - confidence_score: Confidence in simulation quality (optional, default: 0.5)
            auto_priority: If True, automatically assign priority based on confidence

        Returns:
            ReviewItem that was added to the queue

        Raises:
            ValueError: If conversation_data format is invalid

        Examples:
            >>> workflow = ConversationScoringWorkflow()
            >>> conversation = {
            ...     "conversation_id": "conv_001",
            ...     "scenario": "Customer support - refund request",
            ...     "conversation": [
            ...         {"role": "user", "content": "I want a refund"},
            ...         {"role": "agent", "content": "I can help with that..."}
            ...     ],
            ...     "goal_achieved": True,
            ...     "turn_count": 5,
            ...     "confidence_score": 0.75
            ... }
            >>> item = workflow.queue_conversation(conversation)
            >>> item.item_type
            <ItemType.SIMULATED_CONVERSATION: 'simulated_conversation'>
        """
        # Validate required fields
        if "conversation_id" not in conversation_data:
            raise ValueError("conversation_data must contain 'conversation_id'")
        if "scenario" not in conversation_data:
            raise ValueError("conversation_data must contain 'scenario'")
        if "conversation" not in conversation_data:
            raise ValueError("conversation_data must contain 'conversation'")

        conversation_id = conversation_data["conversation_id"]
        confidence_score = conversation_data.get("confidence_score", 0.5)

        # Create review item
        review_item = ReviewItem(
            item_type=ItemType.SIMULATED_CONVERSATION,
            content={
                "scenario": conversation_data["scenario"],
                "conversation": conversation_data["conversation"],
                "goal_achieved": conversation_data.get("goal_achieved", None),
                "turn_count": conversation_data.get("turn_count", len(conversation_data["conversation"])),
                "user_model": conversation_data.get("user_model", ""),
                "metadata": conversation_data.get("metadata", {})
            },
            metadata={
                "conversation_id": str(conversation_id),
                "timestamp": datetime.utcnow().isoformat()
            },
            priority=Priority.MEDIUM,  # Will be overridden if auto_priority=True
            confidence_score=confidence_score
        )

        # Add to queue
        added_item = self.review_queue.add_to_queue(
            review_item,
            auto_priority=auto_priority
        )

        # Initialize score history for this item
        self._score_history[added_item.item_id] = []

        logger.debug(
            f"Queued conversation for scoring: conversation_id={conversation_id}, "
            f"turns={review_item.content['turn_count']}, "
            f"confidence={confidence_score:.3f}"
        )

        return added_item


    def submit_scores(
        self,
        item_id: UUID,
        scores: Dict[str, float],
        reviewer_id: str,
        feedback: Optional[str] = None
    ) -> ReviewItem:
        """
        Submit quality scores for a simulated conversation.

        Collects human ratings across multiple quality dimensions to assess
        the realism and quality of simulated conversations.

        Args:
            item_id: UUID of the conversation item to score
            scores: Dictionary of dimension -> score (0-1)
                Should include scores for dimensions in self.scoring_dimensions
            reviewer_id: ID of the human reviewer
            feedback: Optional textual feedback about the conversation

        Returns:
            Updated ReviewItem

        Raises:
            ValueError: If scores format is invalid
            ConversationScoringError: If item not found or invalid operation

        Examples:
            >>> workflow = ConversationScoringWorkflow()
            >>> item = workflow.submit_scores(
            ...     item_id=some_uuid,
            ...     scores={
            ...         "realism": 0.85,
            ...         "coherence": 0.90,
            ...         "goal_achievement": 0.95,
            ...         "naturalness": 0.80
            ...     },
            ...     reviewer_id="reviewer_123",
            ...     feedback="Good conversation but agent could be more empathetic"
            ... )
            >>> item.human_scores
            {'realism': 0.85, 'coherence': 0.9, ...}
        """
        # Validate scores
        if not scores:
            raise ValueError("scores cannot be empty")

        for dimension, score in scores.items():
            if not 0.0 <= score <= 1.0:
                raise ValueError(
                    f"Score for '{dimension}' must be between 0 and 1, got {score}"
                )

        # Validate reviewer_id
        if not reviewer_id or not reviewer_id.strip():
            raise ValueError("reviewer_id cannot be empty")

        # Get the item
        try:
            item = self.review_queue.get_item(item_id)
        except Exception as e:
            raise ConversationScoringError(f"Failed to get item {item_id}: {e}")

        # Store scores in history (for aggregation across multiple reviewers)
        score_entry = {
            "reviewer_id": reviewer_id,
            "timestamp": datetime.utcnow().isoformat(),
            "scores": scores,
            "feedback": feedback
        }

        if item_id not in self._score_history:
            self._score_history[item_id] = []
        self._score_history[item_id].append(score_entry)

        # Update the review item with the latest scores
        try:
            updated_item = self.review_queue.submit_review(
                item_id=item_id,
                human_scores=scores,
                reviewer_id=reviewer_id,
                feedback=feedback
            )
        except Exception as e:
            raise ConversationScoringError(f"Failed to submit scores: {e}")

        logger.info(
            f"Submitted scores for conversation {item_id} from reviewer {reviewer_id}: "
            f"avg_score={np.mean(list(scores.values())):.3f}"
        )

        return updated_item


    def analyze_simulation_quality(
        self,
        time_period_days: int = 7,
        dimension: Optional[str] = None,
        min_conversation_count: int = 3
    ) -> List[QualityTrend]:
        """
        Analyze simulation quality trends over time.

        Tracks how conversation quality changes over time to identify:
        - Improving or declining simulation quality
        - Systematic issues in conversation generation
        - Patterns in quality across dimensions

        Args:
            time_period_days: Number of days to analyze
            dimension: Optional filter by specific dimension
            min_conversation_count: Minimum conversations required for trend

        Returns:
            List of QualityTrend objects, one per dimension

        Raises:
            InsufficientDataError: If insufficient data for analysis

        Examples:
            >>> workflow = ConversationScoringWorkflow()
            >>> trends = workflow.analyze_simulation_quality(
            ...     time_period_days=7,
            ...     dimension="realism"
            ... )
            >>> trends[0].trend_direction
            'improving'
            >>> trends[0].average_score
            0.85
            >>> trends[0].score_change
            0.05
        """
        # Get all simulated conversation items
        all_items = self.review_queue.get_all_items(
            item_type=ItemType.SIMULATED_CONVERSATION
        )

        # Filter by time period
        cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
        recent_items = [
            item for item in all_items
            if item.created_at >= cutoff_date and item.human_scores
        ]

        if len(recent_items) < min_conversation_count:
            raise InsufficientDataError(
                f"Insufficient conversations for quality analysis: "
                f"need at least {min_conversation_count}, got {len(recent_items)}"
            )

        # Collect scores by dimension
        scores_by_dimension: Dict[str, List[Tuple[datetime, float]]] = {}

        for item in recent_items:
            for dim, score in item.human_scores.items():
                # Filter by dimension if specified
                if dimension and dim != dimension:
                    continue

                if dim not in scores_by_dimension:
                    scores_by_dimension[dim] = []
                scores_by_dimension[dim].append((item.created_at, score))

        # Analyze trends for each dimension
        trends = []

        for dim, score_data in scores_by_dimension.items():
            if len(score_data) < min_conversation_count:
                continue

            # Sort by timestamp
            score_data.sort(key=lambda x: x[0])

            # Extract scores
            scores = [score for _, score in score_data]
            scores_arr = np.array(scores)

            # Calculate average score
            avg_score = float(np.mean(scores_arr))

            # Calculate trend (compare first half vs second half)
            mid_point = len(scores) // 2
            first_half_avg = float(np.mean(scores_arr[:mid_point]))
            second_half_avg = float(np.mean(scores_arr[mid_point:]))
            score_change = second_half_avg - first_half_avg

            # Determine trend direction
            if score_change > 0.05:
                trend_direction = "improving"
            elif score_change < -0.05:
                trend_direction = "declining"
            else:
                trend_direction = "stable"

            # Detect issues
            issues_detected = []

            # Check for low scores
            low_score_count = np.sum(scores_arr < 0.6)
            if low_score_count > len(scores) * 0.3:
                issues_detected.append(
                    f"High rate of low scores: {low_score_count}/{len(scores)} "
                    f"({low_score_count/len(scores):.1%})"
                )

            # Check for declining trend
            if trend_direction == "declining":
                issues_detected.append(
                    f"Declining trend detected: score dropped by {abs(score_change):.3f}"
                )

            # Check for high variance
            score_variance = float(np.var(scores_arr))
            if score_variance > 0.1:
                issues_detected.append(
                    f"High variance in scores: {score_variance:.3f} "
                    "(inconsistent quality)"
                )

            # Dimension-specific issues
            if dim == "realism" and avg_score < 0.7:
                issues_detected.append(
                    "Low realism scores suggest conversations feel artificial"
                )
            elif dim == "coherence" and avg_score < 0.7:
                issues_detected.append(
                    "Low coherence scores suggest logical flow issues"
                )
            elif dim == "goal_achievement" and avg_score < 0.7:
                issues_detected.append(
                    "Low goal achievement suggests conversations don't reach objectives"
                )
            elif dim == "naturalness" and avg_score < 0.7:
                issues_detected.append(
                    "Low naturalness scores suggest unnatural language patterns"
                )

            # Create trend object
            trend = QualityTrend(
                dimension=dim,
                time_period=f"last_{time_period_days}_days",
                average_score=avg_score,
                score_change=score_change,
                conversation_count=len(scores),
                trend_direction=trend_direction,
                issues_detected=issues_detected
            )

            trends.append(trend)

            logger.debug(
                f"Quality trend for {dim}: avg={avg_score:.3f}, "
                f"change={score_change:+.3f}, direction={trend_direction}"
            )

        if not trends:
            raise InsufficientDataError(
                f"No quality trends available for the specified criteria"
            )

        logger.info(
            f"Analyzed simulation quality over {time_period_days} days: "
            f"{len(trends)} dimensions analyzed"
        )

        return trends


    def get_improvement_suggestions(
        self,
        time_period_days: int = 30,
        min_conversation_count: int = 5
    ) -> Dict[str, List[str]]:
        """
        Generate improvement suggestions for the conversation simulator.

        Analyzes scored conversations to identify patterns and provide
        actionable recommendations for improving simulation quality.

        Args:
            time_period_days: Number of days to analyze
            min_conversation_count: Minimum conversations required

        Returns:
            Dictionary mapping dimension -> list of suggestions

        Raises:
            InsufficientDataError: If insufficient data for analysis

        Examples:
            >>> workflow = ConversationScoringWorkflow()
            >>> suggestions = workflow.get_improvement_suggestions()
            >>> suggestions["realism"]
            ['Add more varied user responses', 'Include edge cases']
            >>> suggestions["coherence"]
            ['Improve context tracking', 'Validate logical flow']
        """
        # Get all simulated conversation items
        all_items = self.review_queue.get_all_items(
            item_type=ItemType.SIMULATED_CONVERSATION
        )

        # Filter by time period
        cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
        recent_items = [
            item for item in all_items
            if item.created_at >= cutoff_date and item.human_scores
        ]

        if len(recent_items) < min_conversation_count:
            raise InsufficientDataError(
                f"Insufficient conversations for improvement suggestions: "
                f"need at least {min_conversation_count}, got {len(recent_items)}"
            )

        # Collect scores and feedback by dimension
        dimension_data: Dict[str, Dict[str, Any]] = {}

        for dim in self.scoring_dimensions:
            dimension_data[dim] = {
                "scores": [],
                "low_score_feedback": [],
                "high_score_feedback": []
            }

        for item in recent_items:
            for dim in self.scoring_dimensions:
                if dim in item.human_scores:
                    score = item.human_scores[dim]
                    dimension_data[dim]["scores"].append(score)

                    # Collect feedback for low and high scores
                    if item.feedback:
                        if score < 0.6:
                            dimension_data[dim]["low_score_feedback"].append(item.feedback)
                        elif score > 0.8:
                            dimension_data[dim]["high_score_feedback"].append(item.feedback)

        # Generate suggestions for each dimension
        suggestions: Dict[str, List[str]] = {}

        for dim, data in dimension_data.items():
            if not data["scores"]:
                continue

            dim_suggestions = []
            scores_arr = np.array(data["scores"])
            avg_score = float(np.mean(scores_arr))
            score_variance = float(np.var(scores_arr))

            # General suggestions based on average score
            if avg_score < 0.6:
                dim_suggestions.append(
                    f"Critical: {dim} scores are low (avg={avg_score:.2f}). "
                    "Major improvements needed."
                )
            elif avg_score < 0.7:
                dim_suggestions.append(
                    f"{dim} scores need improvement (avg={avg_score:.2f}). "
                    "Focus on quality enhancements."
                )

            # Suggestions based on variance
            if score_variance > 0.1:
                dim_suggestions.append(
                    f"High variance in {dim} scores (var={score_variance:.3f}). "
                    "Improve consistency across conversations."
                )

            # Dimension-specific suggestions
            if dim == "realism":
                if avg_score < 0.7:
                    dim_suggestions.extend([
                        "Add more varied and realistic user responses",
                        "Include edge cases and unexpected user behaviors",
                        "Model real user frustration and patience patterns",
                        "Use more natural language variations"
                    ])
                if score_variance > 0.1:
                    dim_suggestions.append(
                        "Standardize realism across different scenarios"
                    )

            elif dim == "coherence":
                if avg_score < 0.7:
                    dim_suggestions.extend([
                        "Improve context tracking across conversation turns",
                        "Validate logical flow between messages",
                        "Ensure agent responses address user questions directly",
                        "Maintain consistency in conversation state"
                    ])
                if score_variance > 0.1:
                    dim_suggestions.append(
                        "Apply coherence checks consistently across all conversations"
                    )

            elif dim == "goal_achievement":
                if avg_score < 0.7:
                    dim_suggestions.extend([
                        "Improve goal tracking and completion logic",
                        "Add clearer success criteria for scenarios",
                        "Ensure conversations progress toward goals",
                        "Validate goal achievement before termination"
                    ])
                # Check actual goal achievement rate
                goal_achieved_count = sum(
                    1 for item in recent_items
                    if item.content.get("goal_achieved", False)
                )
                goal_rate = goal_achieved_count / len(recent_items)
                if goal_rate < 0.5:
                    dim_suggestions.append(
                        f"Low goal achievement rate ({goal_rate:.1%}). "
                        "Review scenario difficulty and agent capabilities."
                    )

            elif dim == "naturalness":
                if avg_score < 0.7:
                    dim_suggestions.extend([
                        "Use more natural language patterns",
                        "Reduce overly formal or robotic phrasing",
                        "Add conversational fillers and acknowledgments",
                        "Vary sentence structure and length"
                    ])
                if score_variance > 0.1:
                    dim_suggestions.append(
                        "Maintain consistent naturalness across user models"
                    )

            # Add feedback-based suggestions
            if data["low_score_feedback"]:
                dim_suggestions.append(
                    f"Review {len(data['low_score_feedback'])} low-score feedback items "
                    "for specific improvement areas"
                )

            if data["high_score_feedback"]:
                dim_suggestions.append(
                    f"Analyze {len(data['high_score_feedback'])} high-score examples "
                    "to identify successful patterns"
                )

            suggestions[dim] = dim_suggestions

        logger.info(
            f"Generated improvement suggestions for {len(suggestions)} dimensions "
            f"based on {len(recent_items)} conversations"
        )

        return suggestions


    def aggregate_scores(
        self,
        item_id: UUID,
        aggregation_method: str = "mean"
    ) -> Dict[str, Any]:
        """
        Aggregate scores across multiple reviewers for a conversation.

        Combines scores from multiple reviewers to produce consensus ratings
        and identify areas of agreement/disagreement.

        Args:
            item_id: UUID of the conversation to aggregate scores for
            aggregation_method: Method for aggregation ("mean", "median", "weighted")

        Returns:
            Dictionary containing:
                - consensus_scores: Aggregated scores for each dimension
                - reviewer_count: Number of reviewers
                - score_variance: Variance in scores across reviewers
                - agreement_level: Level of agreement ("high", "medium", "low")
                - individual_scores: List of individual score entries

        Raises:
            ConversationScoringError: If item not found
            InsufficientDataError: If no scores available

        Examples:
            >>> workflow = ConversationScoringWorkflow()
            >>> # After multiple reviewers submit scores...
            >>> aggregated = workflow.aggregate_scores(item_id)
            >>> aggregated["consensus_scores"]
            {'realism': 0.85, 'coherence': 0.88, 'goal_achievement': 0.90}
            >>> aggregated["agreement_level"]
            'high'
        """
        # Validate aggregation method
        if aggregation_method not in ["mean", "median", "weighted"]:
            raise ValueError(
                f"aggregation_method must be 'mean', 'median', or 'weighted', "
                f"got '{aggregation_method}'"
            )

        # Get score history
        if item_id not in self._score_history:
            raise ConversationScoringError(
                f"No score history found for conversation {item_id}"
            )

        score_entries = self._score_history[item_id]

        if not score_entries:
            raise InsufficientDataError(
                f"No scores available for conversation {item_id}"
            )

        # Collect all scores by dimension
        scores_by_dimension: Dict[str, List[float]] = {}

        for entry in score_entries:
            for dimension, score in entry["scores"].items():
                if dimension not in scores_by_dimension:
                    scores_by_dimension[dimension] = []
                scores_by_dimension[dimension].append(score)

        # Aggregate scores
        consensus_scores = {}
        score_variance = {}

        for dimension, scores in scores_by_dimension.items():
            scores_arr = np.array(scores)

            if aggregation_method == "mean":
                consensus_scores[dimension] = float(np.mean(scores_arr))
            elif aggregation_method == "median":
                consensus_scores[dimension] = float(np.median(scores_arr))
            elif aggregation_method == "weighted":
                # For weighted, use recency weighting (more recent = higher weight)
                weights = np.linspace(0.5, 1.0, len(scores))
                consensus_scores[dimension] = float(
                    np.average(scores_arr, weights=weights)
                )

            # Calculate variance
            score_variance[dimension] = float(np.var(scores_arr))

        # Calculate overall agreement level
        avg_variance = np.mean(list(score_variance.values())) if score_variance else 0.0

        if avg_variance < 0.01:
            agreement_level = "high"
        elif avg_variance < 0.05:
            agreement_level = "medium"
        else:
            agreement_level = "low"

        # Build result
        result = {
            "item_id": str(item_id),
            "consensus_scores": consensus_scores,
            "reviewer_count": len(score_entries),
            "score_variance": score_variance,
            "average_variance": float(avg_variance),
            "agreement_level": agreement_level,
            "aggregation_method": aggregation_method,
            "individual_scores": score_entries,
            "timestamp": datetime.utcnow().isoformat()
        }

        logger.info(
            f"Aggregated scores for conversation {item_id}: "
            f"{len(score_entries)} reviewers, "
            f"agreement={agreement_level}"
        )

        return result

    def get_scoring_summary(
        self,
        time_period_days: Optional[int] = None
    ) -> Dict[str, Any]:
        """
        Get a summary of conversation scoring status.

        Provides an overview of:
        - Total conversations scored
        - Average scores by dimension
        - Scoring progress
        - Quality trends

        Args:
            time_period_days: Optional time period for filtering (default: all time)

        Returns:
            Dictionary with scoring summary

        Examples:
            >>> workflow = ConversationScoringWorkflow()
            >>> summary = workflow.get_scoring_summary(time_period_days=7)
            >>> summary["total_conversations"]
            42
            >>> summary["average_scores"]["realism"]
            0.85
            >>> summary["quality_status"]
            'good'
        """
        # Get all simulated conversation items
        all_items = self.review_queue.get_all_items(
            item_type=ItemType.SIMULATED_CONVERSATION
        )

        # Filter by time period if specified
        if time_period_days:
            cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
            items = [item for item in all_items if item.created_at >= cutoff_date]
        else:
            items = all_items

        # Calculate average scores by dimension
        scores_by_dimension: Dict[str, List[float]] = {}

        for item in items:
            if item.human_scores:
                for dimension, score in item.human_scores.items():
                    if dimension not in scores_by_dimension:
                        scores_by_dimension[dimension] = []
                    scores_by_dimension[dimension].append(score)

        average_scores = {
            dimension: float(np.mean(scores))
            for dimension, scores in scores_by_dimension.items()
        }

        # Calculate overall quality status
        if average_scores:
            overall_avg = np.mean(list(average_scores.values()))
            if overall_avg >= 0.8:
                quality_status = "excellent"
            elif overall_avg >= 0.7:
                quality_status = "good"
            elif overall_avg >= 0.6:
                quality_status = "fair"
            else:
                quality_status = "needs_improvement"
        else:
            quality_status = "unknown"

        # Get queue stats
        queue_stats = self.review_queue.get_queue_stats()

        # Calculate goal achievement rate
        goal_achieved_count = sum(
            1 for item in items
            if item.content.get("goal_achieved", False)
        )
        goal_achievement_rate = (
            goal_achieved_count / len(items) if items else 0.0
        )

        return {
            "timestamp": datetime.utcnow().isoformat(),
            "time_period": f"last_{time_period_days}_days" if time_period_days else "all_time",
            "total_conversations": len(items),
            "pending_scoring": queue_stats["by_type"]["simulated_conversation"],
            "completed_scoring": len([item for item in items if item.human_scores]),
            "average_scores": average_scores,
            "quality_status": quality_status,
            "goal_achievement_rate": float(goal_achievement_rate),
            "dimensions_tracked": list(scores_by_dimension.keys()),
            "total_reviewers": len(set(
                item.reviewer_id for item in items if item.reviewer_id
            ))
        }

aggregate_scores(item_id, aggregation_method='mean')

Aggregate scores across multiple reviewers for a conversation.

Combines scores from multiple reviewers to produce consensus ratings and identify areas of agreement/disagreement.

Parameters:

Name	Type	Description	Default
item_id	UUID	UUID of the conversation to aggregate scores for	required
aggregation_method	str	Method for aggregation ("mean", "median", "weighted")	'mean'

Returns:

Type	Description
Dict[str, Any]	Dictionary containing: - consensus_scores: Aggregated scores for each dimension - reviewer_count: Number of reviewers - score_variance: Variance in scores across reviewers - agreement_level: Level of agreement ("high", "medium", "low") - individual_scores: List of individual score entries

Raises:

Type	Description
ConversationScoringError	If item not found
InsufficientDataError	If no scores available

Examples:

>>> workflow = ConversationScoringWorkflow()
>>> # After multiple reviewers submit scores...
>>> aggregated = workflow.aggregate_scores(item_id)
>>> aggregated["consensus_scores"]
{'realism': 0.85, 'coherence': 0.88, 'goal_achievement': 0.90}
>>> aggregated["agreement_level"]
'high'

Source code in uaef/hitl/conversation_scoring.py

def aggregate_scores(
    self,
    item_id: UUID,
    aggregation_method: str = "mean"
) -> Dict[str, Any]:
    """
    Aggregate scores across multiple reviewers for a conversation.

    Combines scores from multiple reviewers to produce consensus ratings
    and identify areas of agreement/disagreement.

    Args:
        item_id: UUID of the conversation to aggregate scores for
        aggregation_method: Method for aggregation ("mean", "median", "weighted")

    Returns:
        Dictionary containing:
            - consensus_scores: Aggregated scores for each dimension
            - reviewer_count: Number of reviewers
            - score_variance: Variance in scores across reviewers
            - agreement_level: Level of agreement ("high", "medium", "low")
            - individual_scores: List of individual score entries

    Raises:
        ConversationScoringError: If item not found
        InsufficientDataError: If no scores available

    Examples:
        >>> workflow = ConversationScoringWorkflow()
        >>> # After multiple reviewers submit scores...
        >>> aggregated = workflow.aggregate_scores(item_id)
        >>> aggregated["consensus_scores"]
        {'realism': 0.85, 'coherence': 0.88, 'goal_achievement': 0.90}
        >>> aggregated["agreement_level"]
        'high'
    """
    # Validate aggregation method
    if aggregation_method not in ["mean", "median", "weighted"]:
        raise ValueError(
            f"aggregation_method must be 'mean', 'median', or 'weighted', "
            f"got '{aggregation_method}'"
        )

    # Get score history
    if item_id not in self._score_history:
        raise ConversationScoringError(
            f"No score history found for conversation {item_id}"
        )

    score_entries = self._score_history[item_id]

    if not score_entries:
        raise InsufficientDataError(
            f"No scores available for conversation {item_id}"
        )

    # Collect all scores by dimension
    scores_by_dimension: Dict[str, List[float]] = {}

    for entry in score_entries:
        for dimension, score in entry["scores"].items():
            if dimension not in scores_by_dimension:
                scores_by_dimension[dimension] = []
            scores_by_dimension[dimension].append(score)

    # Aggregate scores
    consensus_scores = {}
    score_variance = {}

    for dimension, scores in scores_by_dimension.items():
        scores_arr = np.array(scores)

        if aggregation_method == "mean":
            consensus_scores[dimension] = float(np.mean(scores_arr))
        elif aggregation_method == "median":
            consensus_scores[dimension] = float(np.median(scores_arr))
        elif aggregation_method == "weighted":
            # For weighted, use recency weighting (more recent = higher weight)
            weights = np.linspace(0.5, 1.0, len(scores))
            consensus_scores[dimension] = float(
                np.average(scores_arr, weights=weights)
            )

        # Calculate variance
        score_variance[dimension] = float(np.var(scores_arr))

    # Calculate overall agreement level
    avg_variance = np.mean(list(score_variance.values())) if score_variance else 0.0

    if avg_variance < 0.01:
        agreement_level = "high"
    elif avg_variance < 0.05:
        agreement_level = "medium"
    else:
        agreement_level = "low"

    # Build result
    result = {
        "item_id": str(item_id),
        "consensus_scores": consensus_scores,
        "reviewer_count": len(score_entries),
        "score_variance": score_variance,
        "average_variance": float(avg_variance),
        "agreement_level": agreement_level,
        "aggregation_method": aggregation_method,
        "individual_scores": score_entries,
        "timestamp": datetime.utcnow().isoformat()
    }

    logger.info(
        f"Aggregated scores for conversation {item_id}: "
        f"{len(score_entries)} reviewers, "
        f"agreement={agreement_level}"
    )

    return result

analyze_simulation_quality(time_period_days=7, dimension=None, min_conversation_count=3)

Analyze simulation quality trends over time.

Tracks how conversation quality changes over time to identify: - Improving or declining simulation quality - Systematic issues in conversation generation - Patterns in quality across dimensions

Parameters:

Name	Type	Description	Default
time_period_days	int	Number of days to analyze	7
dimension	Optional[str]	Optional filter by specific dimension	None
min_conversation_count	int	Minimum conversations required for trend	3

Returns:

Type	Description
List[QualityTrend]	List of QualityTrend objects, one per dimension

Raises:

Type	Description
InsufficientDataError	If insufficient data for analysis

Examples:

>>> workflow = ConversationScoringWorkflow()
>>> trends = workflow.analyze_simulation_quality(
...     time_period_days=7,
...     dimension="realism"
... )
>>> trends[0].trend_direction
'improving'
>>> trends[0].average_score
0.85
>>> trends[0].score_change
0.05

Source code in uaef/hitl/conversation_scoring.py

def analyze_simulation_quality(
    self,
    time_period_days: int = 7,
    dimension: Optional[str] = None,
    min_conversation_count: int = 3
) -> List[QualityTrend]:
    """
    Analyze simulation quality trends over time.

    Tracks how conversation quality changes over time to identify:
    - Improving or declining simulation quality
    - Systematic issues in conversation generation
    - Patterns in quality across dimensions

    Args:
        time_period_days: Number of days to analyze
        dimension: Optional filter by specific dimension
        min_conversation_count: Minimum conversations required for trend

    Returns:
        List of QualityTrend objects, one per dimension

    Raises:
        InsufficientDataError: If insufficient data for analysis

    Examples:
        >>> workflow = ConversationScoringWorkflow()
        >>> trends = workflow.analyze_simulation_quality(
        ...     time_period_days=7,
        ...     dimension="realism"
        ... )
        >>> trends[0].trend_direction
        'improving'
        >>> trends[0].average_score
        0.85
        >>> trends[0].score_change
        0.05
    """
    # Get all simulated conversation items
    all_items = self.review_queue.get_all_items(
        item_type=ItemType.SIMULATED_CONVERSATION
    )

    # Filter by time period
    cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
    recent_items = [
        item for item in all_items
        if item.created_at >= cutoff_date and item.human_scores
    ]

    if len(recent_items) < min_conversation_count:
        raise InsufficientDataError(
            f"Insufficient conversations for quality analysis: "
            f"need at least {min_conversation_count}, got {len(recent_items)}"
        )

    # Collect scores by dimension
    scores_by_dimension: Dict[str, List[Tuple[datetime, float]]] = {}

    for item in recent_items:
        for dim, score in item.human_scores.items():
            # Filter by dimension if specified
            if dimension and dim != dimension:
                continue

            if dim not in scores_by_dimension:
                scores_by_dimension[dim] = []
            scores_by_dimension[dim].append((item.created_at, score))

    # Analyze trends for each dimension
    trends = []

    for dim, score_data in scores_by_dimension.items():
        if len(score_data) < min_conversation_count:
            continue

        # Sort by timestamp
        score_data.sort(key=lambda x: x[0])

        # Extract scores
        scores = [score for _, score in score_data]
        scores_arr = np.array(scores)

        # Calculate average score
        avg_score = float(np.mean(scores_arr))

        # Calculate trend (compare first half vs second half)
        mid_point = len(scores) // 2
        first_half_avg = float(np.mean(scores_arr[:mid_point]))
        second_half_avg = float(np.mean(scores_arr[mid_point:]))
        score_change = second_half_avg - first_half_avg

        # Determine trend direction
        if score_change > 0.05:
            trend_direction = "improving"
        elif score_change < -0.05:
            trend_direction = "declining"
        else:
            trend_direction = "stable"

        # Detect issues
        issues_detected = []

        # Check for low scores
        low_score_count = np.sum(scores_arr < 0.6)
        if low_score_count > len(scores) * 0.3:
            issues_detected.append(
                f"High rate of low scores: {low_score_count}/{len(scores)} "
                f"({low_score_count/len(scores):.1%})"
            )

        # Check for declining trend
        if trend_direction == "declining":
            issues_detected.append(
                f"Declining trend detected: score dropped by {abs(score_change):.3f}"
            )

        # Check for high variance
        score_variance = float(np.var(scores_arr))
        if score_variance > 0.1:
            issues_detected.append(
                f"High variance in scores: {score_variance:.3f} "
                "(inconsistent quality)"
            )

        # Dimension-specific issues
        if dim == "realism" and avg_score < 0.7:
            issues_detected.append(
                "Low realism scores suggest conversations feel artificial"
            )
        elif dim == "coherence" and avg_score < 0.7:
            issues_detected.append(
                "Low coherence scores suggest logical flow issues"
            )
        elif dim == "goal_achievement" and avg_score < 0.7:
            issues_detected.append(
                "Low goal achievement suggests conversations don't reach objectives"
            )
        elif dim == "naturalness" and avg_score < 0.7:
            issues_detected.append(
                "Low naturalness scores suggest unnatural language patterns"
            )

        # Create trend object
        trend = QualityTrend(
            dimension=dim,
            time_period=f"last_{time_period_days}_days",
            average_score=avg_score,
            score_change=score_change,
            conversation_count=len(scores),
            trend_direction=trend_direction,
            issues_detected=issues_detected
        )

        trends.append(trend)

        logger.debug(
            f"Quality trend for {dim}: avg={avg_score:.3f}, "
            f"change={score_change:+.3f}, direction={trend_direction}"
        )

    if not trends:
        raise InsufficientDataError(
            f"No quality trends available for the specified criteria"
        )

    logger.info(
        f"Analyzed simulation quality over {time_period_days} days: "
        f"{len(trends)} dimensions analyzed"
    )

    return trends

get_improvement_suggestions(time_period_days=30, min_conversation_count=5)

Generate improvement suggestions for the conversation simulator.

Analyzes scored conversations to identify patterns and provide actionable recommendations for improving simulation quality.

Parameters:

Name	Type	Description	Default
time_period_days	int	Number of days to analyze	30
min_conversation_count	int	Minimum conversations required	5

Returns:

Type	Description
Dict[str, List[str]]	Dictionary mapping dimension -> list of suggestions

Raises:

Type	Description
InsufficientDataError	If insufficient data for analysis

Examples:

>>> workflow = ConversationScoringWorkflow()
>>> suggestions = workflow.get_improvement_suggestions()
>>> suggestions["realism"]
['Add more varied user responses', 'Include edge cases']
>>> suggestions["coherence"]
['Improve context tracking', 'Validate logical flow']

Source code in uaef/hitl/conversation_scoring.py

def get_improvement_suggestions(
    self,
    time_period_days: int = 30,
    min_conversation_count: int = 5
) -> Dict[str, List[str]]:
    """
    Generate improvement suggestions for the conversation simulator.

    Analyzes scored conversations to identify patterns and provide
    actionable recommendations for improving simulation quality.

    Args:
        time_period_days: Number of days to analyze
        min_conversation_count: Minimum conversations required

    Returns:
        Dictionary mapping dimension -> list of suggestions

    Raises:
        InsufficientDataError: If insufficient data for analysis

    Examples:
        >>> workflow = ConversationScoringWorkflow()
        >>> suggestions = workflow.get_improvement_suggestions()
        >>> suggestions["realism"]
        ['Add more varied user responses', 'Include edge cases']
        >>> suggestions["coherence"]
        ['Improve context tracking', 'Validate logical flow']
    """
    # Get all simulated conversation items
    all_items = self.review_queue.get_all_items(
        item_type=ItemType.SIMULATED_CONVERSATION
    )

    # Filter by time period
    cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
    recent_items = [
        item for item in all_items
        if item.created_at >= cutoff_date and item.human_scores
    ]

    if len(recent_items) < min_conversation_count:
        raise InsufficientDataError(
            f"Insufficient conversations for improvement suggestions: "
            f"need at least {min_conversation_count}, got {len(recent_items)}"
        )

    # Collect scores and feedback by dimension
    dimension_data: Dict[str, Dict[str, Any]] = {}

    for dim in self.scoring_dimensions:
        dimension_data[dim] = {
            "scores": [],
            "low_score_feedback": [],
            "high_score_feedback": []
        }

    for item in recent_items:
        for dim in self.scoring_dimensions:
            if dim in item.human_scores:
                score = item.human_scores[dim]
                dimension_data[dim]["scores"].append(score)

                # Collect feedback for low and high scores
                if item.feedback:
                    if score < 0.6:
                        dimension_data[dim]["low_score_feedback"].append(item.feedback)
                    elif score > 0.8:
                        dimension_data[dim]["high_score_feedback"].append(item.feedback)

    # Generate suggestions for each dimension
    suggestions: Dict[str, List[str]] = {}

    for dim, data in dimension_data.items():
        if not data["scores"]:
            continue

        dim_suggestions = []
        scores_arr = np.array(data["scores"])
        avg_score = float(np.mean(scores_arr))
        score_variance = float(np.var(scores_arr))

        # General suggestions based on average score
        if avg_score < 0.6:
            dim_suggestions.append(
                f"Critical: {dim} scores are low (avg={avg_score:.2f}). "
                "Major improvements needed."
            )
        elif avg_score < 0.7:
            dim_suggestions.append(
                f"{dim} scores need improvement (avg={avg_score:.2f}). "
                "Focus on quality enhancements."
            )

        # Suggestions based on variance
        if score_variance > 0.1:
            dim_suggestions.append(
                f"High variance in {dim} scores (var={score_variance:.3f}). "
                "Improve consistency across conversations."
            )

        # Dimension-specific suggestions
        if dim == "realism":
            if avg_score < 0.7:
                dim_suggestions.extend([
                    "Add more varied and realistic user responses",
                    "Include edge cases and unexpected user behaviors",
                    "Model real user frustration and patience patterns",
                    "Use more natural language variations"
                ])
            if score_variance > 0.1:
                dim_suggestions.append(
                    "Standardize realism across different scenarios"
                )

        elif dim == "coherence":
            if avg_score < 0.7:
                dim_suggestions.extend([
                    "Improve context tracking across conversation turns",
                    "Validate logical flow between messages",
                    "Ensure agent responses address user questions directly",
                    "Maintain consistency in conversation state"
                ])
            if score_variance > 0.1:
                dim_suggestions.append(
                    "Apply coherence checks consistently across all conversations"
                )

        elif dim == "goal_achievement":
            if avg_score < 0.7:
                dim_suggestions.extend([
                    "Improve goal tracking and completion logic",
                    "Add clearer success criteria for scenarios",
                    "Ensure conversations progress toward goals",
                    "Validate goal achievement before termination"
                ])
            # Check actual goal achievement rate
            goal_achieved_count = sum(
                1 for item in recent_items
                if item.content.get("goal_achieved", False)
            )
            goal_rate = goal_achieved_count / len(recent_items)
            if goal_rate < 0.5:
                dim_suggestions.append(
                    f"Low goal achievement rate ({goal_rate:.1%}). "
                    "Review scenario difficulty and agent capabilities."
                )

        elif dim == "naturalness":
            if avg_score < 0.7:
                dim_suggestions.extend([
                    "Use more natural language patterns",
                    "Reduce overly formal or robotic phrasing",
                    "Add conversational fillers and acknowledgments",
                    "Vary sentence structure and length"
                ])
            if score_variance > 0.1:
                dim_suggestions.append(
                    "Maintain consistent naturalness across user models"
                )

        # Add feedback-based suggestions
        if data["low_score_feedback"]:
            dim_suggestions.append(
                f"Review {len(data['low_score_feedback'])} low-score feedback items "
                "for specific improvement areas"
            )

        if data["high_score_feedback"]:
            dim_suggestions.append(
                f"Analyze {len(data['high_score_feedback'])} high-score examples "
                "to identify successful patterns"
            )

        suggestions[dim] = dim_suggestions

    logger.info(
        f"Generated improvement suggestions for {len(suggestions)} dimensions "
        f"based on {len(recent_items)} conversations"
    )

    return suggestions

get_scoring_summary(time_period_days=None)

Get a summary of conversation scoring status.

Provides an overview of: - Total conversations scored - Average scores by dimension - Scoring progress - Quality trends

Parameters:

Name	Type	Description	Default
time_period_days	Optional[int]	Optional time period for filtering (default: all time)	None

Returns:

Type	Description
Dict[str, Any]	Dictionary with scoring summary

Examples:

>>> workflow = ConversationScoringWorkflow()
>>> summary = workflow.get_scoring_summary(time_period_days=7)
>>> summary["total_conversations"]
42
>>> summary["average_scores"]["realism"]
0.85
>>> summary["quality_status"]
'good'

Source code in uaef/hitl/conversation_scoring.py

def get_scoring_summary(
    self,
    time_period_days: Optional[int] = None
) -> Dict[str, Any]:
    """
    Get a summary of conversation scoring status.

    Provides an overview of:
    - Total conversations scored
    - Average scores by dimension
    - Scoring progress
    - Quality trends

    Args:
        time_period_days: Optional time period for filtering (default: all time)

    Returns:
        Dictionary with scoring summary

    Examples:
        >>> workflow = ConversationScoringWorkflow()
        >>> summary = workflow.get_scoring_summary(time_period_days=7)
        >>> summary["total_conversations"]
        42
        >>> summary["average_scores"]["realism"]
        0.85
        >>> summary["quality_status"]
        'good'
    """
    # Get all simulated conversation items
    all_items = self.review_queue.get_all_items(
        item_type=ItemType.SIMULATED_CONVERSATION
    )

    # Filter by time period if specified
    if time_period_days:
        cutoff_date = datetime.utcnow() - timedelta(days=time_period_days)
        items = [item for item in all_items if item.created_at >= cutoff_date]
    else:
        items = all_items

    # Calculate average scores by dimension
    scores_by_dimension: Dict[str, List[float]] = {}

    for item in items:
        if item.human_scores:
            for dimension, score in item.human_scores.items():
                if dimension not in scores_by_dimension:
                    scores_by_dimension[dimension] = []
                scores_by_dimension[dimension].append(score)

    average_scores = {
        dimension: float(np.mean(scores))
        for dimension, scores in scores_by_dimension.items()
    }

    # Calculate overall quality status
    if average_scores:
        overall_avg = np.mean(list(average_scores.values()))
        if overall_avg >= 0.8:
            quality_status = "excellent"
        elif overall_avg >= 0.7:
            quality_status = "good"
        elif overall_avg >= 0.6:
            quality_status = "fair"
        else:
            quality_status = "needs_improvement"
    else:
        quality_status = "unknown"

    # Get queue stats
    queue_stats = self.review_queue.get_queue_stats()

    # Calculate goal achievement rate
    goal_achieved_count = sum(
        1 for item in items
        if item.content.get("goal_achieved", False)
    )
    goal_achievement_rate = (
        goal_achieved_count / len(items) if items else 0.0
    )

    return {
        "timestamp": datetime.utcnow().isoformat(),
        "time_period": f"last_{time_period_days}_days" if time_period_days else "all_time",
        "total_conversations": len(items),
        "pending_scoring": queue_stats["by_type"]["simulated_conversation"],
        "completed_scoring": len([item for item in items if item.human_scores]),
        "average_scores": average_scores,
        "quality_status": quality_status,
        "goal_achievement_rate": float(goal_achievement_rate),
        "dimensions_tracked": list(scores_by_dimension.keys()),
        "total_reviewers": len(set(
            item.reviewer_id for item in items if item.reviewer_id
        ))
    }

queue_conversation(conversation_data, auto_priority=True)

Queue a simulated conversation for human scoring.

Adds a simulated conversation to the review queue for quality assessment. This is typically called after the conversation simulator generates a conversation.

Parameters:

Name	Type	Description	Default
conversation_data	Dict[str, Any]	Dictionary containing: - conversation_id: Unique identifier for the conversation - scenario: Description of the scenario - conversation: List of message dictionaries with role and content - goal_achieved: Whether the conversation achieved its goal (optional) - turn_count: Number of turns in the conversation (optional) - user_model: User behavior model used (optional) - metadata: Additional metadata (optional) - confidence_score: Confidence in simulation quality (optional, default: 0.5)	required
auto_priority	bool	If True, automatically assign priority based on confidence	True

Returns:

Type	Description
ReviewItem	ReviewItem that was added to the queue

Raises:

Type	Description
ValueError	If conversation_data format is invalid

Examples:

>>> workflow = ConversationScoringWorkflow()
>>> conversation = {
...     "conversation_id": "conv_001",
...     "scenario": "Customer support - refund request",
...     "conversation": [
...         {"role": "user", "content": "I want a refund"},
...         {"role": "agent", "content": "I can help with that..."}
...     ],
...     "goal_achieved": True,
...     "turn_count": 5,
...     "confidence_score": 0.75
... }
>>> item = workflow.queue_conversation(conversation)
>>> item.item_type
<ItemType.SIMULATED_CONVERSATION: 'simulated_conversation'>

Source code in uaef/hitl/conversation_scoring.py

def queue_conversation(
    self,
    conversation_data: Dict[str, Any],
    auto_priority: bool = True
) -> ReviewItem:
    """
    Queue a simulated conversation for human scoring.

    Adds a simulated conversation to the review queue for quality assessment.
    This is typically called after the conversation simulator generates a conversation.

    Args:
        conversation_data: Dictionary containing:
            - conversation_id: Unique identifier for the conversation
            - scenario: Description of the scenario
            - conversation: List of message dictionaries with role and content
            - goal_achieved: Whether the conversation achieved its goal (optional)
            - turn_count: Number of turns in the conversation (optional)
            - user_model: User behavior model used (optional)
            - metadata: Additional metadata (optional)
            - confidence_score: Confidence in simulation quality (optional, default: 0.5)
        auto_priority: If True, automatically assign priority based on confidence

    Returns:
        ReviewItem that was added to the queue

    Raises:
        ValueError: If conversation_data format is invalid

    Examples:
        >>> workflow = ConversationScoringWorkflow()
        >>> conversation = {
        ...     "conversation_id": "conv_001",
        ...     "scenario": "Customer support - refund request",
        ...     "conversation": [
        ...         {"role": "user", "content": "I want a refund"},
        ...         {"role": "agent", "content": "I can help with that..."}
        ...     ],
        ...     "goal_achieved": True,
        ...     "turn_count": 5,
        ...     "confidence_score": 0.75
        ... }
        >>> item = workflow.queue_conversation(conversation)
        >>> item.item_type
        <ItemType.SIMULATED_CONVERSATION: 'simulated_conversation'>
    """
    # Validate required fields
    if "conversation_id" not in conversation_data:
        raise ValueError("conversation_data must contain 'conversation_id'")
    if "scenario" not in conversation_data:
        raise ValueError("conversation_data must contain 'scenario'")
    if "conversation" not in conversation_data:
        raise ValueError("conversation_data must contain 'conversation'")

    conversation_id = conversation_data["conversation_id"]
    confidence_score = conversation_data.get("confidence_score", 0.5)

    # Create review item
    review_item = ReviewItem(
        item_type=ItemType.SIMULATED_CONVERSATION,
        content={
            "scenario": conversation_data["scenario"],
            "conversation": conversation_data["conversation"],
            "goal_achieved": conversation_data.get("goal_achieved", None),
            "turn_count": conversation_data.get("turn_count", len(conversation_data["conversation"])),
            "user_model": conversation_data.get("user_model", ""),
            "metadata": conversation_data.get("metadata", {})
        },
        metadata={
            "conversation_id": str(conversation_id),
            "timestamp": datetime.utcnow().isoformat()
        },
        priority=Priority.MEDIUM,  # Will be overridden if auto_priority=True
        confidence_score=confidence_score
    )

    # Add to queue
    added_item = self.review_queue.add_to_queue(
        review_item,
        auto_priority=auto_priority
    )

    # Initialize score history for this item
    self._score_history[added_item.item_id] = []

    logger.debug(
        f"Queued conversation for scoring: conversation_id={conversation_id}, "
        f"turns={review_item.content['turn_count']}, "
        f"confidence={confidence_score:.3f}"
    )

    return added_item

submit_scores(item_id, scores, reviewer_id, feedback=None)

Submit quality scores for a simulated conversation.

Collects human ratings across multiple quality dimensions to assess the realism and quality of simulated conversations.

Parameters:

Name	Type	Description	Default
item_id	UUID	UUID of the conversation item to score	required
scores	Dict[str, float]	Dictionary of dimension -> score (0-1) Should include scores for dimensions in self.scoring_dimensions	required
reviewer_id	str	ID of the human reviewer	required
feedback	Optional[str]	Optional textual feedback about the conversation	None

Returns:

Type	Description
ReviewItem	Updated ReviewItem

Raises:

Type	Description
ValueError	If scores format is invalid
ConversationScoringError	If item not found or invalid operation

Examples:

>>> workflow = ConversationScoringWorkflow()
>>> item = workflow.submit_scores(
...     item_id=some_uuid,
...     scores={
...         "realism": 0.85,
...         "coherence": 0.90,
...         "goal_achievement": 0.95,
...         "naturalness": 0.80
...     },
...     reviewer_id="reviewer_123",
...     feedback="Good conversation but agent could be more empathetic"
... )
>>> item.human_scores
{'realism': 0.85, 'coherence': 0.9, ...}

Source code in uaef/hitl/conversation_scoring.py

def submit_scores(
    self,
    item_id: UUID,
    scores: Dict[str, float],
    reviewer_id: str,
    feedback: Optional[str] = None
) -> ReviewItem:
    """
    Submit quality scores for a simulated conversation.

    Collects human ratings across multiple quality dimensions to assess
    the realism and quality of simulated conversations.

    Args:
        item_id: UUID of the conversation item to score
        scores: Dictionary of dimension -> score (0-1)
            Should include scores for dimensions in self.scoring_dimensions
        reviewer_id: ID of the human reviewer
        feedback: Optional textual feedback about the conversation

    Returns:
        Updated ReviewItem

    Raises:
        ValueError: If scores format is invalid
        ConversationScoringError: If item not found or invalid operation

    Examples:
        >>> workflow = ConversationScoringWorkflow()
        >>> item = workflow.submit_scores(
        ...     item_id=some_uuid,
        ...     scores={
        ...         "realism": 0.85,
        ...         "coherence": 0.90,
        ...         "goal_achievement": 0.95,
        ...         "naturalness": 0.80
        ...     },
        ...     reviewer_id="reviewer_123",
        ...     feedback="Good conversation but agent could be more empathetic"
        ... )
        >>> item.human_scores
        {'realism': 0.85, 'coherence': 0.9, ...}
    """
    # Validate scores
    if not scores:
        raise ValueError("scores cannot be empty")

    for dimension, score in scores.items():
        if not 0.0 <= score <= 1.0:
            raise ValueError(
                f"Score for '{dimension}' must be between 0 and 1, got {score}"
            )

    # Validate reviewer_id
    if not reviewer_id or not reviewer_id.strip():
        raise ValueError("reviewer_id cannot be empty")

    # Get the item
    try:
        item = self.review_queue.get_item(item_id)
    except Exception as e:
        raise ConversationScoringError(f"Failed to get item {item_id}: {e}")

    # Store scores in history (for aggregation across multiple reviewers)
    score_entry = {
        "reviewer_id": reviewer_id,
        "timestamp": datetime.utcnow().isoformat(),
        "scores": scores,
        "feedback": feedback
    }

    if item_id not in self._score_history:
        self._score_history[item_id] = []
    self._score_history[item_id].append(score_entry)

    # Update the review item with the latest scores
    try:
        updated_item = self.review_queue.submit_review(
            item_id=item_id,
            human_scores=scores,
            reviewer_id=reviewer_id,
            feedback=feedback
        )
    except Exception as e:
        raise ConversationScoringError(f"Failed to submit scores: {e}")

    logger.info(
        f"Submitted scores for conversation {item_id} from reviewer {reviewer_id}: "
        f"avg_score={np.mean(list(scores.values())):.3f}"
    )

    return updated_item

InsufficientDataError

Bases: ConversationScoringError

Raised when insufficient data is available for analysis.

Source code in uaef/hitl/conversation_scoring.py

class InsufficientDataError(ConversationScoringError):
    """Raised when insufficient data is available for analysis."""
    pass

QualityTrend

Represents a trend in conversation quality over time.

Attributes:

Name	Type	Description
dimension		Quality dimension being tracked
time_period		Time period for the trend
average_score		Average score over the period
score_change		Change compared to previous period
conversation_count		Number of conversations in the period
trend_direction		Direction of trend
issues_detected		List of detected issues

Source code in uaef/hitl/conversation_scoring.py

class QualityTrend:
    """
    Represents a trend in conversation quality over time.

    Attributes:
        dimension: Quality dimension being tracked
        time_period: Time period for the trend
        average_score: Average score over the period
        score_change: Change compared to previous period
        conversation_count: Number of conversations in the period
        trend_direction: Direction of trend
        issues_detected: List of detected issues
    """

    def __init__(
        self,
        dimension: str,
        time_period: str,
        average_score: float,
        score_change: float,
        conversation_count: int,
        trend_direction: str,
        issues_detected: List[str]
    ):
        """Initialize quality trend."""
        self.dimension = dimension
        self.time_period = time_period
        self.average_score = average_score
        self.score_change = score_change
        self.conversation_count = conversation_count
        self.trend_direction = trend_direction
        self.issues_detected = issues_detected

    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary representation."""
        return {
            "dimension": self.dimension,
            "time_period": self.time_period,
            "average_score": self.average_score,
            "score_change": self.score_change,
            "conversation_count": self.conversation_count,
            "trend_direction": self.trend_direction,
            "issues_detected": self.issues_detected
        }

to_dict()

Convert to dictionary representation.

Source code in uaef/hitl/conversation_scoring.py

def to_dict(self) -> Dict[str, Any]:
    """Convert to dictionary representation."""
    return {
        "dimension": self.dimension,
        "time_period": self.time_period,
        "average_score": self.average_score,
        "score_change": self.score_change,
        "conversation_count": self.conversation_count,
        "trend_direction": self.trend_direction,
        "issues_detected": self.issues_detected
    }

uaef.hitl.benchmark

Benchmark versioning for HITL workflows.

This module provides versioned benchmarks from human-validated review items, enabling regression testing and tracking evaluation quality over time.

Benchmark

Bases: BaseModel

A versioned benchmark containing human-validated review items.

Attributes:

Name	Type	Description
benchmark_id	UUID	Unique identifier for this benchmark
metadata	BenchmarkMetadata	Benchmark metadata (version, date, metrics, etc.)
items	List[ReviewItem]	List of validated review items

Examples:

>>> benchmark = Benchmark(
...     benchmark_id=uuid4(),
...     metadata=BenchmarkMetadata(...),
...     items=[review_item1, review_item2, ...]
... )

Source code in uaef/hitl/benchmark.py

class Benchmark(BaseModel):
    """
    A versioned benchmark containing human-validated review items.

    Attributes:
        benchmark_id: Unique identifier for this benchmark
        metadata: Benchmark metadata (version, date, metrics, etc.)
        items: List of validated review items

    Examples:
        >>> benchmark = Benchmark(
        ...     benchmark_id=uuid4(),
        ...     metadata=BenchmarkMetadata(...),
        ...     items=[review_item1, review_item2, ...]
        ... )
    """

    benchmark_id: UUID = Field(
        default_factory=uuid4,
        description="Unique identifier for this benchmark"
    )
    metadata: BenchmarkMetadata = Field(
        ...,
        description="Benchmark metadata"
    )
    items: List[ReviewItem] = Field(
        ...,
        description="List of validated review items"
    )

    @field_validator("items")
    @classmethod
    def validate_items(cls, v: List[ReviewItem]) -> List[ReviewItem]:
        """Validate that items list is not empty."""
        if not v:
            raise ValueError("items cannot be empty")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "benchmark_id": "550e8400-e29b-41d4-a716-446655440000",
                "metadata": {
                    "version": "v1.0.0",
                    "creation_date": "2024-01-15T10:00:00Z",
                    "sample_size": 100,
                    "agreement_metrics": {},
                    "description": "Initial benchmark",
                    "metric_names": ["answer_relevance"],
                    "item_types": ["llm_evaluation"]
                },
                "items": []
            }
        }

Config

Pydantic configuration.

Source code in uaef/hitl/benchmark.py

class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "benchmark_id": "550e8400-e29b-41d4-a716-446655440000",
            "metadata": {
                "version": "v1.0.0",
                "creation_date": "2024-01-15T10:00:00Z",
                "sample_size": 100,
                "agreement_metrics": {},
                "description": "Initial benchmark",
                "metric_names": ["answer_relevance"],
                "item_types": ["llm_evaluation"]
            },
            "items": []
        }
    }

validate_items(v) classmethod

Validate that items list is not empty.

Source code in uaef/hitl/benchmark.py

@field_validator("items")
@classmethod
def validate_items(cls, v: List[ReviewItem]) -> List[ReviewItem]:
    """Validate that items list is not empty."""
    if not v:
        raise ValueError("items cannot be empty")
    return v

BenchmarkMetadata

Bases: BaseModel

Metadata for a benchmark version.

Attributes:

Name	Type	Description
version	str	Unique version identifier (e.g., "v1.0.0", "2024-01-15")
creation_date	datetime	When the benchmark was created
sample_size	int	Number of validated items in the benchmark
agreement_metrics	Dict[str, Any]	Agreement statistics for the benchmark
description	str	Human-readable description of the benchmark
metric_names	List[str]	List of metrics included in the benchmark
item_types	List[str]	List of item types included in the benchmark
tags	List[str]	Optional tags for categorization

Examples:

>>> metadata = BenchmarkMetadata(
...     version="v1.0.0",
...     creation_date=datetime.utcnow(),
...     sample_size=100,
...     agreement_metrics={
...         "answer_relevance": {"krippendorff_alpha": 0.85}
...     },
...     description="Initial benchmark with high-quality samples",
...     metric_names=["answer_relevance", "answer_correctness"],
...     item_types=["llm_evaluation"]
... )

Source code in uaef/hitl/benchmark.py

class BenchmarkMetadata(BaseModel):
    """
    Metadata for a benchmark version.

    Attributes:
        version: Unique version identifier (e.g., "v1.0.0", "2024-01-15")
        creation_date: When the benchmark was created
        sample_size: Number of validated items in the benchmark
        agreement_metrics: Agreement statistics for the benchmark
        description: Human-readable description of the benchmark
        metric_names: List of metrics included in the benchmark
        item_types: List of item types included in the benchmark
        tags: Optional tags for categorization

    Examples:
        >>> metadata = BenchmarkMetadata(
        ...     version="v1.0.0",
        ...     creation_date=datetime.utcnow(),
        ...     sample_size=100,
        ...     agreement_metrics={
        ...         "answer_relevance": {"krippendorff_alpha": 0.85}
        ...     },
        ...     description="Initial benchmark with high-quality samples",
        ...     metric_names=["answer_relevance", "answer_correctness"],
        ...     item_types=["llm_evaluation"]
        ... )
    """

    version: str = Field(
        ...,
        description="Unique version identifier",
        min_length=1
    )
    creation_date: datetime = Field(
        ...,
        description="When the benchmark was created"
    )
    sample_size: int = Field(
        ...,
        description="Number of validated items in the benchmark",
        ge=1
    )
    agreement_metrics: Dict[str, Any] = Field(
        ...,
        description="Agreement statistics for the benchmark"
    )
    description: str = Field(
        ...,
        description="Human-readable description of the benchmark",
        min_length=1
    )
    metric_names: List[str] = Field(
        ...,
        description="List of metrics included in the benchmark"
    )
    item_types: List[str] = Field(
        ...,
        description="List of item types included in the benchmark"
    )
    tags: List[str] = Field(
        default_factory=list,
        description="Optional tags for categorization"
    )

    @field_validator("version")
    @classmethod
    def validate_version(cls, v: str) -> str:
        """Validate that version is not empty."""
        if not v or not v.strip():
            raise ValueError("version cannot be empty")
        return v.strip()

    @field_validator("sample_size")
    @classmethod
    def validate_sample_size(cls, v: int) -> int:
        """Validate that sample_size is positive."""
        if v < 1:
            raise ValueError(f"sample_size must be at least 1, got {v}")
        return v

    @field_validator("metric_names")
    @classmethod
    def validate_metric_names(cls, v: List[str]) -> List[str]:
        """Validate that metric_names is not empty."""
        if not v:
            raise ValueError("metric_names cannot be empty")
        return v

    @field_validator("item_types")
    @classmethod
    def validate_item_types(cls, v: List[str]) -> List[str]:
        """Validate that item_types is not empty."""
        if not v:
            raise ValueError("item_types cannot be empty")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "version": "v1.0.0",
                "creation_date": "2024-01-15T10:00:00Z",
                "sample_size": 100,
                "agreement_metrics": {
                    "answer_relevance": {
                        "krippendorff_alpha": 0.85,
                        "mean_score": 0.78
                    }
                },
                "description": "Initial benchmark with high-quality samples",
                "metric_names": ["answer_relevance", "answer_correctness"],
                "item_types": ["llm_evaluation"],
                "tags": ["production", "validated"]
            }
        }

Config

Pydantic configuration.

Source code in uaef/hitl/benchmark.py

class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "version": "v1.0.0",
            "creation_date": "2024-01-15T10:00:00Z",
            "sample_size": 100,
            "agreement_metrics": {
                "answer_relevance": {
                    "krippendorff_alpha": 0.85,
                    "mean_score": 0.78
                }
            },
            "description": "Initial benchmark with high-quality samples",
            "metric_names": ["answer_relevance", "answer_correctness"],
            "item_types": ["llm_evaluation"],
            "tags": ["production", "validated"]
        }
    }

validate_item_types(v) classmethod

Validate that item_types is not empty.

Source code in uaef/hitl/benchmark.py

@field_validator("item_types")
@classmethod
def validate_item_types(cls, v: List[str]) -> List[str]:
    """Validate that item_types is not empty."""
    if not v:
        raise ValueError("item_types cannot be empty")
    return v

validate_metric_names(v) classmethod

Validate that metric_names is not empty.

Source code in uaef/hitl/benchmark.py

@field_validator("metric_names")
@classmethod
def validate_metric_names(cls, v: List[str]) -> List[str]:
    """Validate that metric_names is not empty."""
    if not v:
        raise ValueError("metric_names cannot be empty")
    return v

validate_sample_size(v) classmethod

Validate that sample_size is positive.

Source code in uaef/hitl/benchmark.py

@field_validator("sample_size")
@classmethod
def validate_sample_size(cls, v: int) -> int:
    """Validate that sample_size is positive."""
    if v < 1:
        raise ValueError(f"sample_size must be at least 1, got {v}")
    return v

validate_version(v) classmethod

Validate that version is not empty.

Source code in uaef/hitl/benchmark.py

@field_validator("version")
@classmethod
def validate_version(cls, v: str) -> str:
    """Validate that version is not empty."""
    if not v or not v.strip():
        raise ValueError("version cannot be empty")
    return v.strip()

BenchmarkNotFoundError

Bases: BenchmarkVersioningError

Raised when a benchmark version is not found.

Source code in uaef/hitl/benchmark.py

class BenchmarkNotFoundError(BenchmarkVersioningError):
    """Raised when a benchmark version is not found."""
    pass

BenchmarkVersioning

Manager for versioned benchmarks from human-validated review items.

This class creates and manages versioned benchmarks that can be used for: - Regression testing: Ensure evaluation quality doesn't degrade over time - LLM judge calibration: Track how well LLM scores align with human judgments - Quality tracking: Monitor agreement metrics across benchmark versions - Benchmark comparison: Compare evaluation quality between versions

Benchmarks are stored in-memory by default but can be exported/imported to/from files for persistence and sharing.

Attributes:

Name	Type	Description
_benchmarks	Dict[str, Benchmark]	Internal storage for benchmarks (version -> Benchmark)
_agreement_calculator		Calculator for agreement metrics

Examples:

>>> # Create benchmark versioning manager
>>> versioning = BenchmarkVersioning()

>>> # Create a new benchmark from validated review items
>>> benchmark = versioning.create_benchmark(
...     version="v1.0.0",
...     review_items=validated_items,
...     description="Initial production benchmark",
...     tags=["production", "validated"]
... )

>>> # Retrieve a specific benchmark
>>> benchmark = versioning.get_benchmark("v1.0.0")

>>> # List all available benchmarks
>>> benchmarks = versioning.list_benchmarks()

>>> # Compare two benchmarks
>>> comparison = versioning.compare_benchmarks("v1.0.0", "v2.0.0")

>>> # Export benchmark to file
>>> versioning.export_benchmark("v1.0.0", "benchmark_v1.json")

>>> # Import benchmark from file
>>> versioning.import_benchmark("benchmark_v1.json")

Source code in uaef/hitl/benchmark.py

class BenchmarkVersioning:
    """
    Manager for versioned benchmarks from human-validated review items.

    This class creates and manages versioned benchmarks that can be used for:
    - Regression testing: Ensure evaluation quality doesn't degrade over time
    - LLM judge calibration: Track how well LLM scores align with human judgments
    - Quality tracking: Monitor agreement metrics across benchmark versions
    - Benchmark comparison: Compare evaluation quality between versions

    Benchmarks are stored in-memory by default but can be exported/imported
    to/from files for persistence and sharing.

    Attributes:
        _benchmarks: Internal storage for benchmarks (version -> Benchmark)
        _agreement_calculator: Calculator for agreement metrics

    Examples:
        >>> # Create benchmark versioning manager
        >>> versioning = BenchmarkVersioning()

        >>> # Create a new benchmark from validated review items
        >>> benchmark = versioning.create_benchmark(
        ...     version="v1.0.0",
        ...     review_items=validated_items,
        ...     description="Initial production benchmark",
        ...     tags=["production", "validated"]
        ... )

        >>> # Retrieve a specific benchmark
        >>> benchmark = versioning.get_benchmark("v1.0.0")

        >>> # List all available benchmarks
        >>> benchmarks = versioning.list_benchmarks()

        >>> # Compare two benchmarks
        >>> comparison = versioning.compare_benchmarks("v1.0.0", "v2.0.0")

        >>> # Export benchmark to file
        >>> versioning.export_benchmark("v1.0.0", "benchmark_v1.json")

        >>> # Import benchmark from file
        >>> versioning.import_benchmark("benchmark_v1.json")
    """

    def __init__(
        self,
        disagreement_threshold: float = 0.2,
        min_samples: int = 2
    ):
        """
        Initialize the benchmark versioning manager.

        Args:
            disagreement_threshold: Threshold for identifying disputed items
            min_samples: Minimum samples required for agreement calculations

        Examples:
            >>> # Default settings
            >>> versioning = BenchmarkVersioning()

            >>> # Custom settings
            >>> versioning = BenchmarkVersioning(
            ...     disagreement_threshold=0.15,
            ...     min_samples=3
            ... )
        """
        self._benchmarks: Dict[str, Benchmark] = {}
        self._agreement_calculator = AgreementCalculator(
            disagreement_threshold=disagreement_threshold,
            min_samples=min_samples
        )

        logger.info(
            f"Initialized BenchmarkVersioning with "
            f"disagreement_threshold={disagreement_threshold}, "
            f"min_samples={min_samples}"
        )

    def create_benchmark(
        self,
        version: str,
        review_items: List[ReviewItem],
        description: str,
        tags: Optional[List[str]] = None,
        metric_names: Optional[List[str]] = None,
        item_types: Optional[List[str]] = None
    ) -> Benchmark:
        """
        Create a new benchmark version from validated review items.

        This method:
        1. Validates that all review items have human scores
        2. Calculates agreement metrics for each metric
        3. Creates benchmark metadata
        4. Stores the benchmark for future use

        Args:
            version: Unique version identifier (e.g., "v1.0.0", "2024-01-15")
            review_items: List of validated review items with human scores
            description: Human-readable description of the benchmark
            tags: Optional tags for categorization
            metric_names: Optional list of metrics to include (defaults to all)
            item_types: Optional list of item types to include (defaults to all)

        Returns:
            The created Benchmark object

        Raises:
            ValueError: If version already exists or review_items is empty
            InvalidBenchmarkError: If review items are invalid

        Examples:
            >>> # Create benchmark with all items
            >>> benchmark = versioning.create_benchmark(
            ...     version="v1.0.0",
            ...     review_items=validated_items,
            ...     description="Initial production benchmark"
            ... )

            >>> # Create benchmark with specific metrics
            >>> benchmark = versioning.create_benchmark(
            ...     version="v1.1.0",
            ...     review_items=validated_items,
            ...     description="Benchmark for answer quality metrics",
            ...     metric_names=["answer_relevance", "answer_correctness"],
            ...     tags=["production", "answer_quality"]
            ... )

            >>> # Create benchmark with specific item types
            >>> benchmark = versioning.create_benchmark(
            ...     version="v1.2.0",
            ...     review_items=validated_items,
            ...     description="LLM evaluation benchmark",
            ...     item_types=["llm_evaluation"]
            ... )
        """
        # Validate version doesn't already exist
        if version in self._benchmarks:
            raise ValueError(f"Benchmark version '{version}' already exists")

        # Validate review_items
        if not review_items:
            raise ValueError("review_items cannot be empty")

        # Filter items by type if specified
        if item_types:
            review_items = [
                item for item in review_items
                if item.item_type.value in item_types
            ]

            if not review_items:
                raise ValueError(
                    f"No items found matching item_types: {item_types}"
                )

        # Validate all items have human scores
        items_without_scores = [
            item for item in review_items
            if not item.human_scores
        ]

        if items_without_scores:
            raise InvalidBenchmarkError(
                f"Found {len(items_without_scores)} items without human scores. "
                f"All items must be validated before creating a benchmark."
            )

        # Extract all metric names if not specified
        if metric_names is None:
            metric_names_set = set()
            for item in review_items:
                metric_names_set.update(item.human_scores.keys())
            metric_names = sorted(metric_names_set)

        # Extract all item types if not specified
        if item_types is None:
            item_types_set = set()
            for item in review_items:
                item_types_set.add(item.item_type.value)
            item_types = sorted(item_types_set)

        # Calculate agreement metrics for each metric
        agreement_metrics = {}

        for metric_name in metric_names:
            # Filter items that have this metric
            items_with_metric = [
                item for item in review_items
                if metric_name in item.human_scores
            ]

            if not items_with_metric:
                logger.warning(
                    f"No items found with metric '{metric_name}', skipping"
                )
                continue

            try:
                # Generate agreement report for this metric
                report = self._agreement_calculator.generate_agreement_report(
                    items_with_metric,
                    metric_name,
                    include_llm_correlation=True
                )

                # Extract key metrics for metadata
                agreement_metrics[metric_name] = {
                    "inter_rater_agreement": report.get("inter_rater_agreement"),
                    "llm_human_correlation": report.get("llm_human_correlation"),
                    "disputed_count": len(report.get("disputed_items", [])),
                    "overall_quality": report.get("summary", {}).get(
                        "overall_agreement_quality"
                    )
                }

            except Exception as e:
                logger.warning(
                    f"Could not calculate agreement for metric '{metric_name}': {e}"
                )
                agreement_metrics[metric_name] = {
                    "error": str(e)
                }

        # Create benchmark metadata
        metadata = BenchmarkMetadata(
            version=version,
            creation_date=datetime.utcnow(),
            sample_size=len(review_items),
            agreement_metrics=agreement_metrics,
            description=description,
            metric_names=metric_names,
            item_types=item_types,
            tags=tags or []
        )

        # Create benchmark
        benchmark = Benchmark(
            benchmark_id=uuid4(),
            metadata=metadata,
            items=review_items
        )

        # Store benchmark
        self._benchmarks[version] = benchmark

        logger.info(
            f"Created benchmark version '{version}' with "
            f"{len(review_items)} items, {len(metric_names)} metrics"
        )

        return benchmark

    def get_benchmark(self, version: str) -> Benchmark:
        """
        Retrieve a specific benchmark version.

        Args:
            version: Version identifier of the benchmark to retrieve

        Returns:
            The Benchmark object

        Raises:
            BenchmarkNotFoundError: If version is not found

        Examples:
            >>> benchmark = versioning.get_benchmark("v1.0.0")
            >>> print(f"Sample size: {benchmark.metadata.sample_size}")
            >>> print(f"Metrics: {benchmark.metadata.metric_names}")
        """
        if version not in self._benchmarks:
            raise BenchmarkNotFoundError(
                f"Benchmark version '{version}' not found. "
                f"Available versions: {list(self._benchmarks.keys())}"
            )

        return self._benchmarks[version]

    def list_benchmarks(
        self,
        metric_name: Optional[str] = None,
        item_type: Optional[str] = None,
        start_date: Optional[datetime] = None,
        end_date: Optional[datetime] = None,
        tags: Optional[List[str]] = None
    ) -> List[Dict[str, Any]]:
        """
        List all available benchmarks with optional filtering.

        Args:
            metric_name: Optional filter by metric name
            item_type: Optional filter by item type
            start_date: Optional filter by creation date (inclusive)
            end_date: Optional filter by creation date (inclusive)
            tags: Optional filter by tags (must have all specified tags)

        Returns:
            List of benchmark summaries, each containing:
                - version: Version identifier
                - creation_date: When the benchmark was created
                - sample_size: Number of items
                - metric_names: List of metrics
                - item_types: List of item types
                - description: Benchmark description
                - tags: List of tags

        Examples:
            >>> # List all benchmarks
            >>> benchmarks = versioning.list_benchmarks()
            >>> for b in benchmarks:
            ...     print(f"{b['version']}: {b['sample_size']} items")

            >>> # Filter by metric
            >>> benchmarks = versioning.list_benchmarks(
            ...     metric_name="answer_relevance"
            ... )

            >>> # Filter by date range
            >>> benchmarks = versioning.list_benchmarks(
            ...     start_date=datetime(2024, 1, 1),
            ...     end_date=datetime(2024, 12, 31)
            ... )

            >>> # Filter by tags
            >>> benchmarks = versioning.list_benchmarks(
            ...     tags=["production", "validated"]
            ... )
        """
        results = []

        for version, benchmark in self._benchmarks.items():
            metadata = benchmark.metadata

            # Apply filters
            if metric_name and metric_name not in metadata.metric_names:
                continue

            if item_type and item_type not in metadata.item_types:
                continue

            if start_date and metadata.creation_date < start_date:
                continue

            if end_date and metadata.creation_date > end_date:
                continue

            if tags and not all(tag in metadata.tags for tag in tags):
                continue

            # Add to results
            results.append({
                "version": metadata.version,
                "creation_date": metadata.creation_date.isoformat(),
                "sample_size": metadata.sample_size,
                "metric_names": metadata.metric_names,
                "item_types": metadata.item_types,
                "description": metadata.description,
                "tags": metadata.tags,
                "benchmark_id": str(benchmark.benchmark_id)
            })

        # Sort by creation date (newest first)
        results.sort(key=lambda x: x["creation_date"], reverse=True)

        logger.info(f"Listed {len(results)} benchmarks")

        return results


    def compare_benchmarks(
        self,
        version1: str,
        version2: str,
        metric_names: Optional[List[str]] = None
    ) -> Dict[str, Any]:
        """
        Compare two benchmark versions to track evaluation quality changes.

        This method compares agreement metrics, sample sizes, and quality
        indicators between two benchmark versions to identify improvements
        or regressions in evaluation quality.

        Args:
            version1: First benchmark version (baseline)
            version2: Second benchmark version (comparison)
            metric_names: Optional list of metrics to compare (defaults to all)

        Returns:
            Dictionary containing:
                - version1: First version identifier
                - version2: Second version identifier
                - date1: Creation date of first benchmark
                - date2: Creation date of second benchmark
                - sample_size_change: Change in sample size
                - metric_comparisons: Per-metric comparison results
                    - metric_name: Name of the metric
                    - agreement_change: Change in agreement metrics
                    - quality_change: Change in quality assessment
                - summary: High-level comparison summary
                    - overall_trend: "improved", "degraded", "stable"
                    - significant_changes: List of metrics with significant changes

        Raises:
            BenchmarkNotFoundError: If either version is not found
            ValueError: If versions are the same

        Examples:
            >>> # Compare two versions
            >>> comparison = versioning.compare_benchmarks("v1.0.0", "v2.0.0")
            >>> print(f"Overall trend: {comparison['summary']['overall_trend']}")
            >>> 
            >>> # Check specific metrics
            >>> for metric_comp in comparison['metric_comparisons']:
            ...     print(f"{metric_comp['metric_name']}: "
            ...           f"{metric_comp['quality_change']}")

            >>> # Compare specific metrics only
            >>> comparison = versioning.compare_benchmarks(
            ...     "v1.0.0", "v2.0.0",
            ...     metric_names=["answer_relevance", "answer_correctness"]
            ... )
        """
        # Validate versions
        if version1 == version2:
            raise ValueError("Cannot compare a benchmark version with itself")

        benchmark1 = self.get_benchmark(version1)
        benchmark2 = self.get_benchmark(version2)

        # Determine which metrics to compare
        if metric_names is None:
            # Use intersection of metrics from both benchmarks
            metrics1 = set(benchmark1.metadata.metric_names)
            metrics2 = set(benchmark2.metadata.metric_names)
            metric_names = sorted(metrics1.intersection(metrics2))

            if not metric_names:
                logger.warning(
                    f"No common metrics between {version1} and {version2}"
                )

        # Compare each metric
        metric_comparisons = []

        for metric_name in metric_names:
            # Get agreement metrics for both versions
            metrics1 = benchmark1.metadata.agreement_metrics.get(metric_name, {})
            metrics2 = benchmark2.metadata.agreement_metrics.get(metric_name, {})

            if not metrics1 or not metrics2:
                logger.warning(
                    f"Metric '{metric_name}' not found in one or both benchmarks"
                )
                continue

            # Compare inter-rater agreement
            inter_rater1 = metrics1.get("inter_rater_agreement")
            inter_rater2 = metrics2.get("inter_rater_agreement")

            agreement_change = {}

            if inter_rater1 and inter_rater2:
                alpha1 = inter_rater1.get("krippendorff_alpha", 0.0)
                alpha2 = inter_rater2.get("krippendorff_alpha", 0.0)
                agreement_change["krippendorff_alpha_delta"] = alpha2 - alpha1
                agreement_change["krippendorff_alpha_percent"] = (
                    ((alpha2 - alpha1) / alpha1 * 100) if alpha1 > 0 else 0.0
                )

            # Compare LLM-human correlation
            llm_human1 = metrics1.get("llm_human_correlation")
            llm_human2 = metrics2.get("llm_human_correlation")

            if llm_human1 and llm_human2:
                pearson1 = llm_human1.get("pearson_correlation", 0.0)
                pearson2 = llm_human2.get("pearson_correlation", 0.0)
                agreement_change["pearson_correlation_delta"] = pearson2 - pearson1
                agreement_change["pearson_correlation_percent"] = (
                    ((pearson2 - pearson1) / abs(pearson1) * 100)
                    if pearson1 != 0 else 0.0
                )

            # Compare quality assessments
            quality1 = metrics1.get("overall_quality", "unknown")
            quality2 = metrics2.get("overall_quality", "unknown")

            quality_change = self._assess_quality_change(quality1, quality2)

            # Compare disputed counts
            disputed1 = metrics1.get("disputed_count", 0)
            disputed2 = metrics2.get("disputed_count", 0)

            metric_comparisons.append({
                "metric_name": metric_name,
                "agreement_change": agreement_change,
                "quality_change": quality_change,
                "quality1": quality1,
                "quality2": quality2,
                "disputed_count_change": disputed2 - disputed1
            })

        # Calculate sample size change
        sample_size_change = (
            benchmark2.metadata.sample_size - benchmark1.metadata.sample_size
        )
        sample_size_percent = (
            (sample_size_change / benchmark1.metadata.sample_size * 100)
            if benchmark1.metadata.sample_size > 0 else 0.0
        )

        # Assess overall trend
        overall_trend, significant_changes = self._assess_overall_trend(
            metric_comparisons
        )

        comparison = {
            "version1": version1,
            "version2": version2,
            "date1": benchmark1.metadata.creation_date.isoformat(),
            "date2": benchmark2.metadata.creation_date.isoformat(),
            "sample_size_change": {
                "absolute": sample_size_change,
                "percent": sample_size_percent,
                "size1": benchmark1.metadata.sample_size,
                "size2": benchmark2.metadata.sample_size
            },
            "metric_comparisons": metric_comparisons,
            "summary": {
                "overall_trend": overall_trend,
                "significant_changes": significant_changes,
                "metrics_compared": len(metric_comparisons)
            }
        }

        logger.info(
            f"Compared benchmarks {version1} vs {version2}: "
            f"trend={overall_trend}, {len(significant_changes)} significant changes"
        )

        return comparison

    def _assess_quality_change(self, quality1: str, quality2: str) -> str:
        """
        Assess the change in quality between two benchmarks.

        Returns: "improved", "degraded", or "stable"
        """
        quality_order = {
            "poor": 1,
            "fair": 2,
            "good": 3,
            "excellent": 4,
            "unknown": 0
        }

        score1 = quality_order.get(quality1, 0)
        score2 = quality_order.get(quality2, 0)

        if score2 > score1:
            return "improved"
        elif score2 < score1:
            return "degraded"
        else:
            return "stable"

    def _assess_overall_trend(
        self,
        metric_comparisons: List[Dict[str, Any]]
    ) -> tuple[str, List[str]]:
        """
        Assess overall trend across all metric comparisons.

        Returns:
            Tuple of (overall_trend, significant_changes)
        """
        if not metric_comparisons:
            return "stable", []

        # Count improvements and degradations
        improvements = 0
        degradations = 0
        significant_changes = []

        for comp in metric_comparisons:
            quality_change = comp["quality_change"]
            metric_name = comp["metric_name"]

            if quality_change == "improved":
                improvements += 1
                significant_changes.append(f"{metric_name}: improved")
            elif quality_change == "degraded":
                degradations += 1
                significant_changes.append(f"{metric_name}: degraded")

            # Check for significant agreement changes
            agreement_change = comp.get("agreement_change", {})
            alpha_percent = agreement_change.get("krippendorff_alpha_percent", 0.0)

            if abs(alpha_percent) > 10:  # More than 10% change
                if alpha_percent > 0:
                    significant_changes.append(
                        f"{metric_name}: agreement +{alpha_percent:.1f}%"
                    )
                else:
                    significant_changes.append(
                        f"{metric_name}: agreement {alpha_percent:.1f}%"
                    )

        # Determine overall trend
        if improvements > degradations:
            overall_trend = "improved"
        elif degradations > improvements:
            overall_trend = "degraded"
        else:
            overall_trend = "stable"

        return overall_trend, significant_changes

    def export_benchmark(
        self,
        version: str,
        filepath: str
    ) -> None:
        """
        Export a benchmark to a JSON file for sharing or persistence.

        The exported file contains the complete benchmark including metadata
        and all review items, allowing it to be imported into another system
        or used for regression testing.

        Args:
            version: Version identifier of the benchmark to export
            filepath: Path to the output JSON file

        Raises:
            BenchmarkNotFoundError: If version is not found
            IOError: If file cannot be written

        Examples:
            >>> # Export to file
            >>> versioning.export_benchmark("v1.0.0", "benchmark_v1.json")

            >>> # Export to specific directory
            >>> versioning.export_benchmark(
            ...     "v1.0.0",
            ...     "/path/to/benchmarks/benchmark_v1.json"
            ... )
        """
        benchmark = self.get_benchmark(version)

        # Convert to dict for JSON serialization
        benchmark_dict = {
            "benchmark_id": str(benchmark.benchmark_id),
            "metadata": benchmark.metadata.model_dump(mode="json"),
            "items": [item.model_dump(mode="json") for item in benchmark.items]
        }

        # Write to file
        try:
            path = Path(filepath)
            path.parent.mkdir(parents=True, exist_ok=True)

            with open(path, "w") as f:
                json.dump(benchmark_dict, f, indent=2, default=str)

            logger.info(
                f"Exported benchmark '{version}' to {filepath} "
                f"({len(benchmark.items)} items)"
            )

        except Exception as e:
            raise IOError(f"Failed to export benchmark to {filepath}: {e}")

    def import_benchmark(
        self,
        filepath: str,
        overwrite: bool = False
    ) -> Benchmark:
        """
        Import a benchmark from a JSON file.

        This method loads a previously exported benchmark from a file,
        validates it, and adds it to the available benchmarks.

        Args:
            filepath: Path to the JSON file containing the benchmark
            overwrite: If True, overwrite existing benchmark with same version

        Returns:
            The imported Benchmark object

        Raises:
            FileNotFoundError: If file does not exist
            InvalidBenchmarkError: If file is invalid or corrupted
            ValueError: If version already exists and overwrite is False

        Examples:
            >>> # Import from file
            >>> benchmark = versioning.import_benchmark("benchmark_v1.json")
            >>> print(f"Imported {benchmark.metadata.version}")

            >>> # Import and overwrite existing
            >>> benchmark = versioning.import_benchmark(
            ...     "benchmark_v1.json",
            ...     overwrite=True
            ... )
        """
        # Read file
        try:
            path = Path(filepath)

            if not path.exists():
                raise FileNotFoundError(f"Benchmark file not found: {filepath}")

            with open(path, "r") as f:
                benchmark_dict = json.load(f)

        except json.JSONDecodeError as e:
            raise InvalidBenchmarkError(f"Invalid JSON in {filepath}: {e}")
        except Exception as e:
            raise IOError(f"Failed to read benchmark from {filepath}: {e}")

        # Validate and reconstruct benchmark
        try:
            # Reconstruct metadata
            metadata = BenchmarkMetadata(**benchmark_dict["metadata"])

            # Reconstruct review items
            items = [
                ReviewItem(**item_dict)
                for item_dict in benchmark_dict["items"]
            ]

            # Reconstruct benchmark
            benchmark = Benchmark(
                benchmark_id=UUID(benchmark_dict["benchmark_id"]),
                metadata=metadata,
                items=items
            )

        except Exception as e:
            raise InvalidBenchmarkError(
                f"Failed to reconstruct benchmark from {filepath}: {e}"
            )

        # Check if version already exists
        version = metadata.version

        if version in self._benchmarks and not overwrite:
            raise ValueError(
                f"Benchmark version '{version}' already exists. "
                f"Use overwrite=True to replace it."
            )

        # Store benchmark
        self._benchmarks[version] = benchmark

        logger.info(
            f"Imported benchmark '{version}' from {filepath} "
            f"({len(items)} items)"
        )

        return benchmark

compare_benchmarks(version1, version2, metric_names=None)

Compare two benchmark versions to track evaluation quality changes.

This method compares agreement metrics, sample sizes, and quality indicators between two benchmark versions to identify improvements or regressions in evaluation quality.

Parameters:

Name	Type	Description	Default
version1	str	First benchmark version (baseline)	required
version2	str	Second benchmark version (comparison)	required
metric_names	Optional[List[str]]	Optional list of metrics to compare (defaults to all)	None

Returns:

Type

Description

Dict[str, Any]

Dictionary containing: - version1: First version identifier - version2: Second version identifier - date1: Creation date of first benchmark - date2: Creation date of second benchmark - sample_size_change: Change in sample size - metric_comparisons: Per-metric comparison results - metric_name: Name of the metric - agreement_change: Change in agreement metrics - quality_change: Change in quality assessment - summary: High-level comparison summary - overall_trend: "improved", "degraded", "stable" - significant_changes: List of metrics with significant changes

Raises:

Type	Description
BenchmarkNotFoundError	If either version is not found
ValueError	If versions are the same

Examples:

>>> # Compare two versions
>>> comparison = versioning.compare_benchmarks("v1.0.0", "v2.0.0")
>>> print(f"Overall trend: {comparison['summary']['overall_trend']}")
>>> 
>>> # Check specific metrics
>>> for metric_comp in comparison['metric_comparisons']:
...     print(f"{metric_comp['metric_name']}: "
...           f"{metric_comp['quality_change']}")

>>> # Compare specific metrics only
>>> comparison = versioning.compare_benchmarks(
...     "v1.0.0", "v2.0.0",
...     metric_names=["answer_relevance", "answer_correctness"]
... )

Source code in uaef/hitl/benchmark.py

def compare_benchmarks(
    self,
    version1: str,
    version2: str,
    metric_names: Optional[List[str]] = None
) -> Dict[str, Any]:
    """
    Compare two benchmark versions to track evaluation quality changes.

    This method compares agreement metrics, sample sizes, and quality
    indicators between two benchmark versions to identify improvements
    or regressions in evaluation quality.

    Args:
        version1: First benchmark version (baseline)
        version2: Second benchmark version (comparison)
        metric_names: Optional list of metrics to compare (defaults to all)

    Returns:
        Dictionary containing:
            - version1: First version identifier
            - version2: Second version identifier
            - date1: Creation date of first benchmark
            - date2: Creation date of second benchmark
            - sample_size_change: Change in sample size
            - metric_comparisons: Per-metric comparison results
                - metric_name: Name of the metric
                - agreement_change: Change in agreement metrics
                - quality_change: Change in quality assessment
            - summary: High-level comparison summary
                - overall_trend: "improved", "degraded", "stable"
                - significant_changes: List of metrics with significant changes

    Raises:
        BenchmarkNotFoundError: If either version is not found
        ValueError: If versions are the same

    Examples:
        >>> # Compare two versions
        >>> comparison = versioning.compare_benchmarks("v1.0.0", "v2.0.0")
        >>> print(f"Overall trend: {comparison['summary']['overall_trend']}")
        >>> 
        >>> # Check specific metrics
        >>> for metric_comp in comparison['metric_comparisons']:
        ...     print(f"{metric_comp['metric_name']}: "
        ...           f"{metric_comp['quality_change']}")

        >>> # Compare specific metrics only
        >>> comparison = versioning.compare_benchmarks(
        ...     "v1.0.0", "v2.0.0",
        ...     metric_names=["answer_relevance", "answer_correctness"]
        ... )
    """
    # Validate versions
    if version1 == version2:
        raise ValueError("Cannot compare a benchmark version with itself")

    benchmark1 = self.get_benchmark(version1)
    benchmark2 = self.get_benchmark(version2)

    # Determine which metrics to compare
    if metric_names is None:
        # Use intersection of metrics from both benchmarks
        metrics1 = set(benchmark1.metadata.metric_names)
        metrics2 = set(benchmark2.metadata.metric_names)
        metric_names = sorted(metrics1.intersection(metrics2))

        if not metric_names:
            logger.warning(
                f"No common metrics between {version1} and {version2}"
            )

    # Compare each metric
    metric_comparisons = []

    for metric_name in metric_names:
        # Get agreement metrics for both versions
        metrics1 = benchmark1.metadata.agreement_metrics.get(metric_name, {})
        metrics2 = benchmark2.metadata.agreement_metrics.get(metric_name, {})

        if not metrics1 or not metrics2:
            logger.warning(
                f"Metric '{metric_name}' not found in one or both benchmarks"
            )
            continue

        # Compare inter-rater agreement
        inter_rater1 = metrics1.get("inter_rater_agreement")
        inter_rater2 = metrics2.get("inter_rater_agreement")

        agreement_change = {}

        if inter_rater1 and inter_rater2:
            alpha1 = inter_rater1.get("krippendorff_alpha", 0.0)
            alpha2 = inter_rater2.get("krippendorff_alpha", 0.0)
            agreement_change["krippendorff_alpha_delta"] = alpha2 - alpha1
            agreement_change["krippendorff_alpha_percent"] = (
                ((alpha2 - alpha1) / alpha1 * 100) if alpha1 > 0 else 0.0
            )

        # Compare LLM-human correlation
        llm_human1 = metrics1.get("llm_human_correlation")
        llm_human2 = metrics2.get("llm_human_correlation")

        if llm_human1 and llm_human2:
            pearson1 = llm_human1.get("pearson_correlation", 0.0)
            pearson2 = llm_human2.get("pearson_correlation", 0.0)
            agreement_change["pearson_correlation_delta"] = pearson2 - pearson1
            agreement_change["pearson_correlation_percent"] = (
                ((pearson2 - pearson1) / abs(pearson1) * 100)
                if pearson1 != 0 else 0.0
            )

        # Compare quality assessments
        quality1 = metrics1.get("overall_quality", "unknown")
        quality2 = metrics2.get("overall_quality", "unknown")

        quality_change = self._assess_quality_change(quality1, quality2)

        # Compare disputed counts
        disputed1 = metrics1.get("disputed_count", 0)
        disputed2 = metrics2.get("disputed_count", 0)

        metric_comparisons.append({
            "metric_name": metric_name,
            "agreement_change": agreement_change,
            "quality_change": quality_change,
            "quality1": quality1,
            "quality2": quality2,
            "disputed_count_change": disputed2 - disputed1
        })

    # Calculate sample size change
    sample_size_change = (
        benchmark2.metadata.sample_size - benchmark1.metadata.sample_size
    )
    sample_size_percent = (
        (sample_size_change / benchmark1.metadata.sample_size * 100)
        if benchmark1.metadata.sample_size > 0 else 0.0
    )

    # Assess overall trend
    overall_trend, significant_changes = self._assess_overall_trend(
        metric_comparisons
    )

    comparison = {
        "version1": version1,
        "version2": version2,
        "date1": benchmark1.metadata.creation_date.isoformat(),
        "date2": benchmark2.metadata.creation_date.isoformat(),
        "sample_size_change": {
            "absolute": sample_size_change,
            "percent": sample_size_percent,
            "size1": benchmark1.metadata.sample_size,
            "size2": benchmark2.metadata.sample_size
        },
        "metric_comparisons": metric_comparisons,
        "summary": {
            "overall_trend": overall_trend,
            "significant_changes": significant_changes,
            "metrics_compared": len(metric_comparisons)
        }
    }

    logger.info(
        f"Compared benchmarks {version1} vs {version2}: "
        f"trend={overall_trend}, {len(significant_changes)} significant changes"
    )

    return comparison

create_benchmark(version, review_items, description, tags=None, metric_names=None, item_types=None)

Create a new benchmark version from validated review items.

This method: 1. Validates that all review items have human scores 2. Calculates agreement metrics for each metric 3. Creates benchmark metadata 4. Stores the benchmark for future use

Parameters:

Name	Type	Description	Default
version	str	Unique version identifier (e.g., "v1.0.0", "2024-01-15")	required
review_items	List[ReviewItem]	List of validated review items with human scores	required
description	str	Human-readable description of the benchmark	required
tags	Optional[List[str]]	Optional tags for categorization	None
metric_names	Optional[List[str]]	Optional list of metrics to include (defaults to all)	None
item_types	Optional[List[str]]	Optional list of item types to include (defaults to all)	None

Returns:

Type	Description
Benchmark	The created Benchmark object

Raises:

Type	Description
ValueError	If version already exists or review_items is empty
InvalidBenchmarkError	If review items are invalid

Examples:

>>> # Create benchmark with all items
>>> benchmark = versioning.create_benchmark(
...     version="v1.0.0",
...     review_items=validated_items,
...     description="Initial production benchmark"
... )

>>> # Create benchmark with specific metrics
>>> benchmark = versioning.create_benchmark(
...     version="v1.1.0",
...     review_items=validated_items,
...     description="Benchmark for answer quality metrics",
...     metric_names=["answer_relevance", "answer_correctness"],
...     tags=["production", "answer_quality"]
... )

>>> # Create benchmark with specific item types
>>> benchmark = versioning.create_benchmark(
...     version="v1.2.0",
...     review_items=validated_items,
...     description="LLM evaluation benchmark",
...     item_types=["llm_evaluation"]
... )

Source code in uaef/hitl/benchmark.py

def create_benchmark(
    self,
    version: str,
    review_items: List[ReviewItem],
    description: str,
    tags: Optional[List[str]] = None,
    metric_names: Optional[List[str]] = None,
    item_types: Optional[List[str]] = None
) -> Benchmark:
    """
    Create a new benchmark version from validated review items.

    This method:
    1. Validates that all review items have human scores
    2. Calculates agreement metrics for each metric
    3. Creates benchmark metadata
    4. Stores the benchmark for future use

    Args:
        version: Unique version identifier (e.g., "v1.0.0", "2024-01-15")
        review_items: List of validated review items with human scores
        description: Human-readable description of the benchmark
        tags: Optional tags for categorization
        metric_names: Optional list of metrics to include (defaults to all)
        item_types: Optional list of item types to include (defaults to all)

    Returns:
        The created Benchmark object

    Raises:
        ValueError: If version already exists or review_items is empty
        InvalidBenchmarkError: If review items are invalid

    Examples:
        >>> # Create benchmark with all items
        >>> benchmark = versioning.create_benchmark(
        ...     version="v1.0.0",
        ...     review_items=validated_items,
        ...     description="Initial production benchmark"
        ... )

        >>> # Create benchmark with specific metrics
        >>> benchmark = versioning.create_benchmark(
        ...     version="v1.1.0",
        ...     review_items=validated_items,
        ...     description="Benchmark for answer quality metrics",
        ...     metric_names=["answer_relevance", "answer_correctness"],
        ...     tags=["production", "answer_quality"]
        ... )

        >>> # Create benchmark with specific item types
        >>> benchmark = versioning.create_benchmark(
        ...     version="v1.2.0",
        ...     review_items=validated_items,
        ...     description="LLM evaluation benchmark",
        ...     item_types=["llm_evaluation"]
        ... )
    """
    # Validate version doesn't already exist
    if version in self._benchmarks:
        raise ValueError(f"Benchmark version '{version}' already exists")

    # Validate review_items
    if not review_items:
        raise ValueError("review_items cannot be empty")

    # Filter items by type if specified
    if item_types:
        review_items = [
            item for item in review_items
            if item.item_type.value in item_types
        ]

        if not review_items:
            raise ValueError(
                f"No items found matching item_types: {item_types}"
            )

    # Validate all items have human scores
    items_without_scores = [
        item for item in review_items
        if not item.human_scores
    ]

    if items_without_scores:
        raise InvalidBenchmarkError(
            f"Found {len(items_without_scores)} items without human scores. "
            f"All items must be validated before creating a benchmark."
        )

    # Extract all metric names if not specified
    if metric_names is None:
        metric_names_set = set()
        for item in review_items:
            metric_names_set.update(item.human_scores.keys())
        metric_names = sorted(metric_names_set)

    # Extract all item types if not specified
    if item_types is None:
        item_types_set = set()
        for item in review_items:
            item_types_set.add(item.item_type.value)
        item_types = sorted(item_types_set)

    # Calculate agreement metrics for each metric
    agreement_metrics = {}

    for metric_name in metric_names:
        # Filter items that have this metric
        items_with_metric = [
            item for item in review_items
            if metric_name in item.human_scores
        ]

        if not items_with_metric:
            logger.warning(
                f"No items found with metric '{metric_name}', skipping"
            )
            continue

        try:
            # Generate agreement report for this metric
            report = self._agreement_calculator.generate_agreement_report(
                items_with_metric,
                metric_name,
                include_llm_correlation=True
            )

            # Extract key metrics for metadata
            agreement_metrics[metric_name] = {
                "inter_rater_agreement": report.get("inter_rater_agreement"),
                "llm_human_correlation": report.get("llm_human_correlation"),
                "disputed_count": len(report.get("disputed_items", [])),
                "overall_quality": report.get("summary", {}).get(
                    "overall_agreement_quality"
                )
            }

        except Exception as e:
            logger.warning(
                f"Could not calculate agreement for metric '{metric_name}': {e}"
            )
            agreement_metrics[metric_name] = {
                "error": str(e)
            }

    # Create benchmark metadata
    metadata = BenchmarkMetadata(
        version=version,
        creation_date=datetime.utcnow(),
        sample_size=len(review_items),
        agreement_metrics=agreement_metrics,
        description=description,
        metric_names=metric_names,
        item_types=item_types,
        tags=tags or []
    )

    # Create benchmark
    benchmark = Benchmark(
        benchmark_id=uuid4(),
        metadata=metadata,
        items=review_items
    )

    # Store benchmark
    self._benchmarks[version] = benchmark

    logger.info(
        f"Created benchmark version '{version}' with "
        f"{len(review_items)} items, {len(metric_names)} metrics"
    )

    return benchmark

export_benchmark(version, filepath)

Export a benchmark to a JSON file for sharing or persistence.

The exported file contains the complete benchmark including metadata and all review items, allowing it to be imported into another system or used for regression testing.

Parameters:

Name	Type	Description	Default
version	str	Version identifier of the benchmark to export	required
filepath	str	Path to the output JSON file	required

Raises:

Type	Description
BenchmarkNotFoundError	If version is not found
IOError	If file cannot be written

Examples:

>>> # Export to file
>>> versioning.export_benchmark("v1.0.0", "benchmark_v1.json")

>>> # Export to specific directory
>>> versioning.export_benchmark(
...     "v1.0.0",
...     "/path/to/benchmarks/benchmark_v1.json"
... )

Source code in uaef/hitl/benchmark.py

def export_benchmark(
    self,
    version: str,
    filepath: str
) -> None:
    """
    Export a benchmark to a JSON file for sharing or persistence.

    The exported file contains the complete benchmark including metadata
    and all review items, allowing it to be imported into another system
    or used for regression testing.

    Args:
        version: Version identifier of the benchmark to export
        filepath: Path to the output JSON file

    Raises:
        BenchmarkNotFoundError: If version is not found
        IOError: If file cannot be written

    Examples:
        >>> # Export to file
        >>> versioning.export_benchmark("v1.0.0", "benchmark_v1.json")

        >>> # Export to specific directory
        >>> versioning.export_benchmark(
        ...     "v1.0.0",
        ...     "/path/to/benchmarks/benchmark_v1.json"
        ... )
    """
    benchmark = self.get_benchmark(version)

    # Convert to dict for JSON serialization
    benchmark_dict = {
        "benchmark_id": str(benchmark.benchmark_id),
        "metadata": benchmark.metadata.model_dump(mode="json"),
        "items": [item.model_dump(mode="json") for item in benchmark.items]
    }

    # Write to file
    try:
        path = Path(filepath)
        path.parent.mkdir(parents=True, exist_ok=True)

        with open(path, "w") as f:
            json.dump(benchmark_dict, f, indent=2, default=str)

        logger.info(
            f"Exported benchmark '{version}' to {filepath} "
            f"({len(benchmark.items)} items)"
        )

    except Exception as e:
        raise IOError(f"Failed to export benchmark to {filepath}: {e}")

get_benchmark(version)

Retrieve a specific benchmark version.

Parameters:

Name	Type	Description	Default
version	str	Version identifier of the benchmark to retrieve	required

Returns:

Type	Description
Benchmark	The Benchmark object

Raises:

Type	Description
BenchmarkNotFoundError	If version is not found

Examples:

>>> benchmark = versioning.get_benchmark("v1.0.0")
>>> print(f"Sample size: {benchmark.metadata.sample_size}")
>>> print(f"Metrics: {benchmark.metadata.metric_names}")

Source code in uaef/hitl/benchmark.py

def get_benchmark(self, version: str) -> Benchmark:
    """
    Retrieve a specific benchmark version.

    Args:
        version: Version identifier of the benchmark to retrieve

    Returns:
        The Benchmark object

    Raises:
        BenchmarkNotFoundError: If version is not found

    Examples:
        >>> benchmark = versioning.get_benchmark("v1.0.0")
        >>> print(f"Sample size: {benchmark.metadata.sample_size}")
        >>> print(f"Metrics: {benchmark.metadata.metric_names}")
    """
    if version not in self._benchmarks:
        raise BenchmarkNotFoundError(
            f"Benchmark version '{version}' not found. "
            f"Available versions: {list(self._benchmarks.keys())}"
        )

    return self._benchmarks[version]

import_benchmark(filepath, overwrite=False)

Import a benchmark from a JSON file.

This method loads a previously exported benchmark from a file, validates it, and adds it to the available benchmarks.

Parameters:

Name	Type	Description	Default
filepath	str	Path to the JSON file containing the benchmark	required
overwrite	bool	If True, overwrite existing benchmark with same version	False

Returns:

Type	Description
Benchmark	The imported Benchmark object

Raises:

Type	Description
FileNotFoundError	If file does not exist
InvalidBenchmarkError	If file is invalid or corrupted
ValueError	If version already exists and overwrite is False

Examples:

>>> # Import from file
>>> benchmark = versioning.import_benchmark("benchmark_v1.json")
>>> print(f"Imported {benchmark.metadata.version}")

>>> # Import and overwrite existing
>>> benchmark = versioning.import_benchmark(
...     "benchmark_v1.json",
...     overwrite=True
... )

Source code in uaef/hitl/benchmark.py

def import_benchmark(
    self,
    filepath: str,
    overwrite: bool = False
) -> Benchmark:
    """
    Import a benchmark from a JSON file.

    This method loads a previously exported benchmark from a file,
    validates it, and adds it to the available benchmarks.

    Args:
        filepath: Path to the JSON file containing the benchmark
        overwrite: If True, overwrite existing benchmark with same version

    Returns:
        The imported Benchmark object

    Raises:
        FileNotFoundError: If file does not exist
        InvalidBenchmarkError: If file is invalid or corrupted
        ValueError: If version already exists and overwrite is False

    Examples:
        >>> # Import from file
        >>> benchmark = versioning.import_benchmark("benchmark_v1.json")
        >>> print(f"Imported {benchmark.metadata.version}")

        >>> # Import and overwrite existing
        >>> benchmark = versioning.import_benchmark(
        ...     "benchmark_v1.json",
        ...     overwrite=True
        ... )
    """
    # Read file
    try:
        path = Path(filepath)

        if not path.exists():
            raise FileNotFoundError(f"Benchmark file not found: {filepath}")

        with open(path, "r") as f:
            benchmark_dict = json.load(f)

    except json.JSONDecodeError as e:
        raise InvalidBenchmarkError(f"Invalid JSON in {filepath}: {e}")
    except Exception as e:
        raise IOError(f"Failed to read benchmark from {filepath}: {e}")

    # Validate and reconstruct benchmark
    try:
        # Reconstruct metadata
        metadata = BenchmarkMetadata(**benchmark_dict["metadata"])

        # Reconstruct review items
        items = [
            ReviewItem(**item_dict)
            for item_dict in benchmark_dict["items"]
        ]

        # Reconstruct benchmark
        benchmark = Benchmark(
            benchmark_id=UUID(benchmark_dict["benchmark_id"]),
            metadata=metadata,
            items=items
        )

    except Exception as e:
        raise InvalidBenchmarkError(
            f"Failed to reconstruct benchmark from {filepath}: {e}"
        )

    # Check if version already exists
    version = metadata.version

    if version in self._benchmarks and not overwrite:
        raise ValueError(
            f"Benchmark version '{version}' already exists. "
            f"Use overwrite=True to replace it."
        )

    # Store benchmark
    self._benchmarks[version] = benchmark

    logger.info(
        f"Imported benchmark '{version}' from {filepath} "
        f"({len(items)} items)"
    )

    return benchmark

list_benchmarks(metric_name=None, item_type=None, start_date=None, end_date=None, tags=None)

List all available benchmarks with optional filtering.

Parameters:

Name	Type	Description	Default
metric_name	Optional[str]	Optional filter by metric name	None
item_type	Optional[str]	Optional filter by item type	None
start_date	Optional[datetime]	Optional filter by creation date (inclusive)	None
end_date	Optional[datetime]	Optional filter by creation date (inclusive)	None
tags	Optional[List[str]]	Optional filter by tags (must have all specified tags)	None

Returns:

Type	Description
List[Dict[str, Any]]	List of benchmark summaries, each containing: - version: Version identifier - creation_date: When the benchmark was created - sample_size: Number of items - metric_names: List of metrics - item_types: List of item types - description: Benchmark description - tags: List of tags

Examples:

>>> # List all benchmarks
>>> benchmarks = versioning.list_benchmarks()
>>> for b in benchmarks:
...     print(f"{b['version']}: {b['sample_size']} items")

>>> # Filter by metric
>>> benchmarks = versioning.list_benchmarks(
...     metric_name="answer_relevance"
... )

>>> # Filter by date range
>>> benchmarks = versioning.list_benchmarks(
...     start_date=datetime(2024, 1, 1),
...     end_date=datetime(2024, 12, 31)
... )

>>> # Filter by tags
>>> benchmarks = versioning.list_benchmarks(
...     tags=["production", "validated"]
... )

Source code in uaef/hitl/benchmark.py

def list_benchmarks(
    self,
    metric_name: Optional[str] = None,
    item_type: Optional[str] = None,
    start_date: Optional[datetime] = None,
    end_date: Optional[datetime] = None,
    tags: Optional[List[str]] = None
) -> List[Dict[str, Any]]:
    """
    List all available benchmarks with optional filtering.

    Args:
        metric_name: Optional filter by metric name
        item_type: Optional filter by item type
        start_date: Optional filter by creation date (inclusive)
        end_date: Optional filter by creation date (inclusive)
        tags: Optional filter by tags (must have all specified tags)

    Returns:
        List of benchmark summaries, each containing:
            - version: Version identifier
            - creation_date: When the benchmark was created
            - sample_size: Number of items
            - metric_names: List of metrics
            - item_types: List of item types
            - description: Benchmark description
            - tags: List of tags

    Examples:
        >>> # List all benchmarks
        >>> benchmarks = versioning.list_benchmarks()
        >>> for b in benchmarks:
        ...     print(f"{b['version']}: {b['sample_size']} items")

        >>> # Filter by metric
        >>> benchmarks = versioning.list_benchmarks(
        ...     metric_name="answer_relevance"
        ... )

        >>> # Filter by date range
        >>> benchmarks = versioning.list_benchmarks(
        ...     start_date=datetime(2024, 1, 1),
        ...     end_date=datetime(2024, 12, 31)
        ... )

        >>> # Filter by tags
        >>> benchmarks = versioning.list_benchmarks(
        ...     tags=["production", "validated"]
        ... )
    """
    results = []

    for version, benchmark in self._benchmarks.items():
        metadata = benchmark.metadata

        # Apply filters
        if metric_name and metric_name not in metadata.metric_names:
            continue

        if item_type and item_type not in metadata.item_types:
            continue

        if start_date and metadata.creation_date < start_date:
            continue

        if end_date and metadata.creation_date > end_date:
            continue

        if tags and not all(tag in metadata.tags for tag in tags):
            continue

        # Add to results
        results.append({
            "version": metadata.version,
            "creation_date": metadata.creation_date.isoformat(),
            "sample_size": metadata.sample_size,
            "metric_names": metadata.metric_names,
            "item_types": metadata.item_types,
            "description": metadata.description,
            "tags": metadata.tags,
            "benchmark_id": str(benchmark.benchmark_id)
        })

    # Sort by creation date (newest first)
    results.sort(key=lambda x: x["creation_date"], reverse=True)

    logger.info(f"Listed {len(results)} benchmarks")

    return results

BenchmarkVersioningError

Bases: Exception

Base exception for benchmark versioning errors.

Source code in uaef/hitl/benchmark.py

class BenchmarkVersioningError(Exception):
    """Base exception for benchmark versioning errors."""
    pass

InvalidBenchmarkError

Bases: BenchmarkVersioningError

Raised when a benchmark is invalid or corrupted.

Source code in uaef/hitl/benchmark.py

class InvalidBenchmarkError(BenchmarkVersioningError):
    """Raised when a benchmark is invalid or corrupted."""
    pass