Skip to content

Models

The canonical data model. Every adapter produces these types, and every metric consumes them. See Core Concepts for how they relate.

uaef.models.agent_trace

AgentTrace data model.

AgentTrace

Bases: BaseModel

Canonical representation of an agent execution trace.

This is the standardized format that all framework-specific traces are transformed into for evaluation.

Attributes:

Name Type Description
trace_id UUID

Unique identifier for this trace
session_id Optional[str]

Identifier for the conversation session (optional)
messages List[Message]

List of messages in the conversation
tool_calls List[ToolCall]

List of all tool calls made during execution
input_tokens Optional[int]

Number of input tokens consumed (optional)
output_tokens Optional[int]

Number of output tokens generated (optional)
latency Optional[float]

Total execution time in seconds (optional)
cost Optional[float]

Estimated cost in USD (optional)
metadata Dict[str, Any]

Additional metadata about the trace
framework Optional[str]

Name of the agent framework used (optional)
timestamp datetime

When the trace was created
Source code in uaef/models/agent_trace.py 
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
class AgentTrace(BaseModel):
    """
    Canonical representation of an agent execution trace.

    This is the standardized format that all framework-specific traces
    are transformed into for evaluation.

    Attributes:
        trace_id: Unique identifier for this trace
        session_id: Identifier for the conversation session (optional)
        messages: List of messages in the conversation
        tool_calls: List of all tool calls made during execution
        input_tokens: Number of input tokens consumed (optional)
        output_tokens: Number of output tokens generated (optional)
        latency: Total execution time in seconds (optional)
        cost: Estimated cost in USD (optional)
        metadata: Additional metadata about the trace
        framework: Name of the agent framework used (optional)
        timestamp: When the trace was created
    """

    trace_id: UUID = Field(default_factory=uuid4, description="Unique identifier for this trace")
    session_id: Optional[str] = Field(None, description="Identifier for the conversation session")
    messages: List[Message] = Field(
        default_factory=list,
        description="List of messages in the conversation"
    )
    tool_calls: List[ToolCall] = Field(
        default_factory=list,
        description="List of all tool calls made during execution"
    )
    input_tokens: Optional[int] = Field(
        None,
        description="Number of input tokens consumed",
        ge=0
    )
    output_tokens: Optional[int] = Field(
        None,
        description="Number of output tokens generated",
        ge=0
    )
    latency: Optional[float] = Field(
        None,
        description="Total execution time in seconds",
        ge=0.0
    )
    cost: Optional[float] = Field(
        None,
        description="Estimated cost in USD",
        ge=0.0
    )
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional metadata about the trace"
    )
    framework: Optional[str] = Field(None, description="Name of the agent framework used")
    timestamp: datetime = Field(
        default_factory=datetime.utcnow,
        description="When the trace was created"
    )

    @field_validator("trace_id")
    @classmethod
    def validate_trace_id(cls, v: UUID) -> UUID:
        """Validate that trace_id is a valid UUID."""
        if v is None:
            raise ValueError("trace_id cannot be None")
        return v

    @field_validator("timestamp")
    @classmethod
    def validate_timestamp(cls, v: datetime) -> datetime:
        """Validate timestamp format."""
        if v is None:
            raise ValueError("Timestamp cannot be None")
        return v

    @field_validator("input_tokens", "output_tokens")
    @classmethod
    def validate_token_counts(cls, v: Optional[int]) -> Optional[int]:
        """Validate that token counts are non-negative."""
        if v is not None and v < 0:
            raise ValueError("Token counts must be non-negative")
        return v

    @field_validator("latency", "cost")
    @classmethod
    def validate_positive_values(cls, v: Optional[float]) -> Optional[float]:
        """Validate that latency and cost are non-negative."""
        if v is not None and v < 0:
            raise ValueError("Latency and cost must be non-negative")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "trace_id": "550e8400-e29b-41d4-a716-446655440000",
                "session_id": "session_123",
                "messages": [
                    {
                        "role": "user",
                        "content": "Find customer orders",
                        "timestamp": "2024-01-15T10:30:00Z"
                    }
                ],
                "tool_calls": [],
                "input_tokens": 150,
                "output_tokens": 75,
                "latency": 1.234,
                "cost": 0.0025,
                "metadata": {"model": "gpt-4"},
                "framework": "langgraph",
                "timestamp": "2024-01-15T10:30:00Z"
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/agent_trace.py 
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "trace_id": "550e8400-e29b-41d4-a716-446655440000",
            "session_id": "session_123",
            "messages": [
                {
                    "role": "user",
                    "content": "Find customer orders",
                    "timestamp": "2024-01-15T10:30:00Z"
                }
            ],
            "tool_calls": [],
            "input_tokens": 150,
            "output_tokens": 75,
            "latency": 1.234,
            "cost": 0.0025,
            "metadata": {"model": "gpt-4"},
            "framework": "langgraph",
            "timestamp": "2024-01-15T10:30:00Z"
        }
    }

validate_positive_values(v) classmethod

Validate that latency and cost are non-negative.

Source code in uaef/models/agent_trace.py 
101
102
103
104
105
106
107
@field_validator("latency", "cost")
@classmethod
def validate_positive_values(cls, v: Optional[float]) -> Optional[float]:
    """Validate that latency and cost are non-negative."""
    if v is not None and v < 0:
        raise ValueError("Latency and cost must be non-negative")
    return v

validate_timestamp(v) classmethod

Validate timestamp format.

Source code in uaef/models/agent_trace.py 
85
86
87
88
89
90
91
@field_validator("timestamp")
@classmethod
def validate_timestamp(cls, v: datetime) -> datetime:
    """Validate timestamp format."""
    if v is None:
        raise ValueError("Timestamp cannot be None")
    return v

validate_token_counts(v) classmethod

Validate that token counts are non-negative.

Source code in uaef/models/agent_trace.py 
93
94
95
96
97
98
99
@field_validator("input_tokens", "output_tokens")
@classmethod
def validate_token_counts(cls, v: Optional[int]) -> Optional[int]:
    """Validate that token counts are non-negative."""
    if v is not None and v < 0:
        raise ValueError("Token counts must be non-negative")
    return v

validate_trace_id(v) classmethod

Validate that trace_id is a valid UUID.

Source code in uaef/models/agent_trace.py 
77
78
79
80
81
82
83
@field_validator("trace_id")
@classmethod
def validate_trace_id(cls, v: UUID) -> UUID:
    """Validate that trace_id is a valid UUID."""
    if v is None:
        raise ValueError("trace_id cannot be None")
    return v

uaef.models.message

Message data model.

Message

Bases: BaseModel

Represents a single message in an agent conversation.

Attributes:

Name Type Description
role MessageRole

Role of the message sender (user, assistant, system, tool)
content str

Text content of the message
tool_calls List[ToolCall]

List of tool calls made in this message (optional)
metadata Dict[str, Any]

Additional metadata about the message (optional)
timestamp datetime

When the message was created
Source code in uaef/models/message.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
class Message(BaseModel):
    """
    Represents a single message in an agent conversation.

    Attributes:
        role: Role of the message sender (user, assistant, system, tool)
        content: Text content of the message
        tool_calls: List of tool calls made in this message (optional)
        metadata: Additional metadata about the message (optional)
        timestamp: When the message was created
    """

    role: MessageRole = Field(..., description="Role of the message sender")
    content: str = Field(..., description="Text content of the message")
    tool_calls: List[ToolCall] = Field(
        default_factory=list,
        description="List of tool calls made in this message"
    )
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional metadata about the message"
    )
    timestamp: datetime = Field(..., description="When the message was created")

    @field_validator("role")
    @classmethod
    def validate_role(cls, v: MessageRole) -> MessageRole:
        """Validate that role is a valid enum value."""
        if not isinstance(v, MessageRole):
            raise ValueError(f"Invalid role: {v}. Must be one of {list(MessageRole)}")
        return v

    @field_validator("timestamp")
    @classmethod
    def validate_timestamp(cls, v: datetime) -> datetime:
        """Validate timestamp format."""
        if v is None:
            raise ValueError("Timestamp cannot be None")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "role": "assistant",
                "content": "I'll search the database for customer orders.",
                "tool_calls": [
                    {
                        "name": "search_database",
                        "arguments": {"query": "customer orders"},
                        "timestamp": "2024-01-15T10:30:00Z"
                    }
                ],
                "metadata": {"model": "gpt-4", "temperature": 0.7},
                "timestamp": "2024-01-15T10:30:00Z"
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/message.py 
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "role": "assistant",
            "content": "I'll search the database for customer orders.",
            "tool_calls": [
                {
                    "name": "search_database",
                    "arguments": {"query": "customer orders"},
                    "timestamp": "2024-01-15T10:30:00Z"
                }
            ],
            "metadata": {"model": "gpt-4", "temperature": 0.7},
            "timestamp": "2024-01-15T10:30:00Z"
        }
    }

validate_role(v) classmethod

Validate that role is a valid enum value.

Source code in uaef/models/message.py 
48
49
50
51
52
53
54
@field_validator("role")
@classmethod
def validate_role(cls, v: MessageRole) -> MessageRole:
    """Validate that role is a valid enum value."""
    if not isinstance(v, MessageRole):
        raise ValueError(f"Invalid role: {v}. Must be one of {list(MessageRole)}")
    return v

validate_timestamp(v) classmethod

Validate timestamp format.

Source code in uaef/models/message.py 
56
57
58
59
60
61
62
@field_validator("timestamp")
@classmethod
def validate_timestamp(cls, v: datetime) -> datetime:
    """Validate timestamp format."""
    if v is None:
        raise ValueError("Timestamp cannot be None")
    return v

MessageRole

Bases: str, Enum

Role of the message sender.

Source code in uaef/models/message.py 
15
16
17
18
19
20
21
class MessageRole(str, Enum):
    """Role of the message sender."""

    USER = "user"
    ASSISTANT = "assistant"
    SYSTEM = "system"
    TOOL = "tool"

uaef.models.tool_call

ToolCall data model.

ToolCall

Bases: BaseModel

Represents a single tool invocation by an agent.

Attributes:

Name Type Description
name str

Name of the tool that was called
arguments Dict[str, Any]

Dictionary of arguments passed to the tool
result Optional[Any]

Result returned by the tool (optional)
timestamp datetime

When the tool was called
execution_time Optional[float]

Time taken to execute the tool in seconds (optional)
error Optional[str]

Error message if tool execution failed (optional)
Source code in uaef/models/tool_call.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
class ToolCall(BaseModel):
    """
    Represents a single tool invocation by an agent.

    Attributes:
        name: Name of the tool that was called
        arguments: Dictionary of arguments passed to the tool
        result: Result returned by the tool (optional)
        timestamp: When the tool was called
        execution_time: Time taken to execute the tool in seconds (optional)
        error: Error message if tool execution failed (optional)
    """

    name: str = Field(..., description="Name of the tool that was called", min_length=1)
    arguments: Dict[str, Any] = Field(
        default_factory=dict,
        description="Dictionary of arguments passed to the tool"
    )
    result: Optional[Any] = Field(default=None, description="Result returned by the tool")
    timestamp: datetime = Field(..., description="When the tool was called")
    execution_time: Optional[float] = Field(
        default=None,
        description="Time taken to execute the tool in seconds",
        ge=0.0
    )
    error: Optional[str] = Field(default=None, description="Error message if tool execution failed")

    @field_validator("name")
    @classmethod
    def validate_name(cls, v: str) -> str:
        """Validate that tool name is not empty."""
        if not v or not v.strip():
            raise ValueError("Tool name cannot be empty")
        return v.strip()

    @field_validator("timestamp")
    @classmethod
    def validate_timestamp(cls, v: datetime) -> datetime:
        """Validate timestamp format."""
        if v is None:
            raise ValueError("Timestamp cannot be None")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "name": "search_database",
                "arguments": {"query": "customer orders", "limit": 10},
                "result": {"count": 5, "items": []},
                "timestamp": "2024-01-15T10:30:00Z",
                "execution_time": 0.234,
                "error": None
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/tool_call.py 
55
56
57
58
59
60
61
62
63
64
65
66
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "name": "search_database",
            "arguments": {"query": "customer orders", "limit": 10},
            "result": {"count": 5, "items": []},
            "timestamp": "2024-01-15T10:30:00Z",
            "execution_time": 0.234,
            "error": None
        }
    }

validate_name(v) classmethod

Validate that tool name is not empty.

Source code in uaef/models/tool_call.py 
39
40
41
42
43
44
45
@field_validator("name")
@classmethod
def validate_name(cls, v: str) -> str:
    """Validate that tool name is not empty."""
    if not v or not v.strip():
        raise ValueError("Tool name cannot be empty")
    return v.strip()

validate_timestamp(v) classmethod

Validate timestamp format.

Source code in uaef/models/tool_call.py 
47
48
49
50
51
52
53
@field_validator("timestamp")
@classmethod
def validate_timestamp(cls, v: datetime) -> datetime:
    """Validate timestamp format."""
    if v is None:
        raise ValueError("Timestamp cannot be None")
    return v

uaef.models.multi_agent_trace

MultiAgentTrace data model.

CoordinationEvent

Bases: BaseModel

Represents a coordination event between agents.

Attributes:

Name Type Description
event_id UUID

Unique identifier for this event
from_agent str

ID of the agent initiating the coordination
to_agent str

ID of the agent receiving the coordination
event_type str

Type of coordination (delegation, handoff, collaboration)
message str

Description of the coordination event
timestamp datetime

When the event occurred
metadata Dict[str, Any]

Additional metadata about the event
Source code in uaef/models/multi_agent_trace.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class CoordinationEvent(BaseModel):
    """
    Represents a coordination event between agents.

    Attributes:
        event_id: Unique identifier for this event
        from_agent: ID of the agent initiating the coordination
        to_agent: ID of the agent receiving the coordination
        event_type: Type of coordination (delegation, handoff, collaboration)
        message: Description of the coordination event
        timestamp: When the event occurred
        metadata: Additional metadata about the event
    """

    event_id: UUID = Field(default_factory=uuid4, description="Unique identifier for this event")
    from_agent: str = Field(..., description="ID of the agent initiating the coordination")
    to_agent: str = Field(..., description="ID of the agent receiving the coordination")
    event_type: str = Field(..., description="Type of coordination event")
    message: str = Field(..., description="Description of the coordination event")
    timestamp: datetime = Field(..., description="When the event occurred")
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional metadata about the event"
    )

    @field_validator("from_agent", "to_agent", "event_type", "message")
    @classmethod
    def validate_non_empty(cls, v: str) -> str:
        """Validate that string fields are not empty."""
        if not v or not v.strip():
            raise ValueError("Field cannot be empty")
        return v.strip()

validate_non_empty(v) classmethod

Validate that string fields are not empty.

Source code in uaef/models/multi_agent_trace.py 
51
52
53
54
55
56
57
@field_validator("from_agent", "to_agent", "event_type", "message")
@classmethod
def validate_non_empty(cls, v: str) -> str:
    """Validate that string fields are not empty."""
    if not v or not v.strip():
        raise ValueError("Field cannot be empty")
    return v.strip()

MultiAgentTrace

Bases: BaseModel

Canonical representation of a multi-agent execution trace.

Attributes:

Name Type Description
trace_id UUID

Unique identifier for this multi-agent trace
session_id Optional[str]

Identifier for the conversation session (optional)
agent_traces Dict[str, AgentTrace]

Dictionary mapping agent_id to AgentTrace
coordination_events List[CoordinationEvent]

List of coordination events between agents
workflow_status WorkflowStatus

Status of the overall workflow
total_latency Optional[float]

Total execution time across all agents in seconds (optional)
total_cost Optional[float]

Total estimated cost across all agents in USD (optional)
metadata Dict[str, Any]

Additional metadata about the multi-agent trace
timestamp datetime

When the trace was created
Source code in uaef/models/multi_agent_trace.py 
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
class MultiAgentTrace(BaseModel):
    """
    Canonical representation of a multi-agent execution trace.

    Attributes:
        trace_id: Unique identifier for this multi-agent trace
        session_id: Identifier for the conversation session (optional)
        agent_traces: Dictionary mapping agent_id to AgentTrace
        coordination_events: List of coordination events between agents
        workflow_status: Status of the overall workflow
        total_latency: Total execution time across all agents in seconds (optional)
        total_cost: Total estimated cost across all agents in USD (optional)
        metadata: Additional metadata about the multi-agent trace
        timestamp: When the trace was created
    """

    trace_id: UUID = Field(default_factory=uuid4, description="Unique identifier for this trace")
    session_id: Optional[str] = Field(None, description="Identifier for the conversation session")
    agent_traces: Dict[str, AgentTrace] = Field(
        default_factory=dict,
        description="Dictionary mapping agent_id to AgentTrace"
    )
    coordination_events: List[CoordinationEvent] = Field(
        default_factory=list,
        description="List of coordination events between agents"
    )
    workflow_status: WorkflowStatus = Field(
        ...,
        description="Status of the overall workflow"
    )
    total_latency: Optional[float] = Field(
        None,
        description="Total execution time across all agents in seconds",
        ge=0.0
    )
    total_cost: Optional[float] = Field(
        None,
        description="Total estimated cost across all agents in USD",
        ge=0.0
    )
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional metadata about the multi-agent trace"
    )
    timestamp: datetime = Field(
        default_factory=datetime.utcnow,
        description="When the trace was created"
    )

    @field_validator("trace_id")
    @classmethod
    def validate_trace_id(cls, v: UUID) -> UUID:
        """Validate that trace_id is a valid UUID."""
        if v is None:
            raise ValueError("trace_id cannot be None")
        return v

    @field_validator("workflow_status")
    @classmethod
    def validate_workflow_status(cls, v: WorkflowStatus) -> WorkflowStatus:
        """Validate that workflow_status is a valid enum value."""
        if not isinstance(v, WorkflowStatus):
            raise ValueError(f"Invalid workflow_status: {v}")
        return v

    @field_validator("total_latency", "total_cost")
    @classmethod
    def validate_positive_values(cls, v: Optional[float]) -> Optional[float]:
        """Validate that latency and cost are non-negative."""
        if v is not None and v < 0:
            raise ValueError("Latency and cost must be non-negative")
        return v

    @computed_field  # type: ignore[misc]
    @property
    def messages(self) -> List[Message]:
        """Aggregate messages from all sub-agent traces in order."""
        all_messages: List[Message] = []
        for trace in self.agent_traces.values():
            all_messages.extend(trace.messages)
        return all_messages

    @computed_field  # type: ignore[misc]
    @property
    def tool_calls(self) -> List[ToolCall]:
        """Aggregate tool calls from all sub-agent traces in order."""
        all_tool_calls: List[ToolCall] = []
        for trace in self.agent_traces.values():
            all_tool_calls.extend(trace.tool_calls)
        return all_tool_calls

    @computed_field  # type: ignore[misc]
    @property
    def input_tokens(self) -> Optional[int]:
        """Sum input tokens from all sub-agent traces."""
        totals = [t.input_tokens for t in self.agent_traces.values() if t.input_tokens is not None]
        return sum(totals) if totals else None

    @computed_field  # type: ignore[misc]
    @property
    def output_tokens(self) -> Optional[int]:
        """Sum output tokens from all sub-agent traces."""
        totals = [t.output_tokens for t in self.agent_traces.values() if t.output_tokens is not None]
        return sum(totals) if totals else None

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "trace_id": "550e8400-e29b-41d4-a716-446655440000",
                "session_id": "session_123",
                "agent_traces": {
                    "coordinator": {},
                    "specialist": {}
                },
                "coordination_events": [
                    {
                        "from_agent": "coordinator",
                        "to_agent": "specialist",
                        "event_type": "delegation",
                        "message": "Delegating task to specialist",
                        "timestamp": "2024-01-15T10:30:00Z"
                    }
                ],
                "workflow_status": "completed",
                "total_latency": 2.5,
                "total_cost": 0.005,
                "metadata": {},
                "timestamp": "2024-01-15T10:30:00Z"
            }
        }

input_tokens property

Sum input tokens from all sub-agent traces.

messages property

Aggregate messages from all sub-agent traces in order.

output_tokens property

Sum output tokens from all sub-agent traces.

tool_calls property

Aggregate tool calls from all sub-agent traces in order.

Config

Pydantic configuration.

Source code in uaef/models/multi_agent_trace.py 
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "trace_id": "550e8400-e29b-41d4-a716-446655440000",
            "session_id": "session_123",
            "agent_traces": {
                "coordinator": {},
                "specialist": {}
            },
            "coordination_events": [
                {
                    "from_agent": "coordinator",
                    "to_agent": "specialist",
                    "event_type": "delegation",
                    "message": "Delegating task to specialist",
                    "timestamp": "2024-01-15T10:30:00Z"
                }
            ],
            "workflow_status": "completed",
            "total_latency": 2.5,
            "total_cost": 0.005,
            "metadata": {},
            "timestamp": "2024-01-15T10:30:00Z"
        }
    }

validate_positive_values(v) classmethod

Validate that latency and cost are non-negative.

Source code in uaef/models/multi_agent_trace.py 
125
126
127
128
129
130
131
@field_validator("total_latency", "total_cost")
@classmethod
def validate_positive_values(cls, v: Optional[float]) -> Optional[float]:
    """Validate that latency and cost are non-negative."""
    if v is not None and v < 0:
        raise ValueError("Latency and cost must be non-negative")
    return v

validate_trace_id(v) classmethod

Validate that trace_id is a valid UUID.

Source code in uaef/models/multi_agent_trace.py 
109
110
111
112
113
114
115
@field_validator("trace_id")
@classmethod
def validate_trace_id(cls, v: UUID) -> UUID:
    """Validate that trace_id is a valid UUID."""
    if v is None:
        raise ValueError("trace_id cannot be None")
    return v

validate_workflow_status(v) classmethod

Validate that workflow_status is a valid enum value.

Source code in uaef/models/multi_agent_trace.py 
117
118
119
120
121
122
123
@field_validator("workflow_status")
@classmethod
def validate_workflow_status(cls, v: WorkflowStatus) -> WorkflowStatus:
    """Validate that workflow_status is a valid enum value."""
    if not isinstance(v, WorkflowStatus):
        raise ValueError(f"Invalid workflow_status: {v}")
    return v

WorkflowStatus

Bases: str, Enum

Status of a multi-agent workflow.

Source code in uaef/models/multi_agent_trace.py 
18
19
20
21
22
23
class WorkflowStatus(str, Enum):
    """Status of a multi-agent workflow."""

    COMPLETED = "completed"
    FAILED = "failed"
    PARTIAL = "partial"

uaef.models.ground_truth

GroundTruth data model.

GroundTruth

Bases: BaseModel

Expected correct outputs for validating agent responses.

Attributes:

Name Type Description
expected_output Optional[str]

Expected text response from the agent (optional)
expected_tool_calls List[ToolCall]

List of expected tool calls (optional)
expected_arguments Dict[str, Any]

Dictionary of expected arguments for tools (optional)
context_documents List[str]

List of context documents that should be used (optional)
Source code in uaef/models/ground_truth.py 
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
class GroundTruth(BaseModel):
    """
    Expected correct outputs for validating agent responses.

    Attributes:
        expected_output: Expected text response from the agent (optional)
        expected_tool_calls: List of expected tool calls (optional)
        expected_arguments: Dictionary of expected arguments for tools (optional)
        context_documents: List of context documents that should be used (optional)
    """

    expected_output: Optional[str] = Field(
        None,
        description="Expected text response from the agent"
    )
    expected_tool_calls: List[ToolCall] = Field(
        default_factory=list,
        description="List of expected tool calls"
    )
    expected_arguments: Dict[str, Any] = Field(
        default_factory=dict,
        description="Dictionary of expected arguments for tools"
    )
    context_documents: List[str] = Field(
        default_factory=list,
        description="List of context documents that should be used"
    )

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "expected_output": "I found 5 customer orders matching your query.",
                "expected_tool_calls": [
                    {
                        "name": "search_database",
                        "arguments": {"query": "customer orders", "limit": 10},
                        "timestamp": "2024-01-15T10:30:00Z"
                    }
                ],
                "expected_arguments": {
                    "search_database": {"query": "customer orders"}
                },
                "context_documents": [
                    "Customer database contains order history...",
                    "Search API documentation..."
                ]
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/ground_truth.py 
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "expected_output": "I found 5 customer orders matching your query.",
            "expected_tool_calls": [
                {
                    "name": "search_database",
                    "arguments": {"query": "customer orders", "limit": 10},
                    "timestamp": "2024-01-15T10:30:00Z"
                }
            ],
            "expected_arguments": {
                "search_database": {"query": "customer orders"}
            },
            "context_documents": [
                "Customer database contains order history...",
                "Search API documentation..."
            ]
        }
    }

uaef.models.evaluation_input

EvaluationInput data model.

EvaluationInput

Bases: BaseModel

Input data for evaluation.

Attributes:

Name Type Description
trace Union[AgentTrace, MultiAgentTrace]

Agent trace to evaluate (single or multi-agent)
ground_truth Optional[GroundTruth]

Expected correct outputs (optional)
context List[str]

List of context documents (optional)
evaluation_config List[str]

Configuration for evaluation (will be added separately)
Source code in uaef/models/evaluation_input.py 
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
class EvaluationInput(BaseModel):
    """
    Input data for evaluation.

    Attributes:
        trace: Agent trace to evaluate (single or multi-agent)
        ground_truth: Expected correct outputs (optional)
        context: List of context documents (optional)
        evaluation_config: Configuration for evaluation (will be added separately)
    """

    trace: Union[AgentTrace, MultiAgentTrace] = Field(
        ...,
        description="Agent trace to evaluate (single or multi-agent)"
    )
    ground_truth: Optional[GroundTruth] = Field(
        None,
        description="Expected correct outputs"
    )
    context: List[str] = Field(
        default_factory=list,
        description="List of context documents"
    )

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "trace": {
                    "trace_id": "550e8400-e29b-41d4-a716-446655440000",
                    "messages": [],
                    "tool_calls": []
                },
                "ground_truth": {
                    "expected_output": "Expected response",
                    "expected_tool_calls": []
                },
                "context": ["Context document 1", "Context document 2"]
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/evaluation_input.py 
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "trace": {
                "trace_id": "550e8400-e29b-41d4-a716-446655440000",
                "messages": [],
                "tool_calls": []
            },
            "ground_truth": {
                "expected_output": "Expected response",
                "expected_tool_calls": []
            },
            "context": ["Context document 1", "Context document 2"]
        }
    }

uaef.models.metric_score

MetricScore data model.

MetricScore

Bases: BaseModel

Score for a single metric.

Attributes:

Name Type Description
metric_name str

Name of the metric
score Optional[float]

Numeric score between 0 and 1, or None if metric is not applicable
reasoning Optional[str]

Explanation of the score (optional, typically from LLM judge)
metadata Dict[str, Any]

Additional metadata about the score calculation
timestamp datetime

When the score was calculated
Source code in uaef/models/metric_score.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
class MetricScore(BaseModel):
    """
    Score for a single metric.

    Attributes:
        metric_name: Name of the metric
        score: Numeric score between 0 and 1, or None if metric is not applicable
        reasoning: Explanation of the score (optional, typically from LLM judge)
        metadata: Additional metadata about the score calculation
        timestamp: When the score was calculated
    """

    metric_name: str = Field(..., description="Name of the metric", min_length=1)
    score: Optional[float] = Field(None, description="Numeric score between 0 and 1, or None if not applicable")
    reasoning: Optional[str] = Field(
        None,
        description="Explanation of the score"
    )
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional metadata about the score calculation"
    )
    timestamp: datetime = Field(
        default_factory=datetime.utcnow,
        description="When the score was calculated"
    )

    @field_validator("metric_name")
    @classmethod
    def validate_metric_name(cls, v: str) -> str:
        """Validate that metric name is not empty."""
        if not v or not v.strip():
            raise ValueError("Metric name cannot be empty")
        return v.strip()

    @field_validator("score")
    @classmethod
    def validate_score_bounds(cls, v: Optional[float]) -> Optional[float]:
        """Validate that score is between 0 and 1, or None."""
        if v is not None and not 0.0 <= v <= 1.0:
            raise ValueError(f"Score must be between 0 and 1, got {v}")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "metric_name": "tool_accuracy",
                "score": 0.85,
                "reasoning": "Agent called the correct tools with mostly accurate arguments",
                "metadata": {"incorrect_count": 1, "missed_count": 0},
                "timestamp": "2024-01-15T10:30:00Z"
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/metric_score.py 
55
56
57
58
59
60
61
62
63
64
65
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "metric_name": "tool_accuracy",
            "score": 0.85,
            "reasoning": "Agent called the correct tools with mostly accurate arguments",
            "metadata": {"incorrect_count": 1, "missed_count": 0},
            "timestamp": "2024-01-15T10:30:00Z"
        }
    }

validate_metric_name(v) classmethod

Validate that metric name is not empty.

Source code in uaef/models/metric_score.py 
39
40
41
42
43
44
45
@field_validator("metric_name")
@classmethod
def validate_metric_name(cls, v: str) -> str:
    """Validate that metric name is not empty."""
    if not v or not v.strip():
        raise ValueError("Metric name cannot be empty")
    return v.strip()

validate_score_bounds(v) classmethod

Validate that score is between 0 and 1, or None.

Source code in uaef/models/metric_score.py 
47
48
49
50
51
52
53
@field_validator("score")
@classmethod
def validate_score_bounds(cls, v: Optional[float]) -> Optional[float]:
    """Validate that score is between 0 and 1, or None."""
    if v is not None and not 0.0 <= v <= 1.0:
        raise ValueError(f"Score must be between 0 and 1, got {v}")
    return v

uaef.models.dimension_result

DimensionResult data model.

DimensionResult

Bases: BaseModel

Aggregated result for a single evaluation dimension.

Attributes:

Name Type Description
dimension_name str

Name of the dimension (e.g., "Tool Calling", "Response Quality")
metric_scores List[MetricScore]

List of individual metric scores in this dimension
aggregate_score Optional[float]

Weighted average score for the dimension (0-1), or None if no metric in the dimension was applicable
weight float

Weight of this dimension in overall score calculation
Source code in uaef/models/dimension_result.py 
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
class DimensionResult(BaseModel):
    """
    Aggregated result for a single evaluation dimension.

    Attributes:
        dimension_name: Name of the dimension (e.g., "Tool Calling", "Response Quality")
        metric_scores: List of individual metric scores in this dimension
        aggregate_score: Weighted average score for the dimension (0-1), or
            None if no metric in the dimension was applicable
        weight: Weight of this dimension in overall score calculation
    """

    dimension_name: str = Field(
        ...,
        description="Name of the dimension",
        min_length=1
    )
    metric_scores: List[MetricScore] = Field(
        ...,
        description="List of individual metric scores in this dimension",
        min_length=1
    )
    aggregate_score: Optional[float] = Field(
        ...,
        description="Weighted average score for the dimension, or None if not applicable",
        ge=0.0,
        le=1.0
    )
    weight: float = Field(
        ...,
        description="Weight of this dimension in overall score calculation",
        ge=0.0,
        le=1.0
    )

    @field_validator("dimension_name")
    @classmethod
    def validate_dimension_name(cls, v: str) -> str:
        """Validate that dimension name is not empty."""
        if not v or not v.strip():
            raise ValueError("Dimension name cannot be empty")
        return v.strip()

    def scored_metrics(self) -> List[MetricScore]:
        """Metric scores that have a value.

        Excludes not-applicable or failed metrics (``score is None``), which
        callers must not compare or average.
        """
        return [m for m in self.metric_scores if m.score is not None]

    @field_validator("aggregate_score", "weight")
    @classmethod
    def validate_score_bounds(cls, v: Optional[float]) -> Optional[float]:
        """Validate that scores and weights are between 0 and 1."""
        if v is not None and not 0.0 <= v <= 1.0:
            raise ValueError(f"Value must be between 0 and 1, got {v}")
        return v

    @field_validator("metric_scores")
    @classmethod
    def validate_metric_scores(cls, v: List[MetricScore]) -> List[MetricScore]:
        """Validate that metric_scores is not empty."""
        if not v:
            raise ValueError("metric_scores cannot be empty")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "dimension_name": "Tool Calling",
                "metric_scores": [
                    {
                        "metric_name": "tool_accuracy",
                        "score": 0.85,
                        "timestamp": "2024-01-15T10:30:00Z"
                    },
                    {
                        "metric_name": "parameter_quality",
                        "score": 0.90,
                        "timestamp": "2024-01-15T10:30:00Z"
                    }
                ],
                "aggregate_score": 0.875,
                "weight": 0.4
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/dimension_result.py 
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "dimension_name": "Tool Calling",
            "metric_scores": [
                {
                    "metric_name": "tool_accuracy",
                    "score": 0.85,
                    "timestamp": "2024-01-15T10:30:00Z"
                },
                {
                    "metric_name": "parameter_quality",
                    "score": 0.90,
                    "timestamp": "2024-01-15T10:30:00Z"
                }
            ],
            "aggregate_score": 0.875,
            "weight": 0.4
        }
    }

scored_metrics()

Metric scores that have a value.

Excludes not-applicable or failed metrics (score is None), which callers must not compare or average.

Source code in uaef/models/dimension_result.py 
56
57
58
59
60
61
62
def scored_metrics(self) -> List[MetricScore]:
    """Metric scores that have a value.

    Excludes not-applicable or failed metrics (``score is None``), which
    callers must not compare or average.
    """
    return [m for m in self.metric_scores if m.score is not None]

validate_dimension_name(v) classmethod

Validate that dimension name is not empty.

Source code in uaef/models/dimension_result.py 
48
49
50
51
52
53
54
@field_validator("dimension_name")
@classmethod
def validate_dimension_name(cls, v: str) -> str:
    """Validate that dimension name is not empty."""
    if not v or not v.strip():
        raise ValueError("Dimension name cannot be empty")
    return v.strip()

validate_metric_scores(v) classmethod

Validate that metric_scores is not empty.

Source code in uaef/models/dimension_result.py 
72
73
74
75
76
77
78
@field_validator("metric_scores")
@classmethod
def validate_metric_scores(cls, v: List[MetricScore]) -> List[MetricScore]:
    """Validate that metric_scores is not empty."""
    if not v:
        raise ValueError("metric_scores cannot be empty")
    return v

validate_score_bounds(v) classmethod

Validate that scores and weights are between 0 and 1.

Source code in uaef/models/dimension_result.py 
64
65
66
67
68
69
70
@field_validator("aggregate_score", "weight")
@classmethod
def validate_score_bounds(cls, v: Optional[float]) -> Optional[float]:
    """Validate that scores and weights are between 0 and 1."""
    if v is not None and not 0.0 <= v <= 1.0:
        raise ValueError(f"Value must be between 0 and 1, got {v}")
    return v

uaef.models.evaluation_result

EvaluationResult data model.

EvaluationResult

Bases: BaseModel

Complete evaluation result for an agent trace.

Attributes:

Name Type Description
evaluation_id UUID

Unique identifier for this evaluation
trace_id UUID

ID of the trace that was evaluated
dimension_results List[DimensionResult]

List of results for each dimension
overall_score float

Overall weighted score across all dimensions (0-1)
passed bool

Whether the evaluation passed all thresholds
failures List[str]

List of failure messages for threshold violations
warnings List[str]

List of warning messages
metadata Dict[str, Any]

Additional metadata about the evaluation
timestamp datetime

When the evaluation was completed
Source code in uaef/models/evaluation_result.py 
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
class EvaluationResult(BaseModel):
    """
    Complete evaluation result for an agent trace.

    Attributes:
        evaluation_id: Unique identifier for this evaluation
        trace_id: ID of the trace that was evaluated
        dimension_results: List of results for each dimension
        overall_score: Overall weighted score across all dimensions (0-1)
        passed: Whether the evaluation passed all thresholds
        failures: List of failure messages for threshold violations
        warnings: List of warning messages
        metadata: Additional metadata about the evaluation
        timestamp: When the evaluation was completed
    """

    evaluation_id: UUID = Field(
        default_factory=uuid4,
        description="Unique identifier for this evaluation"
    )
    trace_id: UUID = Field(..., description="ID of the trace that was evaluated")
    experiment_id: Optional[UUID] = Field(
        None,
        description="ID of the experiment this evaluation belongs to (set when persist=True)"
    )
    dimension_results: List[DimensionResult] = Field(
        ...,
        description="List of results for each dimension"
    )
    overall_score: float = Field(
        ...,
        description="Overall weighted score across all dimensions",
        ge=0.0,
        le=1.0
    )
    passed: bool = Field(..., description="Whether the evaluation passed all thresholds")
    failures: List[str] = Field(
        default_factory=list,
        description="List of failure messages for threshold violations"
    )
    warnings: List[str] = Field(
        default_factory=list,
        description="List of warning messages"
    )
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional metadata about the evaluation"
    )
    timestamp: datetime = Field(
        default_factory=datetime.utcnow,
        description="When the evaluation was completed"
    )

    @field_validator("evaluation_id", "trace_id")
    @classmethod
    def validate_uuid(cls, v: UUID) -> UUID:
        """Validate that UUIDs are not None."""
        if v is None:
            raise ValueError("UUID cannot be None")
        return v

    @field_validator("overall_score")
    @classmethod
    def validate_score_bounds(cls, v: float) -> float:
        """Validate that overall score is between 0 and 1."""
        if not 0.0 <= v <= 1.0:
            raise ValueError(f"Overall score must be between 0 and 1, got {v}")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "evaluation_id": "550e8400-e29b-41d4-a716-446655440000",
                "trace_id": "660e8400-e29b-41d4-a716-446655440001",
                "dimension_results": [
                    {
                        "dimension_name": "Tool Calling",
                        "metric_scores": [],
                        "aggregate_score": 0.85,
                        "weight": 0.4
                    }
                ],
                "overall_score": 0.82,
                "passed": True,
                "failures": [],
                "warnings": ["Ground truth not provided for some metrics"],
                "metadata": {"evaluation_mode": "offline"},
                "timestamp": "2024-01-15T10:30:00Z"
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/evaluation_result.py 
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "evaluation_id": "550e8400-e29b-41d4-a716-446655440000",
            "trace_id": "660e8400-e29b-41d4-a716-446655440001",
            "dimension_results": [
                {
                    "dimension_name": "Tool Calling",
                    "metric_scores": [],
                    "aggregate_score": 0.85,
                    "weight": 0.4
                }
            ],
            "overall_score": 0.82,
            "passed": True,
            "failures": [],
            "warnings": ["Ground truth not provided for some metrics"],
            "metadata": {"evaluation_mode": "offline"},
            "timestamp": "2024-01-15T10:30:00Z"
        }
    }

validate_score_bounds(v) classmethod

Validate that overall score is between 0 and 1.

Source code in uaef/models/evaluation_result.py 
76
77
78
79
80
81
82
@field_validator("overall_score")
@classmethod
def validate_score_bounds(cls, v: float) -> float:
    """Validate that overall score is between 0 and 1."""
    if not 0.0 <= v <= 1.0:
        raise ValueError(f"Overall score must be between 0 and 1, got {v}")
    return v

validate_uuid(v) classmethod

Validate that UUIDs are not None.

Source code in uaef/models/evaluation_result.py 
68
69
70
71
72
73
74
@field_validator("evaluation_id", "trace_id")
@classmethod
def validate_uuid(cls, v: UUID) -> UUID:
    """Validate that UUIDs are not None."""
    if v is None:
        raise ValueError("UUID cannot be None")
    return v

uaef.models.evaluation_config

EvaluationConfig and MetricSet data models.

EvaluationConfig

Bases: BaseModel

Configuration for evaluation execution.

Attributes:

Name Type Description
metric_set MetricSet

Set of metrics to calculate
mode EvaluationMode

Evaluation mode (online or offline)
llm_judge_config Dict[str, Any]

Configuration for LLM judge (optional)
parallel_execution bool

Whether to execute metrics in parallel
timeout_seconds int

Timeout for evaluation in seconds (optional)
metadata Dict[str, Any]

Additional configuration metadata
Source code in uaef/models/evaluation_config.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
class EvaluationConfig(BaseModel):
    """
    Configuration for evaluation execution.

    Attributes:
        metric_set: Set of metrics to calculate
        mode: Evaluation mode (online or offline)
        llm_judge_config: Configuration for LLM judge (optional)
        parallel_execution: Whether to execute metrics in parallel
        timeout_seconds: Timeout for evaluation in seconds (optional)
        metadata: Additional configuration metadata
    """

    metric_set: MetricSet = Field(..., description="Set of metrics to calculate")
    mode: EvaluationMode = Field(
        default=EvaluationMode.OFFLINE,
        description="Evaluation mode"
    )
    llm_judge_config: Dict[str, Any] = Field(
        default_factory=dict,
        description="Configuration for LLM judge"
    )
    parallel_execution: bool = Field(
        default=True,
        description="Whether to execute metrics in parallel"
    )
    timeout_seconds: int = Field(
        default=300,
        description="Timeout for evaluation in seconds",
        gt=0
    )
    metadata: Dict[str, Any] = Field(
        default_factory=dict,
        description="Additional configuration metadata"
    )

    @field_validator("mode")
    @classmethod
    def validate_mode(cls, v: EvaluationMode) -> EvaluationMode:
        """Validate that mode is a valid enum value."""
        if not isinstance(v, EvaluationMode):
            raise ValueError(f"Invalid mode: {v}")
        return v

    @field_validator("timeout_seconds")
    @classmethod
    def validate_timeout(cls, v: int) -> int:
        """Validate that timeout is positive."""
        if v <= 0:
            raise ValueError("Timeout must be positive")
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "metric_set": {
                    "dimensions": ["Tool Calling", "Response Quality"],
                    "metrics": ["tool_accuracy", "answer_relevance"],
                    "weights": {"Tool Calling": 0.5, "Response Quality": 0.5}
                },
                "mode": "offline",
                "llm_judge_config": {
                    "model_id": "us.anthropic.claude-sonnet-4-6",
                    "region": "us-east-1"
                },
                "parallel_execution": True,
                "timeout_seconds": 300,
                "metadata": {}
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/evaluation_config.py 
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "metric_set": {
                "dimensions": ["Tool Calling", "Response Quality"],
                "metrics": ["tool_accuracy", "answer_relevance"],
                "weights": {"Tool Calling": 0.5, "Response Quality": 0.5}
            },
            "mode": "offline",
            "llm_judge_config": {
                "model_id": "us.anthropic.claude-sonnet-4-6",
                "region": "us-east-1"
            },
            "parallel_execution": True,
            "timeout_seconds": 300,
            "metadata": {}
        }
    }

validate_mode(v) classmethod

Validate that mode is a valid enum value.

Source code in uaef/models/evaluation_config.py 
133
134
135
136
137
138
139
@field_validator("mode")
@classmethod
def validate_mode(cls, v: EvaluationMode) -> EvaluationMode:
    """Validate that mode is a valid enum value."""
    if not isinstance(v, EvaluationMode):
        raise ValueError(f"Invalid mode: {v}")
    return v

validate_timeout(v) classmethod

Validate that timeout is positive.

Source code in uaef/models/evaluation_config.py 
141
142
143
144
145
146
147
@field_validator("timeout_seconds")
@classmethod
def validate_timeout(cls, v: int) -> int:
    """Validate that timeout is positive."""
    if v <= 0:
        raise ValueError("Timeout must be positive")
    return v

EvaluationMode

Bases: str, Enum

Mode of evaluation execution.

Source code in uaef/models/evaluation_config.py 
12
13
14
15
16
class EvaluationMode(str, Enum):
    """Mode of evaluation execution."""

    ONLINE = "online"
    OFFLINE = "offline"

MetricSet

Bases: BaseModel

Configuration for which metrics to calculate.

Attributes:

Name Type Description
dimensions List[str]

List of dimension names to evaluate
metrics List[str]

List of specific metric names to calculate
weights Dict[str, float]

Dictionary mapping dimension names to weights (0-1)
thresholds Dict[str, float]

Dictionary mapping dimension names to minimum passing scores (0-1)
Source code in uaef/models/evaluation_config.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
class MetricSet(BaseModel):
    """
    Configuration for which metrics to calculate.

    Attributes:
        dimensions: List of dimension names to evaluate
        metrics: List of specific metric names to calculate
        weights: Dictionary mapping dimension names to weights (0-1)
        thresholds: Dictionary mapping dimension names to minimum passing scores (0-1)
    """

    dimensions: List[str] = Field(
        ...,
        description="List of dimension names to evaluate",
        min_length=1
    )
    metrics: List[str] = Field(
        ...,
        description="List of specific metric names to calculate",
        min_length=1
    )
    weights: Dict[str, float] = Field(
        ...,
        description="Dictionary mapping dimension names to weights"
    )
    thresholds: Dict[str, float] = Field(
        default_factory=dict,
        description="Dictionary mapping dimension names to minimum passing scores"
    )

    @field_validator("dimensions", "metrics")
    @classmethod
    def validate_non_empty_list(cls, v: List[str]) -> List[str]:
        """Validate that lists are not empty."""
        if not v:
            raise ValueError("List cannot be empty")
        return v

    @field_validator("weights")
    @classmethod
    def validate_weights(cls, v: Dict[str, float]) -> Dict[str, float]:
        """Validate that all weights are between 0 and 1."""
        for dimension, weight in v.items():
            if not 0.0 <= weight <= 1.0:
                raise ValueError(f"Weight for {dimension} must be between 0 and 1, got {weight}")
        return v

    @field_validator("thresholds")
    @classmethod
    def validate_thresholds(cls, v: Dict[str, float]) -> Dict[str, float]:
        """Validate that all thresholds are between 0 and 1."""
        for dimension, threshold in v.items():
            if not 0.0 <= threshold <= 1.0:
                raise ValueError(
                    f"Threshold for {dimension} must be between 0 and 1, got {threshold}"
                )
        return v

    class Config:
        """Pydantic configuration."""
        json_schema_extra = {
            "example": {
                "dimensions": ["Tool Calling", "Response Quality", "Performance"],
                "metrics": ["tool_accuracy", "answer_relevance", "latency_score"],
                "weights": {
                    "Tool Calling": 0.4,
                    "Response Quality": 0.4,
                    "Performance": 0.2
                },
                "thresholds": {
                    "Tool Calling": 0.7,
                    "Response Quality": 0.8,
                    "Performance": 0.6
                }
            }
        }

Config

Pydantic configuration.

Source code in uaef/models/evaluation_config.py 
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
class Config:
    """Pydantic configuration."""
    json_schema_extra = {
        "example": {
            "dimensions": ["Tool Calling", "Response Quality", "Performance"],
            "metrics": ["tool_accuracy", "answer_relevance", "latency_score"],
            "weights": {
                "Tool Calling": 0.4,
                "Response Quality": 0.4,
                "Performance": 0.2
            },
            "thresholds": {
                "Tool Calling": 0.7,
                "Response Quality": 0.8,
                "Performance": 0.6
            }
        }
    }

validate_non_empty_list(v) classmethod

Validate that lists are not empty.

Source code in uaef/models/evaluation_config.py 
49
50
51
52
53
54
55
@field_validator("dimensions", "metrics")
@classmethod
def validate_non_empty_list(cls, v: List[str]) -> List[str]:
    """Validate that lists are not empty."""
    if not v:
        raise ValueError("List cannot be empty")
    return v

validate_thresholds(v) classmethod

Validate that all thresholds are between 0 and 1.

Source code in uaef/models/evaluation_config.py 
66
67
68
69
70
71
72
73
74
75
@field_validator("thresholds")
@classmethod
def validate_thresholds(cls, v: Dict[str, float]) -> Dict[str, float]:
    """Validate that all thresholds are between 0 and 1."""
    for dimension, threshold in v.items():
        if not 0.0 <= threshold <= 1.0:
            raise ValueError(
                f"Threshold for {dimension} must be between 0 and 1, got {threshold}"
            )
    return v

validate_weights(v) classmethod

Validate that all weights are between 0 and 1.

Source code in uaef/models/evaluation_config.py 
57
58
59
60
61
62
63
64
@field_validator("weights")
@classmethod
def validate_weights(cls, v: Dict[str, float]) -> Dict[str, float]:
    """Validate that all weights are between 0 and 1."""
    for dimension, weight in v.items():
        if not 0.0 <= weight <= 1.0:
            raise ValueError(f"Weight for {dimension} must be between 0 and 1, got {weight}")
    return v