Coverage for gco/models/health_models.py: 100.00%
73 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-30 21:22 +0000
1"""
2Health monitoring data models for GCO (Global Capacity Orchestrator on AWS).
4This module defines dataclasses for health monitoring including:
5- ResourceUtilization: Current CPU/memory/GPU utilization percentages
6- HealthStatus: Complete health status report with utilization and thresholds
8These models are used by the health monitor service to track and report
9cluster health status for load balancer health checks and monitoring.
10"""
12from __future__ import annotations
14from dataclasses import dataclass
15from datetime import datetime
16from typing import TYPE_CHECKING, Literal
18if TYPE_CHECKING:
19 from .cluster_models import ResourceThresholds
22@dataclass
23class RequestedResources:
24 """
25 Resources requested by pending pods (absolute values).
27 Attributes:
28 cpu_vcpus: Total vCPUs requested by pending pods
29 memory_gb: Total GB memory requested by pending pods
30 gpus: Total GPUs requested by pending pods
31 """
33 cpu_vcpus: float
34 memory_gb: float
35 gpus: int = 0
37 def __post_init__(self) -> None:
38 """Validate requested values"""
39 if not isinstance(self.cpu_vcpus, (int, float)) or self.cpu_vcpus < 0.0:
40 raise ValueError(f"cpu_vcpus must be a non-negative number, got {self.cpu_vcpus}")
41 if not isinstance(self.memory_gb, (int, float)) or self.memory_gb < 0.0:
42 raise ValueError(f"memory_gb must be a non-negative number, got {self.memory_gb}")
43 if not isinstance(self.gpus, int) or self.gpus < 0:
44 raise ValueError(f"gpus must be a non-negative integer, got {self.gpus}")
47@dataclass
48class ResourceUtilization:
49 """
50 Current resource utilization metrics for a cluster.
52 Attributes:
53 cpu: CPU utilization percentage (0.0-100.0)
54 memory: Memory utilization percentage (0.0-100.0)
55 gpu: GPU utilization percentage (0.0-100.0)
56 """
58 cpu: float
59 memory: float
60 gpu: float
62 def __post_init__(self) -> None:
63 """Validate utilization values"""
64 for field_name, value in [("cpu", self.cpu), ("memory", self.memory), ("gpu", self.gpu)]:
65 if not isinstance(value, (int, float)) or not 0.0 <= value <= 100.0:
66 raise ValueError(
67 f"{field_name} must be a number between 0.0 and 100.0, got {value}"
68 )
71@dataclass
72class HealthStatus:
73 """Health status report for a cluster"""
75 cluster_id: str
76 region: str
77 timestamp: datetime
78 status: Literal["healthy", "unhealthy"]
79 resource_utilization: ResourceUtilization
80 thresholds: ResourceThresholds # Forward reference to avoid circular import
81 active_jobs: int
82 pending_pods: int = 0
83 pending_requested: RequestedResources | None = None
84 message: str | None = None
86 def __post_init__(self) -> None:
87 """Validate health status"""
88 if not self.cluster_id:
89 raise ValueError("Cluster ID cannot be empty")
91 if not self.region:
92 raise ValueError("Region cannot be empty")
94 if self.active_jobs < 0:
95 raise ValueError("Active jobs count cannot be negative")
97 if self.pending_pods < 0:
98 raise ValueError("Pending pods count cannot be negative")
100 if self.status not in ["healthy", "unhealthy"]:
101 raise ValueError("Status must be 'healthy' or 'unhealthy'")
103 def is_healthy(self) -> bool:
104 """Check if the cluster is healthy based on enabled thresholds."""
105 utilization_ok = all(
106 self.thresholds.is_disabled(name) or value <= getattr(self.thresholds, name)
107 for name, value in (
108 ("cpu_threshold", self.resource_utilization.cpu),
109 ("memory_threshold", self.resource_utilization.memory),
110 ("gpu_threshold", self.resource_utilization.gpu),
111 )
112 )
114 pending_ok = (
115 self.thresholds.is_disabled("pending_pods_threshold")
116 or self.pending_pods <= self.thresholds.pending_pods_threshold
117 )
119 pending_resources_ok = True
120 if self.pending_requested:
121 pending_resources_ok = all(
122 self.thresholds.is_disabled(name) or value <= getattr(self.thresholds, name)
123 for name, value in (
124 (
125 "pending_requested_cpu_vcpus",
126 self.pending_requested.cpu_vcpus,
127 ),
128 (
129 "pending_requested_memory_gb",
130 self.pending_requested.memory_gb,
131 ),
132 ("pending_requested_gpus", self.pending_requested.gpus),
133 )
134 )
136 return utilization_ok and pending_ok and pending_resources_ok
138 def get_threshold_violations(self) -> list[str]:
139 """Get violations for enabled thresholds only."""
140 violations = []
142 if (
143 not self.thresholds.is_disabled("cpu_threshold")
144 and self.resource_utilization.cpu > self.thresholds.cpu_threshold
145 ):
146 violations.append(
147 f"CPU: {self.resource_utilization.cpu:.1f}% > {self.thresholds.cpu_threshold}%"
148 )
150 if (
151 not self.thresholds.is_disabled("memory_threshold")
152 and self.resource_utilization.memory > self.thresholds.memory_threshold
153 ):
154 violations.append(
155 f"Memory: {self.resource_utilization.memory:.1f}% > {self.thresholds.memory_threshold}%"
156 )
158 if (
159 not self.thresholds.is_disabled("gpu_threshold")
160 and self.resource_utilization.gpu > self.thresholds.gpu_threshold
161 ):
162 violations.append(
163 f"GPU: {self.resource_utilization.gpu:.1f}% > {self.thresholds.gpu_threshold}%"
164 )
166 if (
167 not self.thresholds.is_disabled("pending_pods_threshold")
168 and self.pending_pods > self.thresholds.pending_pods_threshold
169 ):
170 violations.append(
171 f"Pending Pods: {self.pending_pods} > {self.thresholds.pending_pods_threshold}"
172 )
174 if self.pending_requested:
175 if (
176 not self.thresholds.is_disabled("pending_requested_cpu_vcpus")
177 and self.pending_requested.cpu_vcpus > self.thresholds.pending_requested_cpu_vcpus
178 ):
179 violations.append(
180 f"Pending CPU: {self.pending_requested.cpu_vcpus:.1f} vCPUs > {self.thresholds.pending_requested_cpu_vcpus} vCPUs"
181 )
182 if (
183 not self.thresholds.is_disabled("pending_requested_memory_gb")
184 and self.pending_requested.memory_gb > self.thresholds.pending_requested_memory_gb
185 ):
186 violations.append(
187 f"Pending Memory: {self.pending_requested.memory_gb:.1f} GB > {self.thresholds.pending_requested_memory_gb} GB"
188 )
189 if (
190 not self.thresholds.is_disabled("pending_requested_gpus")
191 and self.pending_requested.gpus > self.thresholds.pending_requested_gpus
192 ):
193 violations.append(
194 f"Pending GPUs: {self.pending_requested.gpus} > {self.thresholds.pending_requested_gpus}"
195 )
197 return violations