Coverage for gco/stacks/monitoring_stack.py: 97.14%
356 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-30 21:22 +0000
1"""
2Monitoring stack for GCO (Global Capacity Orchestrator on AWS) - Cross-region monitoring and observability.
4This stack creates centralized monitoring resources for all GCO deployments:
5- CloudWatch Dashboard with comprehensive widgets for all regions
6- SNS topic for alerting
7- CloudWatch Alarms for critical metrics
8- Log groups for application logs
9- Anomaly detection for traffic patterns
10- Composite alarms for better signal-to-noise
12Dashboard Sections:
13- Global Accelerator: Flow counts, processed bytes
14- API Gateway: Request counts, latency, error rates
15- Lambda Functions: Invocations, errors, duration, throttles
16- SQS Queues: Message counts, age, dead letter queue depth
17- DynamoDB Tables: Capacity, latency, throttles, errors
18- EKS Clusters: CPU/memory utilization per region
19- FSx for Lustre (when enabled): Throughput, IOPS, free storage
20- Valkey Serverless (when enabled): ECPU, hit rate, latency, bytes used
21- Aurora pgvector (when enabled): ACU utilization, connections, latency, CPU
22- ALBs: Request counts, response times, healthy hosts
23- Applications: Custom metrics from health monitor and manifest processor
25Cross-Region Metrics:
26 CloudWatch metrics are region-specific. This stack handles cross-region
27 monitoring by specifying the `region` parameter on metrics:
28 - Global Accelerator metrics: Always in us-west-2
29 - DynamoDB metrics: In the global region (where tables are deployed)
30 - Regional metrics: In each cluster's region
32Alarms:
33- High CPU/memory utilization on EKS clusters
34- Unhealthy hosts in ALB target groups
35- High response times
36- Manifest processing failures
37- Lambda errors and throttles
38- SQS message age (stuck jobs)
39- DynamoDB throttling and system errors
40- API Gateway 5XX errors
41- Secret rotation failures
42"""
44from typing import TYPE_CHECKING, Any
46from aws_cdk import (
47 CfnOutput,
48 Duration,
49 RemovalPolicy,
50 Stack,
51)
52from aws_cdk import aws_athena as athena
53from aws_cdk import aws_cloudwatch as cloudwatch
54from aws_cdk import aws_cloudwatch_actions as cw_actions
55from aws_cdk import aws_glue as glue
56from aws_cdk import aws_iam as iam
57from aws_cdk import aws_kms as kms
58from aws_cdk import aws_logs as logs
59from aws_cdk import aws_s3 as s3
60from aws_cdk import aws_sns as sns
61from constructs import Construct
63from gco.config.config_loader import ConfigLoader
64from gco.stacks.constants import (
65 COST_ATHENA_RESULTS_PREFIX,
66 COST_GLUE_ALLOCATION_TABLE,
67 COST_REPORT_SCHEDULED_PREFIX,
68 cost_athena_workgroup_name,
69 cost_glue_database_name,
70 cost_report_bucket_name,
71)
73# <pyflowchart-code-diagram> BEGIN - auto-inserted, do not edit
74# Generated at (UTC): 2026-07-18T01:03:40Z
75# Flowchart(s) generated from this file:
76# * ``GCOMonitoringStack.__init__`` -> ``diagrams/code_diagrams/gco/stacks/monitoring_stack.GCOMonitoringStack___init__.html``
77# (PNG: ``diagrams/code_diagrams/gco/stacks/monitoring_stack.GCOMonitoringStack___init__.png``)
78# Regenerate with ``python diagrams/code_diagrams/generate.py``.
79# <pyflowchart-code-diagram> END
82if TYPE_CHECKING:
83 from gco.stacks.api_gateway_global_stack import GCOApiGatewayGlobalStack
84 from gco.stacks.global_stack import GCOGlobalStack
85 from gco.stacks.regional_stack import GCORegionalStack
88class GCOMonitoringStack(Stack):
89 """
90 Cross-region monitoring and observability stack.
92 Creates a centralized CloudWatch dashboard and alarms that aggregate
93 metrics from all regional deployments.
95 Attributes:
96 alert_topic: SNS topic for alarm notifications
97 dashboard: CloudWatch dashboard with all monitoring widgets
98 """
100 def __init__(
101 self,
102 scope: Construct,
103 construct_id: str,
104 config: ConfigLoader,
105 global_stack: GCOGlobalStack,
106 regional_stacks: list[GCORegionalStack],
107 api_gateway_stack: GCOApiGatewayGlobalStack | None = None,
108 **kwargs: Any,
109 ) -> None:
110 # Enable CDK's native cross-region references. The monitoring stack
111 # lives in the monitoring region (by default us-east-2) and needs
112 # resource identifiers from the regional stacks for dashboard
113 # dimensions — specifically the auto-generated FSx file system IDs,
114 # whose values aren't known until deploy time.
115 #
116 # CDK implements this by provisioning a small Lambda-backed custom
117 # resource in each source stack that writes the referenced value to
118 # an SSM parameter in the target region, plus a reader custom
119 # resource in the target stack. Cost is negligible (the Lambdas run
120 # once per deploy) and the pattern is the documented canonical
121 # answer for ``CrossRegionReferencesNotEnabled`` errors.
122 kwargs.setdefault("cross_region_references", True)
123 super().__init__(scope, construct_id, **kwargs)
125 self.config = config
126 self.global_stack = global_stack
127 self.regional_stacks = regional_stacks
128 self.api_gateway_stack = api_gateway_stack
129 self.project_name = config.get_project_name()
130 self.regions = config.get_regions()
132 # Create SNS topic for alerts
133 self.alert_topic = self._create_alert_topic()
135 # Cost monitoring pipeline (on by default): the cost report bucket the
136 # regional cost-monitor services write Parquet allocation reports to,
137 # plus the Glue database/table and Athena workgroup that make the
138 # cross-region data queryable from the CLI.
139 if self.config.get_cost_monitoring_enabled():
140 self._create_cost_report_storage()
141 self._create_cost_analytics()
143 # Create CloudWatch dashboard
144 self.dashboard = self._create_dashboard()
146 # Create alarms
147 self._create_alarms()
149 # Create composite alarms
150 self._create_composite_alarms()
152 # Create custom metrics
153 self._create_custom_metrics()
155 # Export monitoring resources
156 self._create_outputs()
158 # Apply cdk-nag suppressions
159 self._apply_nag_suppressions()
161 def _apply_nag_suppressions(self) -> None:
162 """Apply cdk-nag suppressions for this stack."""
163 from gco.stacks.nag_suppressions import apply_all_suppressions
165 apply_all_suppressions(
166 self,
167 stack_type="monitoring",
168 regions=self.config.get_regions(),
169 global_region=self.config.get_global_region(),
170 project_name=self.project_name,
171 )
173 def _create_alert_topic(self) -> sns.Topic:
174 """Create SNS topic for monitoring alerts"""
175 topic = sns.Topic(
176 self,
177 "GCOAlertTopic",
178 display_name="GCO (Global Capacity Orchestrator on AWS) Monitoring Alerts",
179 enforce_ssl=True,
180 )
181 return topic
183 def _create_cost_report_storage(self) -> None:
184 """Create the central cost report bucket for the cost monitoring pipeline.
186 Every regional cost-monitor service writes Hive-partitioned Parquet
187 allocation reports here:
189 - ``reports/region=<region>/date=<YYYY-MM-DD>/...`` — scheduled
190 reports; the Glue table's partition projection reads this layout.
191 - ``adhoc/region=<region>/date=<YYYY-MM-DD>/...`` — user-requested
192 reports, kept out of the scheduled table so overlapping windows can
193 never double-count in Athena aggregations.
194 - ``athena-results/`` — Athena query results for the cost workgroup.
196 Three constructs mirror the regional-shared bucket pattern:
198 1. ``cost_report_kms_key`` — customer-managed KMS key with annual
199 rotation and a 7-day pending window on destroy.
200 2. ``cost_report_access_logs_bucket`` — the dedicated S3 access-logs
201 destination for the primary bucket.
202 3. ``cost_report_bucket`` — the primary bucket named
203 ``<project>-cost-reports-<account>-<monitoring-region>``. The name
204 is fully deterministic (``cost_report_bucket_name``) because the
205 regional stacks — which deploy *before* this stack — grant their
206 cost-monitor roles write access by literal ARN.
208 Lifecycle policy comes from ``cdk.json`` (``cost_monitoring.reports``):
209 report objects transition to STANDARD_IA after
210 ``transition_to_infrequent_access_days`` and expire after
211 ``retention_days``; Athena results expire after
212 ``athena.query_results_retention_days``.
213 """
214 cost_config = self.config.get_cost_monitoring_config()
215 reports_config = cost_config["reports"]
216 athena_config = cost_config["athena"]
218 # KMS key for the cost report bucket. Annual rotation, 7-day pending
219 # window, destroy-on-teardown — matching the shared-bucket posture.
220 self.cost_report_kms_key = kms.Key(
221 self,
222 "CostReportKmsKey",
223 description=(
224 "Customer-managed KMS key for the GCO cost report bucket in the monitoring stack."
225 ),
226 enable_key_rotation=True,
227 pending_window=Duration.days(7),
228 removal_policy=RemovalPolicy.DESTROY,
229 )
231 kms_actions = [
232 "kms:Encrypt",
233 "kms:Decrypt",
234 "kms:ReEncrypt*",
235 "kms:GenerateDataKey*",
236 "kms:DescribeKey",
237 ]
238 self.cost_report_kms_key.add_to_resource_policy(
239 iam.PolicyStatement(
240 sid="AllowS3ServiceEncryptDecrypt",
241 effect=iam.Effect.ALLOW,
242 principals=[iam.ServicePrincipal("s3.amazonaws.com")],
243 actions=kms_actions,
244 resources=["*"],
245 )
246 )
248 # Retention for the access-logs bucket honors the same `s3_access_logs`
249 # context field used by the central buckets (default 90 days).
250 s3_access_logs_ctx = self.node.try_get_context("s3_access_logs") or {}
251 access_logs_retention_days = int(s3_access_logs_ctx.get("retention_days", 90))
253 self.cost_report_access_logs_bucket = s3.Bucket(
254 self,
255 "CostReportAccessLogsBucket",
256 encryption=s3.BucketEncryption.KMS,
257 encryption_key=self.cost_report_kms_key,
258 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
259 enforce_ssl=True,
260 versioned=True,
261 removal_policy=RemovalPolicy.DESTROY,
262 auto_delete_objects=True,
263 lifecycle_rules=[
264 s3.LifecycleRule(
265 id="ExpireAccessLogs",
266 enabled=True,
267 expiration=Duration.days(access_logs_retention_days),
268 )
269 ],
270 )
272 self.cost_report_bucket = s3.Bucket(
273 self,
274 "CostReportBucket",
275 bucket_name=cost_report_bucket_name(self.project_name, self.account, self.region),
276 encryption=s3.BucketEncryption.KMS,
277 encryption_key=self.cost_report_kms_key,
278 bucket_key_enabled=True,
279 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
280 enforce_ssl=True,
281 versioned=True,
282 removal_policy=RemovalPolicy.DESTROY,
283 auto_delete_objects=True,
284 server_access_logs_bucket=self.cost_report_access_logs_bucket,
285 server_access_logs_prefix="cost-reports/",
286 lifecycle_rules=[
287 # Scheduled + ad-hoc reports share one policy: IA after the
288 # configured transition, expiry after the retention window.
289 s3.LifecycleRule(
290 id="CostReportRetention",
291 enabled=True,
292 prefix=f"{COST_REPORT_SCHEDULED_PREFIX}/",
293 transitions=[
294 s3.Transition(
295 storage_class=s3.StorageClass.INFREQUENT_ACCESS,
296 transition_after=Duration.days(
297 int(reports_config["transition_to_infrequent_access_days"])
298 ),
299 )
300 ],
301 expiration=Duration.days(int(reports_config["retention_days"])),
302 ),
303 s3.LifecycleRule(
304 id="AdhocReportRetention",
305 enabled=True,
306 prefix="adhoc/",
307 transitions=[
308 s3.Transition(
309 storage_class=s3.StorageClass.INFREQUENT_ACCESS,
310 transition_after=Duration.days(
311 int(reports_config["transition_to_infrequent_access_days"])
312 ),
313 )
314 ],
315 expiration=Duration.days(int(reports_config["retention_days"])),
316 ),
317 s3.LifecycleRule(
318 id="ExpireAthenaResults",
319 enabled=True,
320 prefix=f"{COST_ATHENA_RESULTS_PREFIX}/",
321 expiration=Duration.days(int(athena_config["query_results_retention_days"])),
322 ),
323 ],
324 )
326 # Explicit Deny for insecure transport with a verifiable SID,
327 # duplicating enforce_ssl=True per the central-bucket pattern.
328 self.cost_report_bucket.add_to_resource_policy(
329 iam.PolicyStatement(
330 sid="DenyInsecureTransport",
331 effect=iam.Effect.DENY,
332 principals=[iam.AnyPrincipal()],
333 actions=["s3:*"],
334 resources=[
335 self.cost_report_bucket.bucket_arn,
336 f"{self.cost_report_bucket.bucket_arn}/*",
337 ],
338 conditions={"Bool": {"aws:SecureTransport": "false"}},
339 )
340 )
342 from gco.stacks.nag_suppressions import acknowledge_nag_findings
344 cost_replication_reason = (
345 "Cost reports are derived analytics data regenerated continuously "
346 "by the per-region cost-monitor services; there is no durability "
347 "requirement that warrants cross-region replication. Access logs "
348 "do not require replication for the same reason."
349 )
350 acknowledge_nag_findings(
351 self.cost_report_bucket,
352 [
353 {
354 "id": "HIPAA.Security-S3BucketReplicationEnabled",
355 "reason": cost_replication_reason,
356 },
357 {
358 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
359 "reason": cost_replication_reason,
360 },
361 {
362 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
363 "reason": cost_replication_reason,
364 },
365 ],
366 )
368 access_logs_is_self_target_reason = (
369 "This is the server access logs destination bucket for the cost report bucket."
370 )
371 acknowledge_nag_findings(
372 self.cost_report_access_logs_bucket,
373 [
374 {
375 "id": "AwsSolutions-S1",
376 "reason": access_logs_is_self_target_reason,
377 },
378 {
379 "id": "HIPAA.Security-S3BucketLoggingEnabled",
380 "reason": access_logs_is_self_target_reason,
381 },
382 {
383 "id": "NIST.800.53.R5-S3BucketLoggingEnabled",
384 "reason": access_logs_is_self_target_reason,
385 },
386 {
387 "id": "PCI.DSS.321-S3BucketLoggingEnabled",
388 "reason": access_logs_is_self_target_reason,
389 },
390 {
391 "id": "HIPAA.Security-S3BucketReplicationEnabled",
392 "reason": cost_replication_reason,
393 },
394 {
395 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
396 "reason": cost_replication_reason,
397 },
398 {
399 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
400 "reason": cost_replication_reason,
401 },
402 ],
403 )
405 CfnOutput(
406 self,
407 "CostReportBucketName",
408 value=self.cost_report_bucket.bucket_name,
409 description="Central S3 bucket receiving per-region Parquet cost reports",
410 )
412 def _create_cost_analytics(self) -> None:
413 """Create the Glue database/table and Athena workgroup for cost queries.
415 The Glue table reads the scheduled report layout
416 (``reports/region=<region>/date=<YYYY-MM-DD>/*.parquet``) using
417 **partition projection**, so there is no crawler, no scheduled
418 ``MSCK REPAIR``, and no partition-management Lambda — new partitions
419 are queryable the moment an object lands. The ``region`` partition is
420 projected from the configured deployment regions; ``date`` is a native
421 date projection from 2026-01-01 to NOW.
423 The Athena workgroup pins query results to ``athena-results/`` in the
424 cost report bucket (KMS-encrypted, lifecycle-expired) and enforces its
425 configuration so callers cannot redirect results elsewhere.
427 The table schema is the write-side contract of
428 ``gco.services.cost_monitor`` — the columns below must stay in
429 lockstep with the Parquet fields the service emits.
430 """
431 database_name = cost_glue_database_name(self.project_name)
433 self.cost_glue_database = glue.CfnDatabase(
434 self,
435 "CostGlueDatabase",
436 catalog_id=self.account,
437 database_input=glue.CfnDatabase.DatabaseInputProperty(
438 name=database_name,
439 description=(
440 "GCO cost analytics: per-region OpenCost allocation "
441 "reports written by the cost-monitor services."
442 ),
443 ),
444 )
446 # Columns mirror gco/services/cost_monitor.py::ALLOCATION_REPORT_FIELDS.
447 columns = [
448 glue.CfnTable.ColumnProperty(name="window_start", type="timestamp"),
449 glue.CfnTable.ColumnProperty(name="window_end", type="timestamp"),
450 glue.CfnTable.ColumnProperty(name="cluster", type="string"),
451 glue.CfnTable.ColumnProperty(name="namespace", type="string"),
452 glue.CfnTable.ColumnProperty(name="cpu_core_hours", type="double"),
453 glue.CfnTable.ColumnProperty(name="cpu_cost", type="double"),
454 glue.CfnTable.ColumnProperty(name="ram_gib_hours", type="double"),
455 glue.CfnTable.ColumnProperty(name="ram_cost", type="double"),
456 glue.CfnTable.ColumnProperty(name="gpu_hours", type="double"),
457 glue.CfnTable.ColumnProperty(name="gpu_cost", type="double"),
458 glue.CfnTable.ColumnProperty(name="pv_cost", type="double"),
459 glue.CfnTable.ColumnProperty(name="network_cost", type="double"),
460 glue.CfnTable.ColumnProperty(name="load_balancer_cost", type="double"),
461 glue.CfnTable.ColumnProperty(name="shared_cost", type="double"),
462 glue.CfnTable.ColumnProperty(name="external_cost", type="double"),
463 glue.CfnTable.ColumnProperty(name="total_cost", type="double"),
464 glue.CfnTable.ColumnProperty(name="total_efficiency", type="double"),
465 ]
467 scheduled_location = (
468 f"s3://{self.cost_report_bucket.bucket_name}/{COST_REPORT_SCHEDULED_PREFIX}/"
469 )
470 self.cost_glue_table = glue.CfnTable(
471 self,
472 "CostAllocationTable",
473 catalog_id=self.account,
474 database_name=database_name,
475 table_input=glue.CfnTable.TableInputProperty(
476 name=COST_GLUE_ALLOCATION_TABLE,
477 description="Scheduled OpenCost allocation reports (Parquet)",
478 table_type="EXTERNAL_TABLE",
479 parameters={
480 "classification": "parquet",
481 "EXTERNAL": "TRUE",
482 # Partition projection: no crawler, no MSCK. The region
483 # projection is pinned to the deployment's configured
484 # regions; extend deployment_regions.regional and redeploy
485 # to pick up new regions.
486 "projection.enabled": "true",
487 "projection.region.type": "enum",
488 "projection.region.values": ",".join(self.regions),
489 "projection.date.type": "date",
490 "projection.date.range": "2026-01-01,NOW",
491 "projection.date.format": "yyyy-MM-dd",
492 "projection.date.interval": "1",
493 "projection.date.interval.unit": "DAYS",
494 "storage.location.template": (
495 f"{scheduled_location}region=${{region}}/date=${{date}}"
496 ),
497 },
498 partition_keys=[
499 glue.CfnTable.ColumnProperty(name="region", type="string"),
500 glue.CfnTable.ColumnProperty(name="date", type="string"),
501 ],
502 storage_descriptor=glue.CfnTable.StorageDescriptorProperty(
503 location=scheduled_location,
504 input_format=("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"),
505 output_format=(
506 "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"
507 ),
508 serde_info=glue.CfnTable.SerdeInfoProperty(
509 serialization_library=(
510 "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
511 ),
512 ),
513 columns=columns,
514 ),
515 ),
516 )
517 self.cost_glue_table.add_resource_dependency(self.cost_glue_database)
519 self.cost_athena_workgroup = athena.CfnWorkGroup(
520 self,
521 "CostAthenaWorkGroup",
522 name=cost_athena_workgroup_name(self.project_name),
523 description="GCO cost analytics queries over the cost report bucket",
524 recursive_delete_option=True,
525 work_group_configuration=athena.CfnWorkGroup.WorkGroupConfigurationProperty(
526 enforce_work_group_configuration=True,
527 publish_cloud_watch_metrics_enabled=True,
528 result_configuration=athena.CfnWorkGroup.ResultConfigurationProperty(
529 output_location=(
530 f"s3://{self.cost_report_bucket.bucket_name}/{COST_ATHENA_RESULTS_PREFIX}/"
531 ),
532 encryption_configuration=(
533 athena.CfnWorkGroup.EncryptionConfigurationProperty(
534 encryption_option="SSE_KMS",
535 kms_key=self.cost_report_kms_key.key_arn,
536 )
537 ),
538 ),
539 ),
540 )
542 CfnOutput(
543 self,
544 "CostAthenaWorkGroupName",
545 value=cost_athena_workgroup_name(self.project_name),
546 description="Athena workgroup for GCO cost analytics queries",
547 )
548 CfnOutput(
549 self,
550 "CostGlueDatabaseName",
551 value=cost_glue_database_name(self.project_name),
552 description="Glue database containing the cost allocation table",
553 )
555 def _create_dashboard(self) -> cloudwatch.Dashboard:
556 """Create comprehensive CloudWatch dashboard for monitoring"""
557 dashboard = cloudwatch.Dashboard(
558 self,
559 "GCODashboard",
560 period_override=cloudwatch.PeriodOverride.AUTO,
561 )
563 # Add widgets in logical order
564 dashboard.add_widgets(*self._create_global_accelerator_widgets())
565 dashboard.add_widgets(*self._create_api_gateway_widgets())
566 dashboard.add_widgets(*self._create_lambda_widgets())
567 dashboard.add_widgets(*self._create_sqs_widgets())
568 dashboard.add_widgets(*self._create_dynamodb_widgets())
569 dashboard.add_widgets(*self._create_eks_widgets())
570 dashboard.add_widgets(*self._create_gpu_widgets())
571 dashboard.add_widgets(*self._create_fsx_widgets())
572 dashboard.add_widgets(*self._create_valkey_widgets())
573 dashboard.add_widgets(*self._create_aurora_pgvector_widgets())
574 dashboard.add_widgets(*self._create_alb_widgets())
575 dashboard.add_widgets(*self._create_application_widgets())
577 return dashboard
579 def _create_global_accelerator_widgets(self) -> list[cloudwatch.IWidget]:
580 """Create Global Accelerator monitoring widgets.
582 Note: Global Accelerator metrics are only available in us-west-2,
583 regardless of where the accelerator endpoints are located.
584 CloudWatch uses the Accelerator ID (UUID), not the name.
585 """
586 widgets: list[cloudwatch.IWidget] = []
587 if self.global_stack.accelerator_id is None: 587 ↛ 588line 587 didn't jump to line 588 because the condition on line 587 was never true
588 return widgets
590 # Get the accelerator ID from the global stack (CloudWatch uses ID, not name)
591 accelerator_id = self.global_stack.accelerator_id
593 # Global Accelerator metrics are always in us-west-2
594 ga_metrics_region = "us-west-2"
596 # Section header
597 widgets.append(
598 cloudwatch.TextWidget(
599 markdown="# Global Accelerator\nTraffic distribution and connectivity metrics",
600 width=24,
601 height=1,
602 )
603 )
605 # Flow count with anomaly detection
606 flow_count_widget = cloudwatch.GraphWidget(
607 title="Global Accelerator - New Flows",
608 left=[
609 cloudwatch.Metric(
610 namespace="AWS/GlobalAccelerator",
611 metric_name="NewFlowCount",
612 dimensions_map={"Accelerator": accelerator_id},
613 statistic="Sum",
614 period=Duration.minutes(5),
615 region=ga_metrics_region,
616 )
617 ],
618 width=12,
619 height=6,
620 region=ga_metrics_region,
621 )
622 widgets.append(flow_count_widget)
624 # Processed bytes
625 bytes_widget = cloudwatch.GraphWidget(
626 title="Global Accelerator - Processed Bytes",
627 left=[
628 cloudwatch.Metric(
629 namespace="AWS/GlobalAccelerator",
630 metric_name="ProcessedBytesIn",
631 dimensions_map={"Accelerator": accelerator_id},
632 statistic="Sum",
633 period=Duration.minutes(5),
634 region=ga_metrics_region,
635 ),
636 cloudwatch.Metric(
637 namespace="AWS/GlobalAccelerator",
638 metric_name="ProcessedBytesOut",
639 dimensions_map={"Accelerator": accelerator_id},
640 statistic="Sum",
641 period=Duration.minutes(5),
642 region=ga_metrics_region,
643 ),
644 ],
645 width=12,
646 height=6,
647 region=ga_metrics_region,
648 )
649 widgets.append(bytes_widget)
651 return widgets
653 def _create_api_gateway_widgets(self) -> list[cloudwatch.IWidget]:
654 """Create API Gateway monitoring widgets"""
655 widgets: list[cloudwatch.IWidget] = []
657 # Get the actual API name from the api_gateway_stack
658 api_name = (
659 self.api_gateway_stack.api.rest_api_name
660 if self.api_gateway_stack
661 else f"{self.project_name}-global-api"
662 )
664 # API Gateway metrics are in the region where the API is deployed
665 api_gw_region = self.config.get_api_gateway_region()
667 # Section header
668 widgets.append(
669 cloudwatch.TextWidget(
670 markdown="# API Gateway\nRequest metrics, latency, and error rates",
671 width=24,
672 height=1,
673 )
674 )
676 # Request count and latency
677 request_widget = cloudwatch.GraphWidget(
678 title="API Gateway - Requests & Latency",
679 left=[
680 cloudwatch.Metric(
681 namespace="AWS/ApiGateway",
682 metric_name="Count",
683 dimensions_map={"ApiName": api_name},
684 statistic="Sum",
685 period=Duration.minutes(5),
686 region=api_gw_region,
687 )
688 ],
689 right=[
690 cloudwatch.Metric(
691 namespace="AWS/ApiGateway",
692 metric_name="Latency",
693 dimensions_map={"ApiName": api_name},
694 statistic="Average",
695 period=Duration.minutes(5),
696 region=api_gw_region,
697 ),
698 cloudwatch.Metric(
699 namespace="AWS/ApiGateway",
700 metric_name="Latency",
701 dimensions_map={"ApiName": api_name},
702 statistic="p99",
703 period=Duration.minutes(5),
704 region=api_gw_region,
705 ),
706 ],
707 width=12,
708 height=6,
709 region=api_gw_region,
710 )
711 widgets.append(request_widget)
713 # Error rates (4XX and 5XX)
714 error_widget = cloudwatch.GraphWidget(
715 title="API Gateway - Error Rates",
716 left=[
717 cloudwatch.Metric(
718 namespace="AWS/ApiGateway",
719 metric_name="4XXError",
720 dimensions_map={"ApiName": api_name},
721 statistic="Sum",
722 period=Duration.minutes(5),
723 color="#ff7f0e",
724 region=api_gw_region,
725 ),
726 cloudwatch.Metric(
727 namespace="AWS/ApiGateway",
728 metric_name="5XXError",
729 dimensions_map={"ApiName": api_name},
730 statistic="Sum",
731 period=Duration.minutes(5),
732 color="#d62728",
733 region=api_gw_region,
734 ),
735 ],
736 width=12,
737 height=6,
738 region=api_gw_region,
739 )
740 widgets.append(error_widget)
742 return widgets
744 def _create_lambda_widgets(self) -> list[cloudwatch.IWidget]:
745 """Create Lambda function monitoring widgets"""
746 widgets: list[cloudwatch.IWidget] = []
748 # Section header
749 widgets.append(
750 cloudwatch.TextWidget(
751 markdown="# Lambda Functions\nProxy, rotation, and regional Lambda metrics",
752 width=24,
753 height=1,
754 )
755 )
757 # Get API Gateway region for global Lambda functions
758 api_gw_region = self.config.get_api_gateway_region()
760 # Build Lambda function list: (function_name, label, region)
761 lambda_functions: list[tuple[str, str, str]] = []
763 # Add API Gateway Lambda functions if available
764 if self.api_gateway_stack: 764 ↛ 782line 764 didn't jump to line 782 because the condition on line 764 was always true
765 if self.api_gateway_stack.proxy_lambda is not None: 765 ↛ 773line 765 didn't jump to line 773 because the condition on line 765 was always true
766 lambda_functions.append(
767 (
768 self.api_gateway_stack.proxy_lambda.function_name,
769 "API Gateway Proxy",
770 api_gw_region,
771 )
772 )
773 lambda_functions.append(
774 (
775 self.api_gateway_stack.rotation_lambda.function_name,
776 "Secret Rotation",
777 api_gw_region,
778 )
779 )
781 # Add regional Lambda functions from each regional stack
782 for regional_stack in self.regional_stacks:
783 region = regional_stack.deployment_region
784 lambda_functions.extend(
785 [
786 (
787 regional_stack.kubectl_lambda_function_name,
788 f"Kubectl Applier ({region})",
789 region,
790 ),
791 (
792 regional_stack.helm_installer_lambda_function_name,
793 f"Helm Installer ({region})",
794 region,
795 ),
796 ]
797 )
799 # Invocations widget
800 invocations_widget = cloudwatch.GraphWidget(
801 title="Lambda - Invocations",
802 left=[
803 cloudwatch.Metric(
804 namespace="AWS/Lambda",
805 metric_name="Invocations",
806 dimensions_map={"FunctionName": func_name},
807 statistic="Sum",
808 period=Duration.minutes(5),
809 label=label,
810 region=region,
811 )
812 for func_name, label, region in lambda_functions[:5]
813 ],
814 width=12,
815 height=6,
816 )
817 widgets.append(invocations_widget)
819 errors_widget = cloudwatch.GraphWidget(
820 title="Lambda - Errors",
821 left=[
822 cloudwatch.Metric(
823 namespace="AWS/Lambda",
824 metric_name="Errors",
825 dimensions_map={"FunctionName": func_name},
826 statistic="Sum",
827 period=Duration.minutes(5),
828 label=label,
829 color="#d62728",
830 region=region,
831 )
832 for func_name, label, region in lambda_functions[:5]
833 ],
834 width=12,
835 height=6,
836 )
837 widgets.append(errors_widget)
839 # Duration widget
840 duration_widget = cloudwatch.GraphWidget(
841 title="Lambda - Duration (ms)",
842 left=[
843 cloudwatch.Metric(
844 namespace="AWS/Lambda",
845 metric_name="Duration",
846 dimensions_map={"FunctionName": func_name},
847 statistic="Average",
848 period=Duration.minutes(5),
849 label=label,
850 region=region,
851 )
852 for func_name, label, region in lambda_functions[:5]
853 ],
854 width=12,
855 height=6,
856 )
857 widgets.append(duration_widget)
859 # Throttles widget
860 throttles_widget = cloudwatch.GraphWidget(
861 title="Lambda - Throttles & Concurrent Executions",
862 left=[
863 cloudwatch.Metric(
864 namespace="AWS/Lambda",
865 metric_name="Throttles",
866 dimensions_map={"FunctionName": func_name},
867 statistic="Sum",
868 period=Duration.minutes(5),
869 label=f"{label} Throttles",
870 region=region,
871 )
872 for func_name, label, region in lambda_functions[:3]
873 ],
874 right=[
875 cloudwatch.Metric(
876 namespace="AWS/Lambda",
877 metric_name="ConcurrentExecutions",
878 dimensions_map={"FunctionName": func_name},
879 statistic="Maximum",
880 period=Duration.minutes(5),
881 label=f"{label} Concurrent",
882 region=region,
883 )
884 for func_name, label, region in lambda_functions[:3]
885 ],
886 width=12,
887 height=6,
888 )
889 widgets.append(throttles_widget)
891 return widgets
893 def _create_sqs_widgets(self) -> list[cloudwatch.IWidget]:
894 """Create SQS queue monitoring widgets"""
895 widgets: list[cloudwatch.IWidget] = []
897 # Section header
898 widgets.append(
899 cloudwatch.TextWidget(
900 markdown="# SQS Queues\nJob submission queue metrics and dead letter queue",
901 width=24,
902 height=1,
903 )
904 )
906 # Build queue info from regional stacks: (queue_name, dlq_name, region)
907 queue_info = [
908 (
909 regional_stack.job_queue.queue_name,
910 regional_stack.job_dlq.queue_name,
911 regional_stack.deployment_region,
912 )
913 for regional_stack in self.regional_stacks
914 ]
916 # Messages visible and in-flight per region
917 messages_widget = cloudwatch.GraphWidget(
918 title="SQS - Messages (Visible & In-Flight)",
919 left=[
920 cloudwatch.Metric(
921 namespace="AWS/SQS",
922 metric_name="ApproximateNumberOfMessagesVisible",
923 dimensions_map={"QueueName": queue_name},
924 statistic="Average",
925 period=Duration.minutes(1),
926 label=f"{region} Visible",
927 region=region,
928 )
929 for queue_name, _, region in queue_info
930 ],
931 right=[
932 cloudwatch.Metric(
933 namespace="AWS/SQS",
934 metric_name="ApproximateNumberOfMessagesNotVisible",
935 dimensions_map={"QueueName": queue_name},
936 statistic="Average",
937 period=Duration.minutes(1),
938 label=f"{region} In-Flight",
939 region=region,
940 )
941 for queue_name, _, region in queue_info
942 ],
943 width=12,
944 height=6,
945 )
946 widgets.append(messages_widget)
948 # Age of oldest message (critical for detecting stuck jobs)
949 age_widget = cloudwatch.GraphWidget(
950 title="SQS - Age of Oldest Message (seconds)",
951 left=[
952 cloudwatch.Metric(
953 namespace="AWS/SQS",
954 metric_name="ApproximateAgeOfOldestMessage",
955 dimensions_map={"QueueName": queue_name},
956 statistic="Maximum",
957 period=Duration.minutes(1),
958 label=region,
959 region=region,
960 )
961 for queue_name, _, region in queue_info
962 ],
963 width=12,
964 height=6,
965 )
966 widgets.append(age_widget)
968 # Dead letter queue depth
969 dlq_widget = cloudwatch.GraphWidget(
970 title="SQS - Dead Letter Queue Depth",
971 left=[
972 cloudwatch.Metric(
973 namespace="AWS/SQS",
974 metric_name="ApproximateNumberOfMessagesVisible",
975 dimensions_map={"QueueName": dlq_name},
976 statistic="Average",
977 period=Duration.minutes(1),
978 label=f"{region} DLQ",
979 color="#d62728",
980 region=region,
981 )
982 for _, dlq_name, region in queue_info
983 ],
984 width=12,
985 height=6,
986 )
987 widgets.append(dlq_widget)
989 # Messages sent/received/deleted
990 throughput_widget = cloudwatch.GraphWidget(
991 title="SQS - Throughput",
992 left=[
993 cloudwatch.Metric(
994 namespace="AWS/SQS",
995 metric_name="NumberOfMessagesSent",
996 dimensions_map={"QueueName": queue_name},
997 statistic="Sum",
998 period=Duration.minutes(5),
999 label=f"{region} Sent",
1000 region=region,
1001 )
1002 for queue_name, _, region in queue_info
1003 ],
1004 right=[
1005 cloudwatch.Metric(
1006 namespace="AWS/SQS",
1007 metric_name="NumberOfMessagesDeleted",
1008 dimensions_map={"QueueName": queue_name},
1009 statistic="Sum",
1010 period=Duration.minutes(5),
1011 label=f"{region} Processed",
1012 region=region,
1013 )
1014 for queue_name, _, region in queue_info
1015 ],
1016 width=12,
1017 height=6,
1018 )
1019 widgets.append(throughput_widget)
1021 return widgets
1023 def _create_dynamodb_widgets(self) -> list[cloudwatch.IWidget]:
1024 """Create DynamoDB monitoring widgets for job queue, templates, and webhooks tables."""
1025 widgets: list[cloudwatch.IWidget] = []
1027 # Get table names from global stack
1028 templates_table = self.global_stack.templates_table.table_name
1029 webhooks_table = self.global_stack.webhooks_table.table_name
1030 jobs_table = self.global_stack.jobs_table.table_name
1032 # DynamoDB tables are in the global region
1033 global_region = self.config.get_global_region()
1035 # Section header
1036 widgets.append(
1037 cloudwatch.TextWidget(
1038 markdown="# DynamoDB Tables\nJob queue, templates, and webhooks storage metrics",
1039 width=24,
1040 height=1,
1041 )
1042 )
1044 # Read/Write capacity consumed
1045 capacity_widget = cloudwatch.GraphWidget(
1046 title="DynamoDB - Consumed Capacity",
1047 left=[
1048 cloudwatch.Metric(
1049 namespace="AWS/DynamoDB",
1050 metric_name="ConsumedReadCapacityUnits",
1051 dimensions_map={"TableName": jobs_table},
1052 statistic="Sum",
1053 period=Duration.minutes(5),
1054 label="Jobs Read",
1055 region=global_region,
1056 ),
1057 cloudwatch.Metric(
1058 namespace="AWS/DynamoDB",
1059 metric_name="ConsumedReadCapacityUnits",
1060 dimensions_map={"TableName": templates_table},
1061 statistic="Sum",
1062 period=Duration.minutes(5),
1063 label="Templates Read",
1064 region=global_region,
1065 ),
1066 cloudwatch.Metric(
1067 namespace="AWS/DynamoDB",
1068 metric_name="ConsumedReadCapacityUnits",
1069 dimensions_map={"TableName": webhooks_table},
1070 statistic="Sum",
1071 period=Duration.minutes(5),
1072 label="Webhooks Read",
1073 region=global_region,
1074 ),
1075 ],
1076 right=[
1077 cloudwatch.Metric(
1078 namespace="AWS/DynamoDB",
1079 metric_name="ConsumedWriteCapacityUnits",
1080 dimensions_map={"TableName": jobs_table},
1081 statistic="Sum",
1082 period=Duration.minutes(5),
1083 label="Jobs Write",
1084 region=global_region,
1085 ),
1086 cloudwatch.Metric(
1087 namespace="AWS/DynamoDB",
1088 metric_name="ConsumedWriteCapacityUnits",
1089 dimensions_map={"TableName": templates_table},
1090 statistic="Sum",
1091 period=Duration.minutes(5),
1092 label="Templates Write",
1093 region=global_region,
1094 ),
1095 ],
1096 width=12,
1097 height=6,
1098 region=global_region,
1099 )
1100 widgets.append(capacity_widget)
1102 # Latency metrics
1103 latency_widget = cloudwatch.GraphWidget(
1104 title="DynamoDB - Latency (ms)",
1105 left=[
1106 cloudwatch.Metric(
1107 namespace="AWS/DynamoDB",
1108 metric_name="SuccessfulRequestLatency",
1109 dimensions_map={"TableName": jobs_table, "Operation": "GetItem"},
1110 statistic="Average",
1111 period=Duration.minutes(5),
1112 label="Jobs GetItem",
1113 region=global_region,
1114 ),
1115 cloudwatch.Metric(
1116 namespace="AWS/DynamoDB",
1117 metric_name="SuccessfulRequestLatency",
1118 dimensions_map={"TableName": jobs_table, "Operation": "PutItem"},
1119 statistic="Average",
1120 period=Duration.minutes(5),
1121 label="Jobs PutItem",
1122 region=global_region,
1123 ),
1124 cloudwatch.Metric(
1125 namespace="AWS/DynamoDB",
1126 metric_name="SuccessfulRequestLatency",
1127 dimensions_map={"TableName": jobs_table, "Operation": "Query"},
1128 statistic="Average",
1129 period=Duration.minutes(5),
1130 label="Jobs Query",
1131 region=global_region,
1132 ),
1133 ],
1134 width=12,
1135 height=6,
1136 region=global_region,
1137 )
1138 widgets.append(latency_widget)
1140 # Throttled requests
1141 throttle_widget = cloudwatch.GraphWidget(
1142 title="DynamoDB - Throttled Requests",
1143 left=[
1144 cloudwatch.Metric(
1145 namespace="AWS/DynamoDB",
1146 metric_name="ThrottledRequests",
1147 dimensions_map={"TableName": jobs_table},
1148 statistic="Sum",
1149 period=Duration.minutes(5),
1150 label="Jobs",
1151 color="#d62728",
1152 region=global_region,
1153 ),
1154 cloudwatch.Metric(
1155 namespace="AWS/DynamoDB",
1156 metric_name="ThrottledRequests",
1157 dimensions_map={"TableName": templates_table},
1158 statistic="Sum",
1159 period=Duration.minutes(5),
1160 label="Templates",
1161 color="#ff7f0e",
1162 region=global_region,
1163 ),
1164 cloudwatch.Metric(
1165 namespace="AWS/DynamoDB",
1166 metric_name="ThrottledRequests",
1167 dimensions_map={"TableName": webhooks_table},
1168 statistic="Sum",
1169 period=Duration.minutes(5),
1170 label="Webhooks",
1171 color="#9467bd",
1172 region=global_region,
1173 ),
1174 ],
1175 width=12,
1176 height=6,
1177 region=global_region,
1178 )
1179 widgets.append(throttle_widget)
1181 # System errors
1182 errors_widget = cloudwatch.GraphWidget(
1183 title="DynamoDB - System Errors",
1184 left=[
1185 cloudwatch.Metric(
1186 namespace="AWS/DynamoDB",
1187 metric_name="SystemErrors",
1188 dimensions_map={"TableName": jobs_table},
1189 statistic="Sum",
1190 period=Duration.minutes(5),
1191 label="Jobs",
1192 color="#d62728",
1193 region=global_region,
1194 ),
1195 cloudwatch.Metric(
1196 namespace="AWS/DynamoDB",
1197 metric_name="SystemErrors",
1198 dimensions_map={"TableName": templates_table},
1199 statistic="Sum",
1200 period=Duration.minutes(5),
1201 label="Templates",
1202 color="#ff7f0e",
1203 region=global_region,
1204 ),
1205 ],
1206 width=12,
1207 height=6,
1208 region=global_region,
1209 )
1210 widgets.append(errors_widget)
1212 return widgets
1214 def _create_eks_widgets(self) -> list[cloudwatch.IWidget]:
1215 """Create EKS cluster monitoring widgets"""
1216 widgets: list[cloudwatch.IWidget] = []
1218 # Section header
1219 widgets.append(
1220 cloudwatch.TextWidget(
1221 markdown="# EKS Clusters\nCluster resource utilization and node metrics",
1222 width=24,
1223 height=1,
1224 )
1225 )
1227 # Build cluster info from regional stacks: (cluster_name, region)
1228 cluster_info = [
1229 (regional_stack.cluster.cluster_name, regional_stack.deployment_region)
1230 for regional_stack in self.regional_stacks
1231 ]
1233 # EKS cluster status
1234 cluster_status_widget = cloudwatch.SingleValueWidget(
1235 title="EKS Clusters - Failed Requests",
1236 metrics=[
1237 cloudwatch.Metric(
1238 namespace="AWS/EKS",
1239 metric_name="cluster_failed_request_count",
1240 dimensions_map={"cluster_name": cluster_name},
1241 statistic="Sum",
1242 period=Duration.minutes(5),
1243 region=region,
1244 )
1245 for cluster_name, region in cluster_info
1246 ],
1247 width=12,
1248 height=6,
1249 )
1250 widgets.append(cluster_status_widget)
1252 # Container Insights - Node CPU utilization (aggregated across all nodes)
1253 # Note: region parameter enables cross-region metrics in dashboard
1254 cpu_widget = cloudwatch.GraphWidget(
1255 title="EKS Clusters - Node CPU Utilization (%)",
1256 left=[
1257 cloudwatch.Metric(
1258 namespace="ContainerInsights",
1259 metric_name="node_cpu_utilization",
1260 dimensions_map={"ClusterName": cluster_name},
1261 statistic="Average",
1262 period=Duration.minutes(5),
1263 label=region,
1264 region=region,
1265 )
1266 for cluster_name, region in cluster_info
1267 ],
1268 width=12,
1269 height=6,
1270 )
1271 widgets.append(cpu_widget)
1273 # Container Insights - Node Memory utilization (aggregated across all nodes)
1274 memory_widget = cloudwatch.GraphWidget(
1275 title="EKS Clusters - Node Memory Utilization (%)",
1276 left=[
1277 cloudwatch.Metric(
1278 namespace="ContainerInsights",
1279 metric_name="node_memory_utilization",
1280 dimensions_map={"ClusterName": cluster_name},
1281 statistic="Average",
1282 period=Duration.minutes(5),
1283 label=region,
1284 region=region,
1285 )
1286 for cluster_name, region in cluster_info
1287 ],
1288 width=12,
1289 height=6,
1290 )
1291 widgets.append(memory_widget)
1293 # Node status - running pods capacity
1294 node_widget = cloudwatch.GraphWidget(
1295 title="EKS Clusters - Node Pod Capacity",
1296 left=[
1297 cloudwatch.Metric(
1298 namespace="ContainerInsights",
1299 metric_name="node_status_capacity_pods",
1300 dimensions_map={"ClusterName": cluster_name},
1301 statistic="Sum",
1302 period=Duration.minutes(5),
1303 label=f"{region} Capacity",
1304 region=region,
1305 )
1306 for cluster_name, region in cluster_info
1307 ],
1308 right=[
1309 cloudwatch.Metric(
1310 namespace="ContainerInsights",
1311 metric_name="node_number_of_running_pods",
1312 dimensions_map={"ClusterName": cluster_name},
1313 statistic="Sum",
1314 period=Duration.minutes(5),
1315 label=f"{region} Running",
1316 region=region,
1317 )
1318 for cluster_name, region in cluster_info
1319 ],
1320 width=12,
1321 height=6,
1322 )
1323 widgets.append(node_widget)
1325 return widgets
1327 def _create_gpu_widgets(self) -> list[cloudwatch.IWidget]:
1328 """Create GPU monitoring widgets using DCGM Exporter metrics via ContainerInsights."""
1329 widgets: list[cloudwatch.IWidget] = []
1331 widgets.append(
1332 cloudwatch.TextWidget(
1333 markdown="# GPU Metrics\nGPU utilization, memory, and temperature from DCGM Exporter",
1334 width=24,
1335 height=1,
1336 )
1337 )
1339 cluster_info = [
1340 (regional_stack.cluster.cluster_name, regional_stack.deployment_region)
1341 for regional_stack in self.regional_stacks
1342 ]
1344 # GPU utilization percentage
1345 gpu_util_widget = cloudwatch.GraphWidget(
1346 title="GPU Utilization (%)",
1347 left=[
1348 cloudwatch.Metric(
1349 namespace="ContainerInsights",
1350 metric_name="node_gpu_utilization",
1351 dimensions_map={"ClusterName": cluster_name},
1352 statistic="Average",
1353 period=Duration.minutes(5),
1354 label=region,
1355 region=region,
1356 )
1357 for cluster_name, region in cluster_info
1358 ],
1359 width=12,
1360 height=6,
1361 )
1362 widgets.append(gpu_util_widget)
1364 # GPU memory utilization
1365 gpu_mem_widget = cloudwatch.GraphWidget(
1366 title="GPU Memory Utilization (%)",
1367 left=[
1368 cloudwatch.Metric(
1369 namespace="ContainerInsights",
1370 metric_name="node_gpu_memory_utilization",
1371 dimensions_map={"ClusterName": cluster_name},
1372 statistic="Average",
1373 period=Duration.minutes(5),
1374 label=region,
1375 region=region,
1376 )
1377 for cluster_name, region in cluster_info
1378 ],
1379 width=12,
1380 height=6,
1381 )
1382 widgets.append(gpu_mem_widget)
1384 # GPU temperature
1385 gpu_temp_widget = cloudwatch.GraphWidget(
1386 title="GPU Temperature (°C)",
1387 left=[
1388 cloudwatch.Metric(
1389 namespace="ContainerInsights",
1390 metric_name="node_gpu_temperature",
1391 dimensions_map={"ClusterName": cluster_name},
1392 statistic="Maximum",
1393 period=Duration.minutes(5),
1394 label=region,
1395 region=region,
1396 )
1397 for cluster_name, region in cluster_info
1398 ],
1399 width=12,
1400 height=6,
1401 )
1402 widgets.append(gpu_temp_widget)
1404 # GPU count (active GPUs)
1405 gpu_count_widget = cloudwatch.GraphWidget(
1406 title="Active GPU Count",
1407 left=[
1408 cloudwatch.Metric(
1409 namespace="ContainerInsights",
1410 metric_name="node_gpu_limit",
1411 dimensions_map={"ClusterName": cluster_name},
1412 statistic="Sum",
1413 period=Duration.minutes(5),
1414 label=region,
1415 region=region,
1416 )
1417 for cluster_name, region in cluster_info
1418 ],
1419 width=12,
1420 height=6,
1421 )
1422 widgets.append(gpu_count_widget)
1424 return widgets
1426 def _create_fsx_widgets(self) -> list[cloudwatch.IWidget]:
1427 """Create FSx for Lustre monitoring widgets.
1429 Only emits widgets for regions where the FSx file system is actually
1430 provisioned (``regional_stack.fsx_file_system`` is non-None). The
1431 dimension ``FileSystemId`` is the CDK-generated CloudFormation ref
1432 from each regional stack — CDK's ``cross_region_references=True``
1433 (enabled on this stack's constructor) plumbs the value across
1434 regions via SSM + custom resources.
1436 Returns an empty list if no region has FSx enabled — the dashboard
1437 skips the section entirely.
1438 """
1439 # Collect (file_system_id, region) tuples for regions that have FSx on.
1440 # fsx_file_system is either a CfnFileSystem or None; the local
1441 # assignment + is-not-None check lets mypy narrow the type so
1442 # ``.ref`` access typechecks cleanly (a list comprehension with
1443 # the guard in the ``if`` clause does not narrow the value clause).
1444 fsx_info: list[tuple[str, str]] = []
1445 for regional_stack in self.regional_stacks:
1446 fsx = getattr(regional_stack, "fsx_file_system", None)
1447 if fsx is None:
1448 continue
1449 fsx_info.append((fsx.ref, regional_stack.deployment_region))
1450 if not fsx_info:
1451 return []
1453 widgets: list[cloudwatch.IWidget] = []
1455 # Section header
1456 widgets.append(
1457 cloudwatch.TextWidget(
1458 markdown=(
1459 "# FSx for Lustre\n"
1460 "Parallel file system throughput, IOPS, and free storage "
1461 "capacity. Each line below is scoped to the exact GCO "
1462 "file system in its region — so unrelated FSx file "
1463 "systems in the same account do not appear on the "
1464 "dashboard."
1465 ),
1466 width=24,
1467 height=1,
1468 )
1469 )
1471 # Throughput: bytes read vs written
1472 throughput_widget = cloudwatch.GraphWidget(
1473 title="FSx - Throughput (Bytes/sec)",
1474 left=[
1475 cloudwatch.Metric(
1476 namespace="AWS/FSx",
1477 metric_name="DataReadBytes",
1478 dimensions_map={"FileSystemId": fs_id},
1479 statistic="Sum",
1480 period=Duration.minutes(1),
1481 label=f"{region} Read",
1482 region=region,
1483 )
1484 for fs_id, region in fsx_info
1485 ],
1486 right=[
1487 cloudwatch.Metric(
1488 namespace="AWS/FSx",
1489 metric_name="DataWriteBytes",
1490 dimensions_map={"FileSystemId": fs_id},
1491 statistic="Sum",
1492 period=Duration.minutes(1),
1493 label=f"{region} Write",
1494 region=region,
1495 )
1496 for fs_id, region in fsx_info
1497 ],
1498 width=12,
1499 height=6,
1500 )
1501 widgets.append(throughput_widget)
1503 # IOPS: read vs write operations
1504 iops_widget = cloudwatch.GraphWidget(
1505 title="FSx - IOPS",
1506 left=[
1507 cloudwatch.Metric(
1508 namespace="AWS/FSx",
1509 metric_name="DataReadOperations",
1510 dimensions_map={"FileSystemId": fs_id},
1511 statistic="Sum",
1512 period=Duration.minutes(1),
1513 label=f"{region} Read",
1514 region=region,
1515 )
1516 for fs_id, region in fsx_info
1517 ],
1518 right=[
1519 cloudwatch.Metric(
1520 namespace="AWS/FSx",
1521 metric_name="DataWriteOperations",
1522 dimensions_map={"FileSystemId": fs_id},
1523 statistic="Sum",
1524 period=Duration.minutes(1),
1525 label=f"{region} Write",
1526 region=region,
1527 )
1528 for fs_id, region in fsx_info
1529 ],
1530 width=12,
1531 height=6,
1532 )
1533 widgets.append(iops_widget)
1535 # Free storage capacity — the classic "running out of space" signal.
1536 # FreeDataStorageCapacity is emitted in bytes.
1537 free_storage_widget = cloudwatch.GraphWidget(
1538 title="FSx - Free Storage Capacity (Bytes)",
1539 left=[
1540 cloudwatch.Metric(
1541 namespace="AWS/FSx",
1542 metric_name="FreeDataStorageCapacity",
1543 dimensions_map={"FileSystemId": fs_id},
1544 statistic="Minimum",
1545 period=Duration.minutes(5),
1546 label=region,
1547 region=region,
1548 )
1549 for fs_id, region in fsx_info
1550 ],
1551 width=24,
1552 height=6,
1553 )
1554 widgets.append(free_storage_widget)
1556 return widgets
1558 def _create_valkey_widgets(self) -> list[cloudwatch.IWidget]:
1559 """Create Valkey (ElastiCache Serverless) monitoring widgets.
1561 Uses explicit ``clusterId`` dimension values (camelCase — the
1562 ElastiCache Serverless variant; distinct from the node-based
1563 ``CacheClusterId``). The regional stack names its cache
1564 deterministically as ``gco-{deployment_region}``, so we reproduce
1565 that name here and pin each widget to the exact cache in its
1566 region. No SEARCH expression, so the dashboard ignores every
1567 unrelated ElastiCache cluster in the account.
1568 """
1569 valkey_enabled = self.config.get_valkey_config().get("enabled", False)
1570 if not valkey_enabled or not self.regions:
1571 return []
1573 widgets: list[cloudwatch.IWidget] = []
1575 widgets.append(
1576 cloudwatch.TextWidget(
1577 markdown=(
1578 "# Valkey Serverless Cache\n"
1579 "ECPU consumption, storage, hit rate, and request "
1580 "latency — scoped to each region's ``gco-{region}`` "
1581 "cache exactly (no SEARCH)."
1582 ),
1583 width=24,
1584 height=1,
1585 )
1586 )
1588 # Build (cache_name, region) pairs. cache_name is the literal
1589 # ``serverless_cache_name`` the regional stack passes to the
1590 # CfnServerlessCache.
1591 cache_info = [(f"{self.project_name}-{region}", region) for region in self.regions]
1593 # ECPU consumption and cache size per region
1594 for cache_name, region in cache_info:
1595 widgets.append(
1596 cloudwatch.GraphWidget(
1597 title=f"Valkey - ECPU & Cache Size ({region})",
1598 left=[
1599 cloudwatch.Metric(
1600 namespace="AWS/ElastiCache",
1601 metric_name="ElastiCacheProcessingUnits",
1602 dimensions_map={"clusterId": cache_name},
1603 statistic="Sum",
1604 period=Duration.minutes(1),
1605 label="ECPUs",
1606 region=region,
1607 ),
1608 ],
1609 right=[
1610 cloudwatch.Metric(
1611 namespace="AWS/ElastiCache",
1612 metric_name="BytesUsedForCache",
1613 dimensions_map={"clusterId": cache_name},
1614 statistic="Average",
1615 period=Duration.minutes(5),
1616 label="Bytes",
1617 region=region,
1618 ),
1619 ],
1620 width=12,
1621 height=6,
1622 region=region,
1623 )
1624 )
1626 # Hit rate and p99 read/write latency per region
1627 for cache_name, region in cache_info:
1628 widgets.append(
1629 cloudwatch.GraphWidget(
1630 title=f"Valkey - Hit Rate & Latency ({region})",
1631 left=[
1632 cloudwatch.Metric(
1633 namespace="AWS/ElastiCache",
1634 metric_name="CacheHitRate",
1635 dimensions_map={"clusterId": cache_name},
1636 statistic="Average",
1637 period=Duration.minutes(5),
1638 label="Hit Rate %",
1639 region=region,
1640 ),
1641 ],
1642 right=[
1643 cloudwatch.Metric(
1644 namespace="AWS/ElastiCache",
1645 metric_name="SuccessfulReadRequestLatency",
1646 dimensions_map={"clusterId": cache_name},
1647 statistic="p99",
1648 period=Duration.minutes(1),
1649 label="Read p99 µs",
1650 region=region,
1651 ),
1652 cloudwatch.Metric(
1653 namespace="AWS/ElastiCache",
1654 metric_name="SuccessfulWriteRequestLatency",
1655 dimensions_map={"clusterId": cache_name},
1656 statistic="p99",
1657 period=Duration.minutes(1),
1658 label="Write p99 µs",
1659 region=region,
1660 ),
1661 ],
1662 width=12,
1663 height=6,
1664 region=region,
1665 )
1666 )
1668 return widgets
1670 def _create_aurora_pgvector_widgets(self) -> list[cloudwatch.IWidget]:
1671 """Create Aurora Serverless v2 (pgvector) monitoring widgets.
1673 Pins each widget to the exact Aurora cluster provisioned by the
1674 regional stack via ``regional_stack.aurora_cluster.cluster_identifier``.
1675 CDK-generated cluster IDs are CloudFormation tokens; the
1676 ``cross_region_references=True`` flag on this stack handles
1677 plumbing them from each regional stack into the monitoring stack
1678 (us-east-2 by default) through SSM + custom resources.
1680 Returns an empty list when every region has Aurora pgvector
1681 disabled so the dashboard skips the section entirely.
1682 """
1683 # (cluster_identifier, region) pairs for regions with Aurora on.
1684 # Use a guarded loop (not a comprehension) so mypy can narrow the
1685 # Optional[DatabaseCluster] to a real cluster before dereferencing.
1686 aurora_info: list[tuple[str, str]] = []
1687 for regional_stack in self.regional_stacks:
1688 aurora = getattr(regional_stack, "aurora_cluster", None)
1689 if aurora is None:
1690 continue
1691 aurora_info.append((aurora.cluster_identifier, regional_stack.deployment_region))
1692 if not aurora_info:
1693 return []
1695 widgets: list[cloudwatch.IWidget] = []
1697 widgets.append(
1698 cloudwatch.TextWidget(
1699 markdown=(
1700 "# Aurora pgvector (Serverless v2)\n"
1701 "ACU utilization, database connections, query latency, "
1702 "and CPU utilization — pinned to each regional GCO "
1703 "Aurora cluster by ID. ACU utilization is the primary "
1704 "scale/cost signal for Serverless v2."
1705 ),
1706 width=24,
1707 height=1,
1708 )
1709 )
1711 # ACU utilization and capacity
1712 for cluster_id, region in aurora_info:
1713 widgets.append(
1714 cloudwatch.GraphWidget(
1715 title=f"Aurora - ACU Utilization & Capacity ({region})",
1716 left=[
1717 cloudwatch.Metric(
1718 namespace="AWS/RDS",
1719 metric_name="ACUUtilization",
1720 dimensions_map={"DBClusterIdentifier": cluster_id},
1721 statistic="Average",
1722 period=Duration.minutes(1),
1723 label="ACU %",
1724 region=region,
1725 ),
1726 ],
1727 right=[
1728 cloudwatch.Metric(
1729 namespace="AWS/RDS",
1730 metric_name="ServerlessDatabaseCapacity",
1731 dimensions_map={"DBClusterIdentifier": cluster_id},
1732 statistic="Average",
1733 period=Duration.minutes(1),
1734 label="ACUs",
1735 region=region,
1736 ),
1737 ],
1738 width=12,
1739 height=6,
1740 region=region,
1741 )
1742 )
1744 # Database connections and CPU utilization
1745 for cluster_id, region in aurora_info:
1746 widgets.append(
1747 cloudwatch.GraphWidget(
1748 title=f"Aurora - Connections & CPU ({region})",
1749 left=[
1750 cloudwatch.Metric(
1751 namespace="AWS/RDS",
1752 metric_name="DatabaseConnections",
1753 dimensions_map={"DBClusterIdentifier": cluster_id},
1754 statistic="Average",
1755 period=Duration.minutes(1),
1756 label="Connections",
1757 region=region,
1758 ),
1759 ],
1760 right=[
1761 cloudwatch.Metric(
1762 namespace="AWS/RDS",
1763 metric_name="CPUUtilization",
1764 dimensions_map={"DBClusterIdentifier": cluster_id},
1765 statistic="Average",
1766 period=Duration.minutes(1),
1767 label="CPU %",
1768 region=region,
1769 ),
1770 ],
1771 width=12,
1772 height=6,
1773 region=region,
1774 )
1775 )
1777 # Read and write latency p99
1778 for cluster_id, region in aurora_info:
1779 widgets.append(
1780 cloudwatch.GraphWidget(
1781 title=f"Aurora - Query Latency p99 ({region})",
1782 left=[
1783 cloudwatch.Metric(
1784 namespace="AWS/RDS",
1785 metric_name="ReadLatency",
1786 dimensions_map={"DBClusterIdentifier": cluster_id},
1787 statistic="p99",
1788 period=Duration.minutes(1),
1789 label="Read p99",
1790 region=region,
1791 ),
1792 ],
1793 right=[
1794 cloudwatch.Metric(
1795 namespace="AWS/RDS",
1796 metric_name="WriteLatency",
1797 dimensions_map={"DBClusterIdentifier": cluster_id},
1798 statistic="p99",
1799 period=Duration.minutes(1),
1800 label="Write p99",
1801 region=region,
1802 ),
1803 ],
1804 width=24,
1805 height=6,
1806 region=region,
1807 )
1808 )
1810 return widgets
1812 def _create_alb_widgets(self) -> list[cloudwatch.IWidget]:
1813 """Create ALB monitoring widgets scoped to the GCO platform ALB.
1815 ALBs are created by the AWS Load Balancer Controller at runtime
1816 from an Ingress resource (not by CDK), so the exact ALB name
1817 isn't known at synth time. We originally tried reading the ARN
1818 off the regional stack's ``GaRegistration`` custom resource via
1819 ``cross_region_references=True``, but that path races the
1820 custom-resource response pipeline: CDK's cross-region
1821 ``ExportsWriter`` executes ``Fn::GetAtt: [GaRegistration, AlbArn]``
1822 before CloudFormation has the updated response data stored, and
1823 errors with "Vendor response doesn't contain AlbArn attribute".
1825 Instead we use a SEARCH expression with a composite-token
1826 filter. The ALB Controller names the platform ALB
1827 ``k8s-gco-<hash>`` (the namespace is shortened because the
1828 controller enforces a 32-char total name limit); CloudWatch's
1829 ``LoadBalancer`` dimension is the ARN suffix ``app/<name>/<hash>``,
1830 so an unquoted filter ``LoadBalancer=app/k8s-gco-`` performs a
1831 composite-token match (the sequence ``app``, ``k``, ``8``, ``s``,
1832 ``gco`` must appear consecutively in the dimension value).
1833 Double-quoted filters would be exact matches and return nothing
1834 because no ALB's dimension value is literally ``app/k8s-gco-``.
1835 """
1836 widgets: list[cloudwatch.IWidget] = []
1838 # Section header
1839 widgets.append(
1840 cloudwatch.TextWidget(
1841 markdown=(
1842 "# Application Load Balancers\n"
1843 "Request metrics, response time, HTTP errors, and "
1844 "connection counts — scoped via SEARCH composite-token "
1845 "match to ALBs named ``app/k8s-gco-*`` so only the GCO "
1846 "platform ALB in each region appears. Inference ALBs "
1847 "(named per endpoint) and unrelated ALBs in the "
1848 "account are excluded."
1849 ),
1850 width=24,
1851 height=1,
1852 )
1853 )
1855 # Per-region request count
1856 for region in self.regions:
1857 widgets.append(
1858 cloudwatch.GraphWidget(
1859 title=f"ALB - Request Count ({region})",
1860 left=[
1861 cloudwatch.MathExpression(
1862 expression=(
1863 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
1864 'MetricName="RequestCount" '
1865 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'
1866 ),
1867 label="Request Count",
1868 period=Duration.minutes(5),
1869 ),
1870 ],
1871 width=12,
1872 height=6,
1873 region=region,
1874 )
1875 )
1877 # Per-region response time (average and p99)
1878 for region in self.regions:
1879 widgets.append(
1880 cloudwatch.GraphWidget(
1881 title=f"ALB - Response Time ({region})",
1882 left=[
1883 cloudwatch.MathExpression(
1884 expression=(
1885 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
1886 'MetricName="TargetResponseTime" '
1887 'LoadBalancer=app/k8s-gco-\', "Average", 300)'
1888 ),
1889 label="Avg Response Time",
1890 period=Duration.minutes(5),
1891 ),
1892 cloudwatch.MathExpression(
1893 expression=(
1894 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
1895 'MetricName="TargetResponseTime" '
1896 'LoadBalancer=app/k8s-gco-\', "p99", 300)'
1897 ),
1898 label="p99 Response Time",
1899 period=Duration.minutes(5),
1900 ),
1901 ],
1902 width=12,
1903 height=6,
1904 region=region,
1905 )
1906 )
1908 # Per-region HTTP errors (4XX + 5XX from targets)
1909 for region in self.regions:
1910 widgets.append(
1911 cloudwatch.GraphWidget(
1912 title=f"ALB - HTTP Errors ({region})",
1913 left=[
1914 cloudwatch.MathExpression(
1915 expression=(
1916 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
1917 'MetricName="HTTPCode_Target_4XX_Count" '
1918 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'
1919 ),
1920 label="4XX Errors",
1921 period=Duration.minutes(5),
1922 ),
1923 ],
1924 right=[
1925 cloudwatch.MathExpression(
1926 expression=(
1927 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
1928 'MetricName="HTTPCode_Target_5XX_Count" '
1929 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'
1930 ),
1931 label="5XX Errors",
1932 period=Duration.minutes(5),
1933 ),
1934 ],
1935 width=12,
1936 height=6,
1937 region=region,
1938 )
1939 )
1941 # Per-region active connections
1942 for region in self.regions:
1943 widgets.append(
1944 cloudwatch.GraphWidget(
1945 title=f"ALB - Active Connections ({region})",
1946 left=[
1947 cloudwatch.MathExpression(
1948 expression=(
1949 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
1950 'MetricName="ActiveConnectionCount" '
1951 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'
1952 ),
1953 label="Active Connections",
1954 period=Duration.minutes(5),
1955 ),
1956 ],
1957 width=12,
1958 height=6,
1959 region=region,
1960 )
1961 )
1963 return widgets
1965 def _create_application_widgets(self) -> list[cloudwatch.IWidget]:
1966 """Create custom application monitoring widgets"""
1967 widgets: list[cloudwatch.IWidget] = []
1969 # Section header
1970 widgets.append(
1971 cloudwatch.TextWidget(
1972 markdown="# Application Metrics\n"
1973 "Health monitor and manifest processor metrics. "
1974 "Application logs are available in Container Insights at "
1975 "`/aws/containerinsights/<cluster>/application`.",
1976 width=24,
1977 height=1,
1978 )
1979 )
1981 # Build cluster info from regional stacks: (cluster_name, region)
1982 cluster_info = [
1983 (regional_stack.cluster.cluster_name, regional_stack.deployment_region)
1984 for regional_stack in self.regional_stacks
1985 ]
1987 # Health monitor metrics
1988 health_monitor_widget = cloudwatch.GraphWidget(
1989 title="Health Monitor - Resource Utilization",
1990 left=[
1991 cloudwatch.Metric(
1992 namespace="GCO/HealthMonitor",
1993 metric_name="ClusterCpuUtilization",
1994 dimensions_map={
1995 "ClusterName": cluster_name,
1996 "Region": region,
1997 },
1998 statistic="Average",
1999 period=Duration.minutes(5),
2000 label=f"{region} CPU",
2001 region=region,
2002 )
2003 for cluster_name, region in cluster_info
2004 ],
2005 right=[
2006 cloudwatch.Metric(
2007 namespace="GCO/HealthMonitor",
2008 metric_name="ClusterMemoryUtilization",
2009 dimensions_map={
2010 "ClusterName": cluster_name,
2011 "Region": region,
2012 },
2013 statistic="Average",
2014 period=Duration.minutes(5),
2015 label=f"{region} Memory",
2016 region=region,
2017 )
2018 for cluster_name, region in cluster_info
2019 ],
2020 width=12,
2021 height=6,
2022 )
2023 widgets.append(health_monitor_widget)
2025 # Manifest processor metrics
2026 manifest_processor_widget = cloudwatch.GraphWidget(
2027 title="Manifest Processor - Submissions",
2028 left=[
2029 cloudwatch.Metric(
2030 namespace="GCO/ManifestProcessor",
2031 metric_name="ManifestSubmissions",
2032 dimensions_map={
2033 "ClusterName": cluster_name,
2034 "Region": region,
2035 },
2036 statistic="Sum",
2037 period=Duration.minutes(5),
2038 label=f"{region} Submissions",
2039 region=region,
2040 )
2041 for cluster_name, region in cluster_info
2042 ],
2043 right=[
2044 cloudwatch.Metric(
2045 namespace="GCO/ManifestProcessor",
2046 metric_name="ManifestFailures",
2047 dimensions_map={
2048 "ClusterName": cluster_name,
2049 "Region": region,
2050 },
2051 statistic="Sum",
2052 period=Duration.minutes(5),
2053 label=f"{region} Failures",
2054 color="#d62728",
2055 region=region,
2056 )
2057 for cluster_name, region in cluster_info
2058 ],
2059 width=12,
2060 height=6,
2061 )
2062 widgets.append(manifest_processor_widget)
2064 # Container Insights - Pod restarts (indicates application issues)
2065 pod_restarts_widget = cloudwatch.GraphWidget(
2066 title="Container Insights - Pod Restarts",
2067 left=[
2068 cloudwatch.Metric(
2069 namespace="ContainerInsights",
2070 metric_name="pod_number_of_container_restarts",
2071 dimensions_map={"ClusterName": cluster_name},
2072 statistic="Sum",
2073 period=Duration.minutes(5),
2074 label=f"{region}",
2075 region=region,
2076 )
2077 for cluster_name, region in cluster_info
2078 ],
2079 width=12,
2080 height=6,
2081 )
2082 widgets.append(pod_restarts_widget)
2084 # Secret rotation Lambda metrics (Secrets Manager doesn't publish rotation metrics,
2085 # so we monitor the rotation Lambda function instead)
2086 if self.api_gateway_stack: 2086 ↛ 2122line 2086 didn't jump to line 2122 because the condition on line 2086 was always true
2087 rotation_function_name = self.api_gateway_stack.rotation_lambda.function_name
2088 api_gw_region = self.config.get_api_gateway_region()
2090 rotation_widget = cloudwatch.GraphWidget(
2091 title="Secret Rotation Lambda - Invocations & Errors",
2092 left=[
2093 cloudwatch.Metric(
2094 namespace="AWS/Lambda",
2095 metric_name="Invocations",
2096 dimensions_map={"FunctionName": rotation_function_name},
2097 statistic="Sum",
2098 period=Duration.hours(1),
2099 label="Invocations",
2100 color="#2ca02c",
2101 region=api_gw_region,
2102 ),
2103 ],
2104 right=[
2105 cloudwatch.Metric(
2106 namespace="AWS/Lambda",
2107 metric_name="Errors",
2108 dimensions_map={"FunctionName": rotation_function_name},
2109 statistic="Sum",
2110 period=Duration.hours(1),
2111 label="Errors",
2112 color="#d62728",
2113 region=api_gw_region,
2114 ),
2115 ],
2116 width=12,
2117 height=6,
2118 )
2119 widgets.append(rotation_widget)
2120 else:
2121 # Fallback text widget if api_gateway_stack not available
2122 fallback_widget = cloudwatch.TextWidget(
2123 markdown="**Secret Rotation:** API Gateway stack not configured. "
2124 "Rotation Lambda metrics unavailable.",
2125 width=12,
2126 height=6,
2127 )
2128 widgets.append(fallback_widget)
2130 return widgets
2132 def _create_alarms(self) -> None:
2133 """Create CloudWatch alarms"""
2134 self._create_global_accelerator_alarms()
2135 self._create_api_gateway_alarms()
2136 self._create_lambda_alarms()
2137 self._create_sqs_alarms()
2138 self._create_dynamodb_alarms()
2139 self._create_eks_alarms()
2140 self._create_alb_alarms()
2141 self._create_application_alarms()
2143 def _create_global_accelerator_alarms(self) -> None:
2144 """Create Global Accelerator alarms.
2146 Note: Global Accelerator metrics are only available in us-west-2.
2147 CloudWatch Alarms must be in the same region as the metrics they monitor.
2148 Since this monitoring stack may be deployed in a different region,
2149 we skip GA alarms here. To monitor GA, either:
2150 1. Create alarms manually in us-west-2
2151 2. Use CloudWatch cross-region dashboard widgets (which we do)
2152 3. Deploy a separate alarm stack in us-west-2
2153 """
2154 # GA alarms skipped - metrics only available in us-west-2
2155 # Dashboard widgets use region parameter to display GA metrics correctly
2156 pass
2158 def _create_api_gateway_alarms(self) -> None:
2159 """Create API Gateway alarms"""
2160 # Get the actual API name from the api_gateway_stack
2161 api_name = (
2162 self.api_gateway_stack.api.rest_api_name
2163 if self.api_gateway_stack
2164 else f"{self.project_name}-global-api"
2165 )
2167 # High 5XX error rate
2168 api_5xx_alarm = cloudwatch.Alarm(
2169 self,
2170 "ApiGateway5xxAlarm",
2171 alarm_description="API Gateway has high 5XX error rate",
2172 metric=cloudwatch.Metric(
2173 namespace="AWS/ApiGateway",
2174 metric_name="5XXError",
2175 dimensions_map={"ApiName": api_name},
2176 statistic="Sum",
2177 period=Duration.minutes(5),
2178 ),
2179 threshold=10,
2180 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2181 evaluation_periods=2,
2182 datapoints_to_alarm=2,
2183 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2184 )
2185 api_5xx_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2187 # High latency
2188 api_latency_alarm = cloudwatch.Alarm(
2189 self,
2190 "ApiGatewayHighLatencyAlarm",
2191 alarm_description="API Gateway has high latency",
2192 metric=cloudwatch.Metric(
2193 namespace="AWS/ApiGateway",
2194 metric_name="Latency",
2195 dimensions_map={"ApiName": api_name},
2196 statistic="p99",
2197 period=Duration.minutes(5),
2198 ),
2199 threshold=10000, # 10 seconds
2200 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2201 evaluation_periods=3,
2202 datapoints_to_alarm=2,
2203 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2204 )
2205 api_latency_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2207 def _create_lambda_alarms(self) -> None:
2208 """Create Lambda function alarms."""
2209 if self.api_gateway_stack is None: 2209 ↛ 2210line 2209 didn't jump to line 2210 because the condition on line 2209 was never true
2210 return
2212 proxy_lambda = self.api_gateway_stack.proxy_lambda
2213 if proxy_lambda is not None: 2213 ↛ 2253line 2213 didn't jump to line 2253 because the condition on line 2213 was always true
2214 proxy_function_name = proxy_lambda.function_name
2215 proxy_errors_alarm = cloudwatch.Alarm(
2216 self,
2217 "ProxyLambdaErrorsAlarm",
2218 alarm_description="API Gateway proxy Lambda has errors",
2219 metric=cloudwatch.Metric(
2220 namespace="AWS/Lambda",
2221 metric_name="Errors",
2222 dimensions_map={"FunctionName": proxy_function_name},
2223 statistic="Sum",
2224 period=Duration.minutes(5),
2225 ),
2226 threshold=5,
2227 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2228 evaluation_periods=2,
2229 datapoints_to_alarm=2,
2230 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2231 )
2232 proxy_errors_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2234 proxy_throttles_alarm = cloudwatch.Alarm(
2235 self,
2236 "ProxyLambdaThrottlesAlarm",
2237 alarm_description="API Gateway proxy Lambda is being throttled",
2238 metric=cloudwatch.Metric(
2239 namespace="AWS/Lambda",
2240 metric_name="Throttles",
2241 dimensions_map={"FunctionName": proxy_function_name},
2242 statistic="Sum",
2243 period=Duration.minutes(5),
2244 ),
2245 threshold=1,
2246 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2247 evaluation_periods=2,
2248 datapoints_to_alarm=2,
2249 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2250 )
2251 proxy_throttles_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2253 rotation_errors_alarm = cloudwatch.Alarm(
2254 self,
2255 "RotationLambdaErrorsAlarm",
2256 alarm_description="Secret rotation Lambda has errors",
2257 metric=cloudwatch.Metric(
2258 namespace="AWS/Lambda",
2259 metric_name="Errors",
2260 dimensions_map={
2261 "FunctionName": self.api_gateway_stack.rotation_lambda.function_name
2262 },
2263 statistic="Sum",
2264 period=Duration.hours(1),
2265 ),
2266 threshold=1,
2267 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2268 evaluation_periods=1,
2269 datapoints_to_alarm=1,
2270 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2271 )
2272 rotation_errors_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2274 def _create_sqs_alarms(self) -> None:
2275 """Create SQS queue alarms"""
2276 for regional_stack in self.regional_stacks:
2277 region = regional_stack.deployment_region
2278 queue_name = regional_stack.job_queue.queue_name
2279 dlq_name = regional_stack.job_dlq.queue_name
2280 region_id = region.replace("-", "").title()
2282 # Old message alarm (stuck jobs)
2283 old_message_alarm = cloudwatch.Alarm(
2284 self,
2285 f"SqsOldMessageAlarm{region_id}",
2286 alarm_description=f"SQS queue in {region} has old messages (potential stuck jobs)",
2287 metric=cloudwatch.Metric(
2288 namespace="AWS/SQS",
2289 metric_name="ApproximateAgeOfOldestMessage",
2290 dimensions_map={"QueueName": queue_name},
2291 statistic="Maximum",
2292 period=Duration.minutes(5),
2293 ),
2294 threshold=3600, # 1 hour
2295 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2296 evaluation_periods=2,
2297 datapoints_to_alarm=2,
2298 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2299 )
2300 old_message_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2302 # Dead letter queue alarm
2303 dlq_alarm = cloudwatch.Alarm(
2304 self,
2305 f"SqsDlqAlarm{region_id}",
2306 alarm_description=f"SQS dead letter queue in {region} has messages",
2307 metric=cloudwatch.Metric(
2308 namespace="AWS/SQS",
2309 metric_name="ApproximateNumberOfMessagesVisible",
2310 dimensions_map={"QueueName": dlq_name},
2311 statistic="Sum",
2312 period=Duration.minutes(5),
2313 ),
2314 threshold=1,
2315 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2316 evaluation_periods=1,
2317 datapoints_to_alarm=1,
2318 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2319 )
2320 dlq_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2322 def _create_dynamodb_alarms(self) -> None:
2323 """Create DynamoDB alarms for job queue, templates, and webhooks tables."""
2324 # Get table names from global stack
2325 jobs_table = self.global_stack.jobs_table.table_name
2327 # DynamoDB tables are in the global region
2328 global_region = self.config.get_global_region()
2330 # Jobs table throttling alarm
2331 jobs_throttle_alarm = cloudwatch.Alarm(
2332 self,
2333 "DynamoDBJobsThrottleAlarm",
2334 alarm_description="DynamoDB jobs table is being throttled",
2335 metric=cloudwatch.Metric(
2336 namespace="AWS/DynamoDB",
2337 metric_name="ThrottledRequests",
2338 dimensions_map={"TableName": jobs_table},
2339 statistic="Sum",
2340 period=Duration.minutes(5),
2341 region=global_region,
2342 ),
2343 threshold=1,
2344 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2345 evaluation_periods=2,
2346 datapoints_to_alarm=2,
2347 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2348 )
2349 jobs_throttle_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2351 # Jobs table system errors alarm
2352 jobs_errors_alarm = cloudwatch.Alarm(
2353 self,
2354 "DynamoDBJobsErrorsAlarm",
2355 alarm_description="DynamoDB jobs table has system errors",
2356 metric=cloudwatch.Metric(
2357 namespace="AWS/DynamoDB",
2358 metric_name="SystemErrors",
2359 dimensions_map={"TableName": jobs_table},
2360 statistic="Sum",
2361 period=Duration.minutes(5),
2362 region=global_region,
2363 ),
2364 threshold=1,
2365 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2366 evaluation_periods=1,
2367 datapoints_to_alarm=1,
2368 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2369 )
2370 jobs_errors_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2372 def _create_eks_alarms(self) -> None:
2373 """Create EKS cluster alarms"""
2374 for regional_stack in self.regional_stacks:
2375 region = regional_stack.deployment_region
2376 cluster_name = regional_stack.cluster.cluster_name
2377 region_id = region.replace("-", "").title()
2379 # High CPU utilization alarm (node-level metric)
2380 high_cpu_alarm = cloudwatch.Alarm(
2381 self,
2382 f"EksHighCpuAlarm{region_id}",
2383 alarm_description=f"EKS cluster {cluster_name} has high CPU utilization",
2384 metric=cloudwatch.Metric(
2385 namespace="ContainerInsights",
2386 metric_name="node_cpu_utilization",
2387 dimensions_map={"ClusterName": cluster_name},
2388 statistic="Average",
2389 period=Duration.minutes(5),
2390 ),
2391 threshold=80,
2392 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2393 evaluation_periods=3,
2394 datapoints_to_alarm=2,
2395 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2396 )
2397 high_cpu_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2399 # High memory utilization alarm (node-level metric)
2400 high_memory_alarm = cloudwatch.Alarm(
2401 self,
2402 f"EksHighMemoryAlarm{region_id}",
2403 alarm_description=f"EKS cluster {cluster_name} has high memory utilization",
2404 metric=cloudwatch.Metric(
2405 namespace="ContainerInsights",
2406 metric_name="node_memory_utilization",
2407 dimensions_map={"ClusterName": cluster_name},
2408 statistic="Average",
2409 period=Duration.minutes(5),
2410 ),
2411 threshold=85,
2412 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2413 evaluation_periods=3,
2414 datapoints_to_alarm=2,
2415 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2416 )
2417 high_memory_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2419 def _create_alb_alarms(self) -> None:
2420 """Create ALB alarms.
2422 Status: no alarms created yet, even though we now have the ALB
2423 ARN at deploy time via the GA registration custom resource (which
2424 also feeds the dashboard widgets). Adding per-ALB alarms here is
2425 a straightforward enhancement — derive the ``LoadBalancer``
2426 dimension the same way ``_create_alb_widgets`` does
2427 (``Fn.split(":loadbalancer/", alb_arn)[1]``) and wire it into
2428 ``cloudwatch.Alarm`` constructs.
2430 For now we rely on:
2431 1. Dashboard widgets pinned to each platform ALB (see
2432 ``_create_alb_widgets``)
2433 2. EKS Container Insights alarms for pod/node health
2434 3. API Gateway alarms for request-level monitoring
2435 """
2436 # TODO: Add UnHealthyHostCount / 5XXCount alarms using the ARN
2437 # returned by regional_stack.ga_registration.get_att_string("AlbArn").
2438 # The test suite explicitly documents that the ALB alarm count is
2439 # currently zero (test_alb_unhealthy_hosts_alarm_skipped); update
2440 # that test when adding real alarms.
2441 pass
2443 def _create_application_alarms(self) -> None:
2444 """Create application-specific alarms"""
2445 for regional_stack in self.regional_stacks:
2446 region = regional_stack.deployment_region
2447 cluster_name = regional_stack.cluster.cluster_name
2448 region_id = region.replace("-", "").title()
2450 # High manifest failure rate alarm
2451 high_failure_rate_alarm = cloudwatch.Alarm(
2452 self,
2453 f"ManifestHighFailureRateAlarm{region_id}",
2454 alarm_description=f"Manifest processor in {region} has high failure rate",
2455 metric=cloudwatch.Metric(
2456 namespace="GCO/ManifestProcessor",
2457 metric_name="ManifestFailures",
2458 dimensions_map={"ClusterName": cluster_name, "Region": region},
2459 statistic="Sum",
2460 period=Duration.minutes(5),
2461 ),
2462 threshold=10,
2463 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2464 evaluation_periods=2,
2465 datapoints_to_alarm=2,
2466 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2467 )
2468 high_failure_rate_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2470 def _create_composite_alarms(self) -> None:
2471 """Create composite alarms for better signal-to-noise ratio"""
2473 # Store individual alarms for composite alarm references
2474 regional_alarms: dict[str, list[cloudwatch.Alarm]] = {}
2476 for regional_stack in self.regional_stacks:
2477 region = regional_stack.deployment_region
2478 cluster_name = regional_stack.cluster.cluster_name
2479 region_id = region.replace("-", "").title()
2480 regional_alarms[region] = []
2482 # Create regional health composite alarm
2483 # Triggers when multiple issues occur in the same region
2484 eks_cpu_alarm = cloudwatch.Alarm(
2485 self,
2486 f"CompositeEksCpu{region_id}",
2487 metric=cloudwatch.Metric(
2488 namespace="ContainerInsights",
2489 metric_name="node_cpu_utilization",
2490 dimensions_map={"ClusterName": cluster_name},
2491 statistic="Average",
2492 period=Duration.minutes(5),
2493 ),
2494 threshold=90,
2495 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2496 evaluation_periods=2,
2497 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2498 )
2499 regional_alarms[region].append(eks_cpu_alarm)
2501 eks_memory_alarm = cloudwatch.Alarm(
2502 self,
2503 f"CompositeEksMemory{region_id}",
2504 metric=cloudwatch.Metric(
2505 namespace="ContainerInsights",
2506 metric_name="node_memory_utilization",
2507 dimensions_map={"ClusterName": cluster_name},
2508 statistic="Average",
2509 period=Duration.minutes(5),
2510 ),
2511 threshold=90,
2512 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2513 evaluation_periods=2,
2514 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2515 )
2516 regional_alarms[region].append(eks_memory_alarm)
2518 # Create composite alarm for critical regional issues
2519 for region, alarms in regional_alarms.items():
2520 region_id = region.replace("-", "").title()
2521 if len(alarms) >= 2: 2521 ↛ 2519line 2521 didn't jump to line 2519 because the condition on line 2521 was always true
2522 composite_alarm = cloudwatch.CompositeAlarm(
2523 self,
2524 f"RegionalCriticalAlarm{region_id}",
2525 alarm_description=f"Critical: Multiple issues detected in {region}",
2526 alarm_rule=cloudwatch.AlarmRule.all_of(*alarms),
2527 )
2528 composite_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2530 # API Gateway + Lambda composite alarm (only if api_gateway_stack is available)
2531 if self.api_gateway_stack and self.api_gateway_stack.proxy_lambda is not None: 2531 ↛ exitline 2531 didn't return from function '_create_composite_alarms' because the condition on line 2531 was always true
2532 api_name = self.api_gateway_stack.api.rest_api_name
2533 proxy_function_name = self.api_gateway_stack.proxy_lambda.function_name
2535 api_error_alarm = cloudwatch.Alarm(
2536 self,
2537 "CompositeApiErrors",
2538 metric=cloudwatch.Metric(
2539 namespace="AWS/ApiGateway",
2540 metric_name="5XXError",
2541 dimensions_map={"ApiName": api_name},
2542 statistic="Sum",
2543 period=Duration.minutes(5),
2544 ),
2545 threshold=5,
2546 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2547 evaluation_periods=2,
2548 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2549 )
2551 lambda_error_alarm = cloudwatch.Alarm(
2552 self,
2553 "CompositeLambdaErrors",
2554 metric=cloudwatch.Metric(
2555 namespace="AWS/Lambda",
2556 metric_name="Errors",
2557 dimensions_map={"FunctionName": proxy_function_name},
2558 statistic="Sum",
2559 period=Duration.minutes(5),
2560 ),
2561 threshold=3,
2562 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2563 evaluation_periods=2,
2564 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2565 )
2567 api_lambda_composite = cloudwatch.CompositeAlarm(
2568 self,
2569 "ApiLambdaCompositeAlarm",
2570 alarm_description="Critical: Both API Gateway and Lambda proxy have errors",
2571 alarm_rule=cloudwatch.AlarmRule.all_of(api_error_alarm, lambda_error_alarm),
2572 )
2573 api_lambda_composite.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2575 def _create_custom_metrics(self) -> None:
2576 """Create custom metric filters and log groups"""
2577 for regional_stack in self.regional_stacks:
2578 region = regional_stack.deployment_region
2579 region_id = region.replace("-", "").title()
2581 # Health monitor log group
2582 # log_group_name intentionally omitted - let CDK generate unique name
2583 logs.LogGroup(
2584 self,
2585 f"HealthMonitorLogGroup{region_id}",
2586 retention=logs.RetentionDays.ONE_MONTH,
2587 removal_policy=RemovalPolicy.DESTROY,
2588 )
2590 # Manifest processor log group
2591 # log_group_name intentionally omitted - let CDK generate unique name
2592 logs.LogGroup(
2593 self,
2594 f"ManifestProcessorLogGroup{region_id}",
2595 retention=logs.RetentionDays.ONE_MONTH,
2596 removal_policy=RemovalPolicy.DESTROY,
2597 )
2599 def _create_outputs(self) -> None:
2600 """Create CloudFormation outputs"""
2601 CfnOutput(
2602 self,
2603 "DashboardUrl",
2604 value=f"https://console.aws.amazon.com/cloudwatch/home?region={self.region}#dashboards:name={self.dashboard.dashboard_name}",
2605 description="CloudWatch Dashboard URL",
2606 )
2608 CfnOutput(
2609 self,
2610 "AlertTopicArn",
2611 value=self.alert_topic.topic_arn,
2612 description="SNS Topic ARN for monitoring alerts",
2613 )
2615 CfnOutput(
2616 self,
2617 "AlarmCount",
2618 value="See CloudWatch Alarms console for full list",
2619 description="Monitoring alarms created",
2620 )