Coverage for gco/stacks/regional_stack.py: 97.76%
842 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-30 21:22 +0000
1"""
2Regional stack for GCO (Global Capacity Orchestrator on AWS) - EKS cluster and ALB per region.
4This is the largest stack in the project (~3200 lines) and creates all regional
5resources for a single AWS region. One instance is deployed per region defined
6in cdk.json.
8Resources Created:
9 VPC & Networking:
10 - VPC spanning every AZ in the region, public subnets (NAT), private subnets (EKS and ALB)
11 - 2 NAT Gateways for high availability
12 - VPC endpoints for ECR, S3, STS, Secrets Manager, SSM, CloudWatch
13 - VPC Flow Logs (CloudWatch Logs, 30-day retention)
15 EKS Cluster (Auto Mode):
16 - Managed control plane with full logging (API, Audit, Authenticator, Controller Manager, Scheduler)
17 - Built-in NodePools: system, general-purpose
18 - Custom NodePools: gpu-x86-pool, gpu-arm-pool, gpu-inference-pool,
19 gpu-efa-pool, mooncake-efa-pool, neuron-pool, cpu-general-pool
20 - IRSA roles for service accounts (Secrets Manager, SQS, DynamoDB, CloudWatch, S3, EFS)
22 Load Balancing:
23 - Internal ALB created from Gateway API resources by the self-managed
24 AWS Load Balancer Controller
25 - Always-deployed regional API bridge reaches the ALB through a VPC Lambda;
26 direct caller access is optional in ``aws`` and required elsewhere
27 - Global Accelerator endpoint registration in commercial ``aws`` only
29 Storage:
30 - EFS with dynamic provisioning (CSI driver, access points, encryption at rest + in transit)
31 - FSx for Lustre (optional, toggled via cdk.json)
32 - Valkey Serverless cache (optional)
33 - Aurora Serverless v2 with pgvector (optional)
35 Lambda Functions:
36 - kubectl-applier: applies K8s manifests during deployment
37 - helm-installer: installs Helm charts (KEDA, Volcano, KubeRay, etc.)
38 - ga-registration: registers the ALB with Global Accelerator in ``aws``
39 - regional-api-proxy: separate-stack VPC proxy used by the always-on
40 aggregation bridge and by optional direct callers in ``aws`` or the
41 required regional workload ingress in other partitions
43 Container Images:
44 - ECR repositories + Docker image builds for health-monitor, manifest-processor,
45 inference-proxy, inference-monitor, queue-processor
47 SQS:
48 - Regional job queue + dead letter queue (for gco jobs submit-sqs)
50Key Design Decisions:
51 - EKS Auto Mode handles node provisioning — no managed node groups or Karpenter provisioners
52 - NodePools use WhenEmpty consolidation for inference to avoid disrupting long-running pods
53 - IRSA (IAM Roles for Service Accounts) for least-privilege pod-level AWS access
54 - All optional features (FSx, Valkey, Aurora) are toggled via cdk.json context variables
55 - Template variables in K8s manifests ({{PLACEHOLDER}}) are replaced at deploy time
57Dependencies:
58 - GCOGlobalStack (partition-wide state and, in ``aws``, Global Accelerator endpoint groups)
59 - GCOApiGatewayGlobalStack (for auth secret ARN)
61Modification Guide:
62 - To add a new NodePool: add a YAML manifest in lambda/kubectl-applier-simple/manifests/ (40-49 range)
63 - To add a new service: add ECR image build here, Dockerfile in dockerfiles/, manifest in manifests/
64 - To add a new optional feature: add a cdk.json context toggle, guard with if/else in this file
65 - To change EKS version: update KUBERNETES_VERSION in constants.py
66"""
68from __future__ import annotations
70import os
71import re
72from dataclasses import dataclass
73from datetime import UTC, datetime
74from pathlib import Path
75from typing import Any
77import aws_cdk.aws_eks_v2 as eks
78import yaml
79from aws_cdk import (
80 Acknowledgment,
81 CfnJson,
82 CfnOutput,
83 CfnTag,
84 CustomResource,
85 Duration,
86 Fn,
87 RemovalPolicy,
88 Stack,
89 Validations,
90)
91from aws_cdk import aws_ec2 as ec2
92from aws_cdk import aws_ecr as ecr
93from aws_cdk import aws_ecr_assets as ecr_assets
94from aws_cdk import aws_efs as efs
95from aws_cdk import aws_eks as eks_l1 # L1 constructs (CfnPodIdentityAssociation)
96from aws_cdk import aws_events as events
97from aws_cdk import aws_events_targets as events_targets
98from aws_cdk import aws_fsx as fsx
99from aws_cdk import aws_iam as iam
100from aws_cdk import aws_kms as kms
101from aws_cdk import aws_lambda as lambda_
102from aws_cdk import aws_logs as logs
103from aws_cdk import aws_s3 as s3
104from aws_cdk import aws_sns as sns
105from aws_cdk import aws_sqs as sqs
106from aws_cdk import aws_ssm as ssm
107from aws_cdk import aws_stepfunctions as sfn
108from aws_cdk import aws_stepfunctions_tasks as sfn_tasks
109from aws_cdk import custom_resources as cr
110from constructs import Construct
112from gco.config.config_loader import ConfigLoader
113from gco.stacks.aws_load_balancer_controller_policy import (
114 aws_load_balancer_controller_policy_document,
115)
116from gco.stacks.constants import (
117 AURORA_POSTGRES_VERSION,
118 EKS_ADDON_CLOUDWATCH_OBSERVABILITY,
119 EKS_ADDON_EFS_CSI_DRIVER,
120 EKS_ADDON_FSX_CSI_DRIVER,
121 EKS_ADDON_METRICS_SERVER,
122 EKS_ADDON_POD_IDENTITY_AGENT,
123 EKS_UNSUPPORTED_AZ_IDS,
124 LAMBDA_PYTHON_RUNTIME,
125 MOONCAKE_MASTER_DEFAULT_IMAGE,
126 api_gateway_auth_secret_name,
127 backend_tls_certificate_arn_parameter_name,
128 cluster_shared_ssm_parameter_prefix,
129 cost_report_bucket_name,
130 regional_shared_bucket_name_prefix,
131 regional_shared_ssm_parameter_prefix,
132)
134# <pyflowchart-code-diagram> BEGIN - auto-inserted, do not edit
135# Generated at (UTC): 2026-07-18T01:03:40Z
136# Flowchart(s) generated from this file:
137# * ``GCORegionalStack.__init__`` -> ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack___init__.html``
138# (PNG: ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack___init__.png``)
139# * ``GCORegionalStack._get_volcano_image_mirror_config`` -> ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack__get_volcano_image_mirror_config.html``
140# (PNG: ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack__get_volcano_image_mirror_config.png``)
141# Regenerate with ``python diagrams/code_diagrams/generate.py``.
142# <pyflowchart-code-diagram> END
145_LIVE_VALIDATION_PROVIDER_LOG_CONTEXT = "gco_live_validation_retain_provider_log_groups"
148@dataclass(frozen=True)
149class SharedBucketIdentity:
150 """Identity of the always-on ``Cluster_Shared_Bucket`` owned by ``GCOGlobalStack``.
152 Every regional stack resolves this identity from the three SSM parameters
153 ``/gco/cluster-shared-bucket/{name,arn,region}`` published by
154 ``GCOGlobalStack`` in the global region. The three values are used to
155 grant IAM permissions on the bucket to the regional job-pod role and to
156 populate the ``gco-cluster-shared-bucket`` ConfigMap applied to every
157 regional EKS cluster. Frozen so it can be safely shared across helper
158 methods without accidental mutation.
159 """
161 name: str
162 arn: str
163 region: str
166def _compute_kubectl_cluster_shared_replacements(
167 shared: SharedBucketIdentity,
168) -> dict[str, str]:
169 """Build the ``{{CLUSTER_SHARED_BUCKET*}}`` kubectl-applier replacements.
171 Pure helper kept at module scope so property and presence tests can
172 inspect the output without synthesizing a full regional stack. The
173 three keys are always populated — there is no feature toggle — because
174 the ``gco-cluster-shared-bucket`` ConfigMap is applied unconditionally
175 on every regional cluster.
176 """
177 return {
178 "{{CLUSTER_SHARED_BUCKET}}": shared.name,
179 "{{CLUSTER_SHARED_BUCKET_ARN}}": shared.arn,
180 "{{CLUSTER_SHARED_BUCKET_REGION}}": shared.region,
181 }
184#: StorageClass name for in-cluster observability PVCs (Prometheus, Grafana,
185#: Alertmanager). The value overrides reference this name, and the gated gp3
186#: StorageClass manifest (25-storage-observability-gp3.yaml) declares it. A
187#: synth test asserts the two stay in lockstep. The manifest keeps this name
188#: static (a placeholder in ``metadata.name`` would fail k8s schema
189#: validation), so the toggle gate lives in an annotation value instead.
190_OBSERVABILITY_STORAGE_CLASS = "gco-observability-gp3"
193_SERVICE_IMAGE_BUILD_INPUTS = (
194 "dockerfiles/health-monitor-dockerfile",
195 "dockerfiles/manifest-processor-dockerfile",
196 "dockerfiles/inference-proxy-dockerfile",
197 "dockerfiles/inference-monitor-dockerfile",
198 "dockerfiles/queue-processor-dockerfile",
199 "dockerfiles/cost-monitor-dockerfile",
200)
201_SERVICE_IMAGE_COMMON_EXCLUDES = (
202 "cli/**",
203 "gco/stacks/**",
204 "dockerfiles/README.md",
205)
208def _service_image_asset_excludes(*included_paths: str) -> list[str]:
209 """Exclude inputs that cannot affect one production service image."""
210 included = set(included_paths)
211 return list(_SERVICE_IMAGE_COMMON_EXCLUDES) + [
212 path for path in _SERVICE_IMAGE_BUILD_INPUTS if path not in included
213 ]
216def _compute_kubectl_observability_replacements(
217 enabled: bool, *, grafana_admin_password_rotation_schedule: str = ""
218) -> dict[str, str]:
219 """Build the kubectl-applier replacements that gate the observability manifests.
221 Pure helper kept at module scope so presence/absence can be asserted
222 without synthesizing a full regional stack. When observability is enabled
223 the ``{{CLUSTER_OBSERVABILITY_ENABLED}}`` gate resolves to ``"true"`` so the
224 gp3 StorageClass, ServiceMonitors, dashboards, and credential-rotation
225 CronJob render and apply, and ``{{GRAFANA_ADMIN_PASSWORD_ROTATION_SCHEDULE}}``
226 resolves to the configured cron. When disabled the dict is empty, so those
227 manifests keep an unreplaced ``{{...}}`` token and the applier skips them —
228 the same optional-feature gating FSx and Valkey already rely on.
229 """
230 if not enabled:
231 return {}
232 return {
233 "{{CLUSTER_OBSERVABILITY_ENABLED}}": "true",
234 "{{GRAFANA_ADMIN_PASSWORD_ROTATION_SCHEDULE}}": grafana_admin_password_rotation_schedule,
235 }
238def _augment_trusted_registries_with_project_ecr(
239 base: list[str],
240 *,
241 account: str,
242 regions: list[str],
243 global_region: str,
244 url_suffix: str,
245) -> list[str]:
246 """Return the configured trusted registries plus the project's own ECR.
248 The new ``gco images build`` flow pushes images to a per-account ECR
249 registry under ``<account>.dkr.ecr.<region>.<url-suffix>/gco/<name>``.
250 Without this augmentation the queue/manifest validators would treat
251 those URIs as untrusted and reject every job that uses one — which
252 defeats the whole point of the image registry feature.
254 Returns the unique union of the operator-configured ``base`` list
255 plus the per-region project ECR hostnames (one per deployed region,
256 plus the global region where ``gco-global`` provisions the source
257 repo). Order is stable so the rendered ConfigMap doesn't churn
258 between deploys.
259 """
260 augmented: list[str] = list(base)
261 seen = set(augmented)
262 targets = list(dict.fromkeys([global_region, *regions]))
263 if account:
264 for region in targets:
265 host = f"{account}.dkr.ecr.{region}.{url_suffix}"
266 if host not in seen:
267 augmented.append(host)
268 seen.add(host)
269 return augmented
272def _deployment_timestamp() -> str:
273 """Return the synth-time token that deliberately retriggers convergence."""
274 return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
277def _load_helm_chart_order() -> list[str]:
278 """Return helm chart names in their canonical install order.
280 Reads ``lambda/helm-installer/charts.yaml`` (the source of truth, in file
281 order) so the Step Functions state machine has exactly one task per chart,
282 in the same order every deploy — kueue stays last because its mutating
283 webhook intercepts every Job/Deployment. Missing or malformed chart data
284 aborts synthesis rather than silently omitting every Helm install/uninstall.
285 """
286 charts_path = Path(__file__).resolve().parents[2] / "lambda" / "helm-installer" / "charts.yaml"
287 try:
288 with open(charts_path, encoding="utf-8") as f:
289 data = yaml.safe_load(f)
290 except (OSError, yaml.YAMLError) as exc:
291 raise RuntimeError(f"Unable to load Helm chart order from {charts_path}: {exc}") from exc
292 if not isinstance(data, dict): 292 ↛ 293line 292 didn't jump to line 293 because the condition on line 292 was never true
293 raise RuntimeError(f"Helm chart config {charts_path} must be an object")
294 charts = data.get("charts")
295 if not isinstance(charts, dict) or not charts:
296 raise RuntimeError(
297 f"Helm chart config {charts_path} must contain a non-empty charts object"
298 )
299 if any(not isinstance(name, str) or not name for name in charts): 299 ↛ 300line 299 didn't jump to line 300 because the condition on line 299 was never true
300 raise RuntimeError(f"Helm chart config {charts_path} contains an invalid chart name")
301 return list(charts)
304class GCORegionalStack(Stack):
305 """
306 Regional resources stack for a single AWS region.
308 Creates EKS cluster, load balancers, and supporting infrastructure
309 for running GCO services in a specific region.
311 Attributes:
312 vpc: VPC with public/private subnets
313 cluster: EKS Auto Mode cluster
314 """
316 @staticmethod
317 def _create_irsa_role(
318 scope: GCORegionalStack,
319 id: str,
320 oidc_provider_arn: str,
321 oidc_issuer_url: str,
322 service_account_names: list[str],
323 namespaces: list[str],
324 *,
325 include_pod_identity: bool = True,
326 ) -> iam.Role:
327 """Create an OIDC IRSA role, optionally trusted by EKS Pod Identity.
329 IRSA is the primary credential mechanism — it works reliably on EKS Auto
330 Mode by projecting a service-account token that the AWS SDK exchanges for
331 temporary credentials via the OIDC provider. General platform roles retain
332 Pod Identity as a secondary path; controller roles can disable it to keep
333 their trust policy bound to one exact Kubernetes service account.
335 Uses CfnJson to defer OIDC condition key resolution to deploy time,
336 because the issuer URL is a CloudFormation token that can't be used
337 as a Python dict key at synth time.
338 """
339 # Strip https:// from issuer URL for the OIDC condition
340 issuer = Fn.select(1, Fn.split("//", oidc_issuer_url))
342 # Build OIDC conditions using CfnJson to defer token resolution
343 # The issuer URL is a CFN token — can't be used as a dict key at synth time
344 aud_key = Fn.join("", [issuer, ":aud"])
345 sub_key = Fn.join("", [issuer, ":sub"])
347 conditions_json = CfnJson(
348 scope,
349 f"{id}OidcConditions",
350 value={
351 aud_key: "sts.amazonaws.com",
352 sub_key: [
353 f"system:serviceaccount:{ns}:{sa}"
354 for ns in namespaces
355 for sa in service_account_names
356 ],
357 },
358 )
360 role = iam.Role(
361 scope,
362 id,
363 assumed_by=iam.FederatedPrincipal(
364 federated=oidc_provider_arn,
365 conditions={
366 "StringEquals": conditions_json,
367 },
368 assume_role_action="sts:AssumeRoleWithWebIdentity",
369 ),
370 )
372 if include_pod_identity:
373 # Secondary credential path for platform workloads. Dedicated
374 # controllers such as LBC deliberately remain OIDC-only.
375 assert role.assume_role_policy is not None
376 role.assume_role_policy.add_statements(
377 iam.PolicyStatement(
378 effect=iam.Effect.ALLOW,
379 principals=[iam.ServicePrincipal("pods.eks.amazonaws.com")],
380 actions=["sts:AssumeRole", "sts:TagSession"],
381 )
382 )
383 return role
385 def __init__(
386 self,
387 scope: Construct,
388 construct_id: str,
389 config: ConfigLoader,
390 region: str,
391 auth_secret_arn: str,
392 **kwargs: Any,
393 ) -> None:
394 super().__init__(scope, construct_id, **kwargs)
396 self.config = config
397 self.deployment_region = region
398 self.auth_secret_arn = auth_secret_arn
399 self.alb_arn: str | None = None
400 retain_provider_logs = self.node.try_get_context(_LIVE_VALIDATION_PROVIDER_LOG_CONTEXT)
401 self.provider_log_group_removal_policy = (
402 RemovalPolicy.RETAIN
403 if retain_provider_logs is True
404 or (
405 isinstance(retain_provider_logs, str)
406 and retain_provider_logs.strip().casefold() == "true"
407 )
408 else RemovalPolicy.DESTROY
409 )
410 supports_global_accelerator = getattr(config, "supports_global_accelerator", None)
411 self.global_accelerator_enabled = (
412 bool(supports_global_accelerator()) if callable(supports_global_accelerator) else True
413 )
415 # Get cluster configuration for this region
416 cluster_config = self.config.get_cluster_config(region)
417 self.cluster_config = cluster_config
419 # Create VPC for the EKS cluster.
420 #
421 # ``max_azs=99`` is the CDK idiom for "span every Availability Zone the
422 # region offers" — CDK caps the value at the number of AZs actually
423 # returned for this account+region, so each AZ gets one public and one
424 # private subnet. This only enumerates the *real* AZ list when the stack
425 # is environment-specific (account + region both resolved); app.py sets
426 # the account from CDK_DEFAULT_ACCOUNT for exactly this reason. In an
427 # environment-agnostic synth (no account, e.g. some CI paths) CDK falls
428 # back to a fixed placeholder AZ list rather than the full set.
429 self.vpc = ec2.Vpc(
430 self,
431 "GCOVpc",
432 # vpc_name intentionally omitted - let CDK generate unique name
433 max_azs=99, # use every AZ in the region (each AZ gets 1 public + 1 private subnet)
434 nat_gateways=2, # For high availability
435 subnet_configuration=[
436 ec2.SubnetConfiguration(
437 name="PublicSubnet", subnet_type=ec2.SubnetType.PUBLIC, cidr_mask=24
438 ),
439 ec2.SubnetConfiguration(
440 name="PrivateSubnet",
441 subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS,
442 cidr_mask=24,
443 ),
444 ],
445 )
447 # Enable VPC Flow Logs for network traffic analysis and security monitoring
448 self._create_vpc_flow_logs()
450 # Create SQS queue for job ingestion
451 self._create_sqs_queue()
453 # Create ECR repositories and build Docker images
454 self._create_container_images()
456 # Pre-create the execution role shared by every ``cr.AwsCustomResource``
457 # in this stack. See ``_create_aws_custom_resource_role`` for the full
458 # rationale — in short, CDK's default behavior of auto-generating a
459 # Lambda role per ``AwsCustomResource`` (and then merging all the
460 # ``policy=`` statements onto it during deploy) triggers an IAM
461 # propagation race on cold creates. We sidestep the race by creating
462 # a single long-lived role up front and attaching policies to it as
463 # each consumer is built; every ``AwsCustomResource`` then passes
464 # ``role=self.aws_custom_resource_role`` instead of ``policy=``, so
465 # the singleton Lambda runs against a role whose inline policy has
466 # already replicated globally.
467 self._create_aws_custom_resource_role()
469 # Resolve this region's fixed ACM certificate ARN from the global
470 # backend-TLS registry before rendering the HTTPS-only Gateway.
471 self.backend_tls_certificate_arn = self._resolve_backend_tls_certificate_arn()
473 # Create EKS cluster
474 self._create_eks_cluster(cluster_config)
476 # Optional Volcano image mirror (cdk.json ``volcano_image_mirror``).
477 # When enabled this resolves ``self.volcano_mirror_registry`` — the
478 # gco/* ECR namespace that Volcano's ``basic.image_registry`` is
479 # redirected to, so its docker.io-only images are pulled from the
480 # project's own ECR (populated out-of-band by
481 # ``gco images mirror``) instead of rate-limited Docker
482 # Hub. Creates no CloudFormation resources; must run before
483 # ``_apply_kubernetes_manifests`` builds the ``HelmInstallCharts`` custom
484 # resource, which reads the override via ``_helm_chart_value_overrides()``.
485 self._configure_volcano_image_mirror()
487 # Resolve the always-on Cluster_Shared_Bucket identity from SSM
488 # (owned by GCOGlobalStack) and attach RW + KMS grants to the
489 # job-pod role. Runs unconditionally — the ConfigMap and IAM
490 # statements are always present on every regional cluster. Must
491 # run after
492 # _create_pod_identity_associations (which created service_account_role)
493 # and before _apply_kubernetes_manifests (which consumes the
494 # replacements in the KubectlApplyManifests CustomResource).
495 self.cluster_shared_identity = self._resolve_cluster_shared_bucket_from_ssm()
496 self._grant_cluster_shared_bucket_to_job_role(self.cluster_shared_identity)
498 # Create the always-on general-purpose regional bucket (KMS key +
499 # access-logs bucket + primary bucket). Provisioned unconditionally —
500 # there is no cdk.json toggle and no feature flag gating its existence —
501 # in addition to the central buckets owned by GCOGlobalStack.
502 self._create_regional_shared_bucket()
504 # Create EFS for shared storage
505 self._create_efs()
507 # Create FSx for Lustre (if enabled) for high-performance storage
508 self._create_fsx_lustre()
510 # Create Valkey Serverless cache (if enabled) for K/V caching
511 self._create_valkey_cache()
513 # Create Aurora Serverless v2 + pgvector (if enabled) for vector DB
514 self._create_aurora_pgvector()
516 # Discover and publish the Gateway ALB in every partition. Global
517 # Accelerator registration is an optional extension of this same exact
518 # ownership path where the service is available.
519 self._create_ga_registration_lambda()
521 # Provider framework Lambdas can emit their final delete-event log after
522 # CloudFormation has otherwise finished the custom resource. Strict live
523 # validation retains this explicit group through stack deletion so the
524 # harness can remove the same checkpointed generation after every target
525 # stack is absent. Ordinary deployments keep DESTROY semantics and do not
526 # accumulate retained groups.
527 self.helm_installer_provider_log_group = logs.LogGroup(
528 self,
529 "HelmInstallerProviderLogGroup",
530 retention=logs.RetentionDays.ONE_WEEK,
531 removal_policy=self.provider_log_group_removal_policy,
532 )
534 # Create Helm installer Lambda for KEDA and other Helm-based installations
535 self._create_helm_installer_lambda()
537 # Apply Kubernetes manifests (after EFS so IDs are available)
538 self._apply_kubernetes_manifests()
540 # Create CloudFormation drift detection (daily schedule + SNS alerts)
541 self._create_drift_detection()
543 # Create dedicated IAM role for MCP server
544 self._create_mcp_role()
546 # Export cluster information
547 self._create_outputs()
549 # Apply cdk-nag suppressions for this stack
550 self._apply_nag_suppressions()
552 def _create_vpc_flow_logs(self) -> None:
553 """Create VPC Flow Logs for network traffic monitoring.
555 Flow logs capture information about IP traffic going to and from
556 network interfaces in the VPC. This is required for security
557 monitoring and compliance (HIPAA, SOC2, etc.).
558 """
559 # Create CloudWatch Log Group for flow logs
560 flow_log_group = logs.LogGroup(
561 self,
562 "VpcFlowLogGroup",
563 # log_group_name intentionally omitted - let CDK generate unique name
564 retention=logs.RetentionDays.ONE_MONTH,
565 removal_policy=RemovalPolicy.DESTROY,
566 )
568 # Create IAM role for VPC Flow Logs
569 flow_log_role = iam.Role(
570 self,
571 "VpcFlowLogRole",
572 assumed_by=iam.ServicePrincipal("vpc-flow-logs.amazonaws.com"),
573 )
575 flow_log_role.add_to_policy(
576 iam.PolicyStatement(
577 actions=[
578 "logs:CreateLogStream",
579 "logs:PutLogEvents",
580 "logs:DescribeLogGroups",
581 "logs:DescribeLogStreams",
582 ],
583 resources=[flow_log_group.log_group_arn, f"{flow_log_group.log_group_arn}:*"],
584 )
585 )
587 # Create VPC Flow Log
588 ec2.FlowLog(
589 self,
590 "VpcFlowLog",
591 resource_type=ec2.FlowLogResourceType.from_vpc(self.vpc),
592 destination=ec2.FlowLogDestination.to_cloud_watch_logs(flow_log_group, flow_log_role),
593 traffic_type=ec2.FlowLogTrafficType.ALL,
594 )
596 def _apply_nag_suppressions(self) -> None:
597 """Apply cdk-nag suppressions for this stack."""
598 from gco.stacks.nag_suppressions import apply_all_suppressions
600 apply_all_suppressions(
601 self,
602 stack_type="regional",
603 regions=self.config.get_regions(),
604 global_region=self.config.get_global_region(),
605 api_gateway_region=self.config.get_api_gateway_region(),
606 project_name=self.config.get_project_name(),
607 )
609 def _create_sqs_queue(self) -> None:
610 """Create SQS queue for job ingestion.
612 Creates an SQS queue that serves as the default job ingestion point
613 for this region. Jobs submitted to this queue are processed by the
614 manifest processor and KEDA scales based on queue depth.
616 Also creates a dead-letter queue for failed messages.
617 Both queues use server-side encryption with AWS managed keys.
618 """
619 project_name = self.config.get_project_name()
621 # Create dead-letter queue for failed messages
622 self.job_dlq = sqs.Queue(
623 self,
624 "JobDeadLetterQueue",
625 queue_name=f"{project_name}-jobs-dlq-{self.deployment_region}",
626 retention_period=Duration.days(14),
627 removal_policy=RemovalPolicy.DESTROY,
628 enforce_ssl=True, # Require SSL for all requests
629 encryption=sqs.QueueEncryption.SQS_MANAGED, # Server-side encryption
630 )
632 # Create main job queue
633 self.job_queue = sqs.Queue(
634 self,
635 "JobQueue",
636 queue_name=f"{project_name}-jobs-{self.deployment_region}",
637 visibility_timeout=Duration.minutes(5), # Match Lambda timeout
638 retention_period=Duration.days(7),
639 dead_letter_queue=sqs.DeadLetterQueue(
640 max_receive_count=3, # Move to DLQ after 3 failed attempts
641 queue=self.job_dlq,
642 ),
643 removal_policy=RemovalPolicy.DESTROY,
644 enforce_ssl=True, # Require SSL for all requests
645 encryption=sqs.QueueEncryption.SQS_MANAGED, # Server-side encryption
646 )
648 # Output queue information
649 CfnOutput(
650 self,
651 "JobQueueUrl",
652 value=self.job_queue.queue_url,
653 description=f"SQS Job Queue URL for {self.deployment_region}",
654 export_name=f"{project_name}-job-queue-url-{self.deployment_region}",
655 )
657 CfnOutput(
658 self,
659 "JobQueueArn",
660 value=self.job_queue.queue_arn,
661 description=f"SQS Job Queue ARN for {self.deployment_region}",
662 export_name=f"{project_name}-job-queue-arn-{self.deployment_region}",
663 )
665 CfnOutput(
666 self,
667 "JobDlqUrl",
668 value=self.job_dlq.queue_url,
669 description=f"SQS Dead Letter Queue URL for {self.deployment_region}",
670 export_name=f"{project_name}-job-dlq-url-{self.deployment_region}",
671 )
673 def _create_aws_custom_resource_role(self) -> None:
674 """Pre-create the execution role shared by every ``AwsCustomResource``.
676 CDK's ``cr.AwsCustomResource`` defaults to auto-generating a per-
677 construct Lambda execution role from the ``policy=`` parameter.
678 Internally, CDK deduplicates those auto-generated roles onto a
679 single *singleton* provider Lambda (logical id prefix
680 ``AWS679f53fac002430cb0da5b7982bd22872``), and merges each custom
681 resource's policy statements onto that Lambda's role at stack
682 create time. On cold deploys, CloudFormation invokes the Lambda
683 within 2-3 seconds of attaching a new policy statement, which is
684 faster than IAM's global propagation window. The symptom is a
685 ``iam:PassRole NOT authorized`` failure on whichever addon role
686 update happens to run right after its ``iam:PassRole`` policy
687 statement was attached but before it had replicated.
689 The fix is to create the role up front, attach every policy
690 statement the stack will need during stack creation, and pass
691 ``role=self.aws_custom_resource_role`` to every
692 ``AwsCustomResource`` instead of ``policy=``. Because the role
693 already exists — and its inline policy has had minutes to
694 replicate by the time any ``AwsCustomResource`` actually fires —
695 the race disappears entirely.
697 This method creates the role with the statements we can compute
698 without a cluster reference (EKS ``UpdateAddon`` / ``DescribeAddon``
699 scoped to this cluster, and SSM ``GetParameter`` for the endpoint
700 group ARN). ``iam:PassRole`` statements for individual addon
701 roles (EFS CSI, FSx CSI, CloudWatch Observability) are appended
702 by each ``_create_*_addon`` method after the corresponding IRSA
703 role has been created, so every PassRole ``resources=`` list
704 stays precise (no wildcards) and cdk-nag stays happy.
705 """
706 project_name = self.config.get_project_name()
707 global_region = self.config.get_global_region()
709 self.aws_custom_resource_role = iam.Role(
710 self,
711 "AwsCustomResourceRole",
712 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
713 description=(
714 "Shared execution role for every cr.AwsCustomResource in this "
715 "stack. Pre-created to avoid the IAM policy propagation race "
716 "that occurs when CDK auto-generates per-CR roles and the "
717 "singleton provider Lambda fires before the freshly-attached "
718 "policy has replicated globally."
719 ),
720 managed_policies=[
721 iam.ManagedPolicy.from_aws_managed_policy_name(
722 "service-role/AWSLambdaBasicExecutionRole"
723 ),
724 ],
725 )
727 # EKS UpdateAddon / DescribeAddon — used by the three updateAddon
728 # custom resources (EFS CSI, FSx CSI, CloudWatch Observability).
729 # Scoped to this cluster's addons by ARN.
730 self.aws_custom_resource_role.add_to_policy(
731 iam.PolicyStatement(
732 effect=iam.Effect.ALLOW,
733 actions=["eks:UpdateAddon", "eks:DescribeAddon"],
734 resources=[
735 f"arn:{self.partition}:eks:{self.deployment_region}:{self.account}"
736 f":addon/{self.cluster_config.cluster_name}/*"
737 ],
738 )
739 )
741 # SSM GetParameter — used by the GetEndpointGroupArn custom
742 # resource in _create_ga_registration_lambda to read the ARN of
743 # the Global Accelerator endpoint group published by the global
744 # stack during its deploy.
745 self.aws_custom_resource_role.add_to_policy(
746 iam.PolicyStatement(
747 effect=iam.Effect.ALLOW,
748 actions=["ssm:GetParameter"],
749 resources=[
750 f"arn:{self.partition}:ssm:{global_region}:{self.account}:"
751 f"parameter/{project_name}/*"
752 ],
753 )
754 )
756 # cdk-nag suppressions: the two wildcard-bearing ARNs above are
757 # intentional and both scoped as tightly as AWS IAM permits.
758 #
759 # - The ``eks:UpdateAddon`` / ``eks:DescribeAddon`` statement uses
760 # ``addon/<cluster>/*`` as its resource because the same shared
761 # role is consumed by three different updateAddon custom
762 # resources (EFS CSI, FSx CSI, CloudWatch Observability). Each
763 # addon has its own ARN and we'd otherwise need three separate
764 # statements that each grant access to a known addon name. The
765 # wildcard is scoped to a single cluster in a single region in
766 # a single account — it cannot be used against any addon
767 # belonging to a different cluster or a different service.
768 #
769 # - The ``ssm:GetParameter`` statement uses
770 # ``parameter/<project>/*`` because the exact parameter name
771 # (``endpoint-group-<region>-arn``) is only known at Global
772 # Accelerator registration time and the endpoint path
773 # structure is ``<project>/<parameter>``. Scoping to the
774 # project prefix restricts access to parameters owned by this
775 # project only.
776 from gco.stacks.nag_suppressions import acknowledge_nag_findings
778 acknowledge_nag_findings(
779 self.aws_custom_resource_role,
780 [
781 {
782 "id": "AwsSolutions-IAM5",
783 "reason": (
784 "Scoped to a single EKS cluster's addons "
785 "(addon/<cluster>/*) and this project's SSM "
786 "parameters (parameter/<project>/*). Both wildcards "
787 "are as tight as AWS IAM permits: addon names and "
788 "parameter names are not known at stack synthesis "
789 "time because the addons are created later in the "
790 "same stack and the GA endpoint group ARN is "
791 "published by a separate stack during deploy. The "
792 "shared role pattern itself is deliberate — see "
793 "_create_aws_custom_resource_role docstring for why "
794 "we pre-create instead of letting CDK auto-generate "
795 "per-CR roles."
796 ),
797 "appliesTo": [
798 f"Resource::arn:<AWS::Partition>:eks:{self.deployment_region}"
799 f":<AWS::AccountId>:addon/{self.cluster_config.cluster_name}/*",
800 f"Resource::arn:<AWS::Partition>:ssm:{global_region}"
801 f":<AWS::AccountId>:parameter/{project_name}/*",
802 ],
803 },
804 ],
805 )
807 def _resolve_backend_tls_certificate_arn(self) -> str:
808 """Read this region's stable imported ACM ARN from global-region SSM.
810 The certificate manager publishes one fixed ARN per workload region.
811 The regional Ingress consumes the token directly, which creates a
812 CloudFormation dependency ensuring the certificate exists before the
813 HTTPS listener is reconciled. The shared custom-resource role is
814 already restricted to this project's SSM namespace.
815 """
816 project_name = self.config.get_project_name()
817 parameter_name = backend_tls_certificate_arn_parameter_name(
818 project_name, self.deployment_region
819 )
820 reader = cr.AwsCustomResource(
821 self,
822 "GetBackendTlsCertificateArn",
823 on_create=cr.AwsSdkCall(
824 service="SSM",
825 action="getParameter",
826 parameters={"Name": parameter_name},
827 region=self.config.get_global_region(),
828 physical_resource_id=cr.PhysicalResourceId.of(
829 f"{project_name}-backend-tls-certificate-{self.deployment_region}"
830 ),
831 ),
832 on_update=cr.AwsSdkCall(
833 service="SSM",
834 action="getParameter",
835 parameters={"Name": parameter_name},
836 region=self.config.get_global_region(),
837 ),
838 role=self.aws_custom_resource_role,
839 )
840 reader.node.add_dependency(self.aws_custom_resource_role)
841 return str(reader.get_response_field("Parameter.Value"))
843 def _create_container_images(self) -> None:
844 """Create ECR repositories and build Docker images for services"""
846 # Create ECR repository for health monitor
847 self.health_monitor_repo = ecr.Repository(
848 self,
849 "HealthMonitorRepo",
850 # repository_name intentionally omitted - let CDK generate unique name
851 removal_policy=RemovalPolicy.DESTROY, # For dev/test; use RETAIN for production
852 empty_on_delete=True, # Clean up images on stack deletion
853 image_scan_on_push=True, # Enable vulnerability scanning on push
854 )
856 # All Docker images target AMD64 (x86_64) to match EKS Auto Mode's
857 # default system nodepool.
859 # Build and push health monitor Docker image
860 self.health_monitor_image = ecr_assets.DockerImageAsset(
861 self,
862 "HealthMonitorImage",
863 directory=".", # Root directory
864 file="dockerfiles/health-monitor-dockerfile",
865 platform=ecr_assets.Platform.LINUX_AMD64,
866 exclude=_service_image_asset_excludes(
867 "dockerfiles/health-monitor-dockerfile",
868 ),
869 )
871 # Create ECR repository for manifest processor
872 self.manifest_processor_repo = ecr.Repository(
873 self,
874 "ManifestProcessorRepo",
875 # repository_name intentionally omitted - let CDK generate unique name
876 removal_policy=RemovalPolicy.DESTROY,
877 empty_on_delete=True,
878 image_scan_on_push=True, # Enable vulnerability scanning on push
879 )
881 # Build and push manifest processor Docker image
882 self.manifest_processor_image = ecr_assets.DockerImageAsset(
883 self,
884 "ManifestProcessorImage",
885 directory=".",
886 file="dockerfiles/manifest-processor-dockerfile",
887 platform=ecr_assets.Platform.LINUX_AMD64,
888 exclude=_service_image_asset_excludes(
889 "dockerfiles/manifest-processor-dockerfile",
890 ),
891 )
893 # Create and build the inference-only data-plane proxy image. Keeping
894 # this separate from manifest-processor prevents model traffic from
895 # sharing its Kubernetes API/RBAC and queue-worker process surface.
896 self.inference_proxy_repo = ecr.Repository(
897 self,
898 "InferenceProxyRepo",
899 removal_policy=RemovalPolicy.DESTROY,
900 empty_on_delete=True,
901 image_scan_on_push=True,
902 )
903 self.inference_proxy_image = ecr_assets.DockerImageAsset(
904 self,
905 "InferenceProxyImage",
906 directory=".",
907 file="dockerfiles/inference-proxy-dockerfile",
908 platform=ecr_assets.Platform.LINUX_AMD64,
909 exclude=_service_image_asset_excludes(
910 "dockerfiles/inference-proxy-dockerfile",
911 ),
912 )
914 # Output image URIs for reference
915 CfnOutput(
916 self,
917 "HealthMonitorImageUri",
918 value=self.health_monitor_image.image_uri,
919 description="Health Monitor Docker image URI",
920 )
922 CfnOutput(
923 self,
924 "ManifestProcessorImageUri",
925 value=self.manifest_processor_image.image_uri,
926 description="Manifest Processor Docker image URI",
927 )
929 CfnOutput(
930 self,
931 "InferenceProxyImageUri",
932 value=self.inference_proxy_image.image_uri,
933 description="Inference Proxy Docker image URI",
934 )
936 # Build and push inference monitor Docker image
937 self.inference_monitor_image = ecr_assets.DockerImageAsset(
938 self,
939 "InferenceMonitorImage",
940 directory=".",
941 file="dockerfiles/inference-monitor-dockerfile",
942 platform=ecr_assets.Platform.LINUX_AMD64,
943 exclude=_service_image_asset_excludes(
944 "dockerfiles/inference-monitor-dockerfile",
945 ),
946 )
948 CfnOutput(
949 self,
950 "InferenceMonitorImageUri",
951 value=self.inference_monitor_image.image_uri,
952 description="Inference Monitor Docker image URI",
953 )
955 # Build and push queue processor Docker image (if enabled).
956 # The queue processor is a KEDA ScaledJob that consumes manifests from
957 # the regional SQS queue. It can be disabled in cdk.json if users want
958 # to implement their own consumer. When disabled, the post-helm-sqs-consumer.yaml
959 # manifest is skipped (unreplaced template variables cause it to be skipped).
960 queue_processor_config = self.node.try_get_context("queue_processor") or {}
961 self.queue_processor_enabled = queue_processor_config.get("enabled", True)
963 if self.queue_processor_enabled:
964 self.queue_processor_image = ecr_assets.DockerImageAsset(
965 self,
966 "QueueProcessorImage",
967 directory=".",
968 file="dockerfiles/queue-processor-dockerfile",
969 platform=ecr_assets.Platform.LINUX_AMD64,
970 exclude=_service_image_asset_excludes(
971 "dockerfiles/queue-processor-dockerfile",
972 ),
973 )
975 CfnOutput(
976 self,
977 "QueueProcessorImageUri",
978 value=self.queue_processor_image.image_uri,
979 description="Queue Processor Docker image URI",
980 )
982 # Build and push the cost-monitor image only when the cost monitoring
983 # pipeline deploys to this region — skipping the build keeps opted-out
984 # deployments' synth/deploy time unchanged (same gating rationale as
985 # the queue processor above).
986 if self._cost_monitoring_active(): 986 ↛ exitline 986 didn't return from function '_create_container_images' because the condition on line 986 was always true
987 self.cost_monitor_image = ecr_assets.DockerImageAsset(
988 self,
989 "CostMonitorImage",
990 directory=".",
991 file="dockerfiles/cost-monitor-dockerfile",
992 platform=ecr_assets.Platform.LINUX_AMD64,
993 exclude=_service_image_asset_excludes(
994 "dockerfiles/cost-monitor-dockerfile",
995 ),
996 )
998 CfnOutput(
999 self,
1000 "CostMonitorImageUri",
1001 value=self.cost_monitor_image.image_uri,
1002 description="Cost Monitor Docker image URI",
1003 )
1005 def _resolve_unsupported_az_names(self) -> list[str]:
1006 """Resolve this region's EKS-unsupported AZ *IDs* to this account's AZ *names*.
1008 EKS rejects cluster subnets in a small set of Availability Zones,
1009 published by AZ ID (``EKS_UNSUPPORTED_AZ_IDS``). AZ *names* are
1010 randomized per account, so the disallowed ``use1-az3`` may be
1011 ``us-east-1e`` in one account and a different name in another — we must
1012 map ID -> name for the deploy account.
1014 Returns an empty list when the region has no restriction (the common
1015 case) or when the deploy account is not resolved. A credentialed,
1016 environment-specific synth fails closed if EC2 cannot resolve every
1017 restricted AZ ID; selecting all private subnets in that case could hand
1018 EKS a known-unsupported control-plane subnet.
1019 """
1020 unsupported_ids = EKS_UNSUPPORTED_AZ_IDS.get(self.deployment_region, ())
1021 if not unsupported_ids:
1022 return []
1023 # Only reach EC2 during a credentialed, environment-specific synth or
1024 # deploy. The CDK CLI exports CDK_DEFAULT_ACCOUNT from the active
1025 # identity; unit tests and agnostic synth don't, so we never call AWS
1026 # (nor block synthesis on missing credentials) there.
1027 if not os.environ.get("CDK_DEFAULT_ACCOUNT"):
1028 return []
1029 try:
1030 import boto3
1031 from botocore.config import Config
1033 ec2_client = boto3.client(
1034 "ec2",
1035 region_name=self.deployment_region,
1036 config=Config(connect_timeout=5, read_timeout=5, retries={"max_attempts": 2}),
1037 )
1038 response = ec2_client.describe_availability_zones(
1039 Filters=[{"Name": "zone-id", "Values": list(unsupported_ids)}]
1040 )
1041 except Exception as exc:
1042 raise RuntimeError(
1043 f"Unable to resolve EKS-unsupported Availability Zones in "
1044 f"{self.deployment_region}: {exc}"
1045 ) from exc
1047 zones = response.get("AvailabilityZones")
1048 if not isinstance(zones, list): 1048 ↛ 1049line 1048 didn't jump to line 1049 because the condition on line 1048 was never true
1049 raise RuntimeError(
1050 f"EC2 returned malformed Availability Zone data for {self.deployment_region}"
1051 )
1052 names_by_id = {
1053 zone_id: zone_name
1054 for zone in zones
1055 if isinstance(zone, dict)
1056 and isinstance((zone_id := zone.get("ZoneId")), str)
1057 and isinstance((zone_name := zone.get("ZoneName")), str)
1058 and zone_id in unsupported_ids
1059 and zone_name
1060 }
1061 missing_ids = [zone_id for zone_id in unsupported_ids if zone_id not in names_by_id]
1062 if missing_ids: 1062 ↛ 1063line 1062 didn't jump to line 1063 because the condition on line 1062 was never true
1063 raise RuntimeError(
1064 f"EC2 did not resolve EKS-unsupported Availability Zone IDs in "
1065 f"{self.deployment_region}: {', '.join(missing_ids)}"
1066 )
1067 return [names_by_id[zone_id] for zone_id in unsupported_ids]
1069 def _eks_control_plane_subnets(self) -> ec2.SubnetSelection:
1070 """Private-subnet selection for the EKS control plane, excluding any AZ
1071 EKS does not support for cluster subnets.
1073 Records the outcome on ``self`` (``eks_unsupported_az_names`` and
1074 ``eks_control_plane_subnets``) so tests and operators can introspect
1075 exactly which subnets the cluster was given.
1076 """
1077 unsupported = set(self._resolve_unsupported_az_names())
1078 usable = [
1079 subnet
1080 for subnet in self.vpc.private_subnets
1081 if subnet.availability_zone not in unsupported
1082 ]
1083 self.eks_unsupported_az_names = sorted(unsupported)
1084 self.eks_control_plane_subnets = usable
1085 if not unsupported:
1086 # No restricted AZ in this region: keep the subnet-type selection so
1087 # the synthesized template is identical to before for the common case.
1088 return ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)
1089 return ec2.SubnetSelection(subnets=usable)
1091 def _create_eks_cluster(self, cluster_config: Any) -> None:
1092 """Create the EKS cluster with auto mode and GPU node groups"""
1094 # Create cluster admin role
1095 # role_name intentionally omitted - let CDK generate unique name
1096 cluster_admin_role = iam.Role(
1097 self,
1098 "ClusterAdminRole",
1099 assumed_by=iam.ServicePrincipal("eks.amazonaws.com"),
1100 managed_policies=[
1101 iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEKSClusterPolicy")
1102 ],
1103 )
1105 # Create node group role
1106 # role_name intentionally omitted - let CDK generate unique name
1107 iam.Role(
1108 self,
1109 "NodeGroupRole",
1110 assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
1111 managed_policies=[
1112 iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEKSWorkerNodePolicy"),
1113 iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEKS_CNI_Policy"),
1114 iam.ManagedPolicy.from_aws_managed_policy_name(
1115 "AmazonEC2ContainerRegistryReadOnly"
1116 ),
1117 ],
1118 )
1120 # Create EKS Auto Mode cluster with built-in system and general-purpose nodepools
1121 # Auto Mode automatically manages compute resources and comes with essential addons
1122 # Get endpoint access configuration
1123 eks_config = self.config.get_eks_cluster_config()
1124 endpoint_access_mode = eks_config.get("endpoint_access", "PRIVATE")
1126 # Map config string to EKS EndpointAccess enum
1127 endpoint_access = (
1128 eks.EndpointAccess.PRIVATE
1129 if endpoint_access_mode == "PRIVATE"
1130 else eks.EndpointAccess.PUBLIC_AND_PRIVATE
1131 )
1133 # Create KMS key for EKS secrets encryption
1134 self.eks_encryption_key = kms.Key(
1135 self,
1136 "EksSecretsEncryptionKey",
1137 description="KMS key for EKS Kubernetes secrets encryption",
1138 enable_key_rotation=True,
1139 removal_policy=RemovalPolicy.RETAIN,
1140 )
1142 # Get Kubernetes version - use custom version if not available in CDK enum
1143 k8s_version_str = cluster_config.kubernetes_version
1144 try:
1145 k8s_version = getattr(eks.KubernetesVersion, f"V{k8s_version_str.replace('.', '_')}")
1146 except AttributeError:
1147 # Version not in CDK enum yet, use custom version
1148 k8s_version = eks.KubernetesVersion.of(k8s_version_str)
1150 self.cluster = eks.Cluster(
1151 self,
1152 "GCOEksCluster",
1153 cluster_name=cluster_config.cluster_name,
1154 version=k8s_version, # Use configured version for Auto Mode with DRA support
1155 vpc=self.vpc,
1156 compute=eks.ComputeConfig(
1157 # Enable both built-in node pools - Auto Mode manages these automatically
1158 node_pools=["system", "general-purpose"]
1159 ),
1160 # SECURITY: Endpoint access controlled via cdk.json eks_cluster.endpoint_access
1161 # PRIVATE (default): EKS API accessible only from within VPC - most secure
1162 # Job submission works via API Gateway → Lambda (in VPC) or SQS
1163 # For kubectl access, use a bastion host, VPN, or AWS SSM Session Manager
1164 # PUBLIC_AND_PRIVATE: EKS API accessible from internet and VPC
1165 # Allows direct kubectl access but less secure
1166 endpoint_access=endpoint_access,
1167 role=cluster_admin_role,
1168 # The VPC spans every AZ, but EKS refuses control-plane subnets in a
1169 # few AZs (by stable AZ ID; see EKS_UNSUPPORTED_AZ_IDS). Select the
1170 # private subnets in supported AZs only — worker/other subnets in the
1171 # excluded AZs still exist in the VPC.
1172 vpc_subnets=[self._eks_control_plane_subnets()],
1173 # Enable all control plane logging for security and compliance
1174 cluster_logging=[
1175 eks.ClusterLoggingTypes.API,
1176 eks.ClusterLoggingTypes.AUDIT,
1177 eks.ClusterLoggingTypes.AUTHENTICATOR,
1178 eks.ClusterLoggingTypes.CONTROLLER_MANAGER,
1179 eks.ClusterLoggingTypes.SCHEDULER,
1180 ],
1181 # SECURITY: Enable envelope encryption for Kubernetes secrets using KMS
1182 secrets_encryption_key=self.eks_encryption_key,
1183 )
1185 # The EKS cluster security group's auto-generated ingress rule allows
1186 # 443 from the VPC CIDR, expressed as a CloudFormation token. cdk-nag's
1187 # SG-ingress rules can't resolve the token and throw; scope the
1188 # acknowledgment to the cluster construct so it can't mask an
1189 # open-ingress finding elsewhere in the stack.
1190 from gco.stacks.nag_suppressions import acknowledge_security_group_cidr_findings
1192 acknowledge_security_group_cidr_findings(
1193 self.cluster,
1194 reason=(
1195 "The EKS Auto Mode cluster security group allows HTTPS (443) "
1196 "ingress from the VPC CIDR only, referenced via an "
1197 "``Fn::GetAtt`` token that cdk-nag cannot resolve at synth "
1198 "time. Ingress is restricted to intra-VPC traffic, the "
1199 "tightest possible source for the Kubernetes API and webhook "
1200 "endpoints."
1201 ),
1202 )
1204 # Auto Mode comes with essential addons pre-configured:
1205 # - AWS Load Balancer Controller (for ALB/NLB integration)
1206 # - CoreDNS, kube-proxy, VPC CNI (standard Kubernetes components)
1208 # OIDC provider for IRSA — the primary credential injection mechanism.
1209 # IRSA uses projected service-account tokens exchanged via the OIDC provider
1210 # for temporary AWS credentials. This works reliably on EKS Auto Mode.
1211 self.oidc_provider = eks.OidcProviderNative(
1212 self,
1213 "OidcProvider",
1214 url=self.cluster.cluster_open_id_connect_issuer_url,
1215 )
1217 # Pod Identity Agent add-on — registers the admission webhook that injects
1218 # Pod Identity credentials. On Auto Mode the DaemonSet schedules 0 pods
1219 # (the agent is built into the node), but the add-on registration is still
1220 # needed for the control-plane webhook. Kept as a secondary credential path.
1221 self._create_pod_identity_agent_addon()
1223 # Add Metrics Server add-on for HPA and resource monitoring
1224 self._create_metrics_server_addon()
1226 # Add EFS CSI Driver add-on for shared storage
1227 self._create_efs_csi_driver_addon()
1229 # Add CloudWatch Observability add-on for Container Insights metrics
1230 self._create_cloudwatch_observability_addon()
1232 # NOTE: GPU compute is configured via Karpenter NodePools (not managed node groups)
1233 # NodePool manifests are located in lambda/kubectl-applier-simple/manifests/:
1234 # - 40-nodepool-gpu-x86.yaml: active x86_64 general GPU instances (g4dn, g5, g6/g6e/g6f/gr6/gr6f, g7/g7e; deprecated p3/p3dn excluded)
1235 # - 41-nodepool-gpu-arm.yaml: ARM64 GPU instances (g5g)
1236 # - 42-nodepool-inference.yaml: inference-optimized GPU instances
1237 # - 43-nodepool-efa.yaml: EFA-enabled instances (p4d, p4de, p5/p5e/p5en, p6-b200/p6-b300/p6e-gb200)
1238 # - 44-nodepool-neuron.yaml: Trainium/Inferentia instances
1239 # These will be applied by the kubectl Lambda custom resource (created below)
1241 # Create IRSA role for service account to access secrets
1242 self._create_service_account_role()
1244 # Create kubectl Lambda for applying Kubernetes manifests
1245 self._create_kubectl_lambda()
1247 # ── Shared toleration config for EKS add-ons ──────────────────────────
1248 # All GCO nodepools apply taints (nvidia.com/gpu, aws.amazon.com/neuron,
1249 # vpc.amazonaws.com/efa) that prevent DaemonSet pods from scheduling.
1250 # Every add-on that runs a DaemonSet (or may schedule on tainted nodes)
1251 # must tolerate these taints so that storage drivers, metrics agents, and
1252 # other infrastructure components work on every node type.
1253 _ADDON_NODE_TOLERATIONS = [
1254 {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"},
1255 {"key": "aws.amazon.com/neuron", "operator": "Exists", "effect": "NoSchedule"},
1256 {"key": "vpc.amazonaws.com/efa", "operator": "Exists", "effect": "NoSchedule"},
1257 ]
1259 def _create_pod_identity_agent_addon(self) -> None:
1260 """Create EKS Pod Identity Agent add-on.
1262 On Auto Mode the DaemonSet schedules 0 pods (the agent is built into
1263 the node runtime), but the add-on registration is still required for
1264 the control-plane admission webhook that injects Pod Identity tokens.
1265 """
1266 eks.Addon(
1267 self,
1268 "PodIdentityAgentAddon",
1269 cluster=self.cluster, # type: ignore[arg-type]
1270 addon_name="eks-pod-identity-agent",
1271 addon_version=EKS_ADDON_POD_IDENTITY_AGENT,
1272 preserve_on_delete=False,
1273 configuration_values={
1274 "tolerations": self._ADDON_NODE_TOLERATIONS,
1275 },
1276 )
1278 def _create_metrics_server_addon(self) -> None:
1279 """Create Metrics Server add-on for resource metrics.
1281 The Metrics Server collects resource metrics from kubelets and exposes
1282 them via the Kubernetes API server. This is required for:
1283 - Horizontal Pod Autoscaler (HPA)
1284 - Vertical Pod Autoscaler (VPA)
1285 - kubectl top commands
1286 - Resource monitoring dashboards
1288 Note: Metrics Server doesn't require an IRSA role as it only needs
1289 in-cluster permissions which are handled by its service account.
1290 """
1291 eks.Addon(
1292 self,
1293 "MetricsServerAddon",
1294 cluster=self.cluster, # type: ignore[arg-type]
1295 addon_name="metrics-server",
1296 addon_version=EKS_ADDON_METRICS_SERVER,
1297 preserve_on_delete=False,
1298 configuration_values={
1299 "tolerations": self._ADDON_NODE_TOLERATIONS,
1300 },
1301 )
1303 def _create_efs_csi_driver_addon(self) -> None:
1304 """Create EFS CSI Driver add-on for shared storage support.
1306 The EFS CSI driver enables Kubernetes pods to mount EFS file systems
1307 as persistent volumes. This is required for the shared storage feature.
1309 We create a Pod Identity role for the EFS CSI driver and update the add-on
1310 to use it via a custom resource after the add-on is created.
1311 """
1312 # Create IAM role for EFS CSI Driver using IRSA + Pod Identity
1313 self.efs_csi_role = GCORegionalStack._create_irsa_role(
1314 self,
1315 "EfsCsiDriverRole",
1316 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1317 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1318 service_account_names=["efs-csi-controller-sa"],
1319 namespaces=["kube-system"],
1320 )
1322 # Add EFS CSI driver permissions
1323 self.efs_csi_role.add_managed_policy(
1324 iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AmazonEFSCSIDriverPolicy")
1325 )
1327 # Create EFS CSI Driver add-on
1328 efs_addon = eks.Addon(
1329 self,
1330 "EfsCsiDriverAddon",
1331 cluster=self.cluster, # type: ignore[arg-type]
1332 addon_name="aws-efs-csi-driver",
1333 addon_version=EKS_ADDON_EFS_CSI_DRIVER,
1334 preserve_on_delete=False,
1335 configuration_values={
1336 "node": {
1337 "tolerations": self._ADDON_NODE_TOLERATIONS,
1338 },
1339 "controller": {
1340 "tolerations": self._ADDON_NODE_TOLERATIONS,
1341 },
1342 },
1343 )
1345 # Append the PassRole statement for the EFS CSI role to the shared
1346 # AwsCustomResource execution role. See the role's creation in
1347 # _create_aws_custom_resource_role for the full rationale on why
1348 # we pre-create + attach up-front instead of letting CDK
1349 # auto-generate per-CR roles.
1350 self.aws_custom_resource_role.add_to_policy(
1351 iam.PolicyStatement(
1352 effect=iam.Effect.ALLOW,
1353 actions=["iam:PassRole"],
1354 resources=[self.efs_csi_role.role_arn],
1355 )
1356 )
1358 # Update the add-on to use the IRSA role via custom resource
1359 # This is needed because the eks v2 alpha Addon doesn't support service_account_role directly
1360 update_addon = cr.AwsCustomResource(
1361 self,
1362 "UpdateEfsCsiAddonRole",
1363 on_create=cr.AwsSdkCall(
1364 service="EKS",
1365 action="updateAddon",
1366 parameters={
1367 "clusterName": self.cluster.cluster_name,
1368 "addonName": "aws-efs-csi-driver",
1369 "serviceAccountRoleArn": self.efs_csi_role.role_arn,
1370 },
1371 physical_resource_id=cr.PhysicalResourceId.of(
1372 f"{self.cluster.cluster_name}-efs-csi-role-update"
1373 ),
1374 ),
1375 on_update=cr.AwsSdkCall(
1376 service="EKS",
1377 action="updateAddon",
1378 parameters={
1379 "clusterName": self.cluster.cluster_name,
1380 "addonName": "aws-efs-csi-driver",
1381 "serviceAccountRoleArn": self.efs_csi_role.role_arn,
1382 },
1383 ),
1384 role=self.aws_custom_resource_role,
1385 )
1387 # Ensure the update happens after the add-on is created. We also
1388 # depend on the shared execution role so CloudFormation has fully
1389 # attached + replicated its inline policy before the Lambda fires.
1390 update_addon.node.add_dependency(efs_addon)
1391 update_addon.node.add_dependency(self.efs_csi_role)
1392 update_addon.node.add_dependency(self.aws_custom_resource_role)
1394 # Expose the update-addon resource so _apply_kubernetes_manifests can
1395 # make the kubectl Lambda wait for the IRSA annotation patch to land
1396 # before it tries to rollout-restart the efs-csi-controller. Without
1397 # this ordering, the restart could fire before EKS has re-attached
1398 # the role ARN, leaving the new pods just as credential-less as the
1399 # old ones and causing every EFS CreateAccessPoint to fail with a
1400 # 401 from IMDS.
1401 self._efs_csi_addon_role_update = update_addon
1403 def _create_cloudwatch_observability_addon(self) -> None:
1404 """Create CloudWatch Observability add-on for Container Insights.
1406 The CloudWatch Observability add-on enables Container Insights metrics
1407 for the EKS cluster, providing visibility into:
1408 - Cluster CPU and memory utilization
1409 - Node-level metrics
1410 - Pod and container metrics
1411 - Application logs (optional)
1413 These metrics are used by the monitoring dashboard to display
1414 cluster health and resource utilization.
1415 """
1417 # Create IAM role for CloudWatch agent using IRSA + Pod Identity
1418 self.cloudwatch_role = GCORegionalStack._create_irsa_role(
1419 self,
1420 "CloudWatchObservabilityRole",
1421 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1422 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1423 service_account_names=["cloudwatch-agent"],
1424 namespaces=["amazon-cloudwatch"],
1425 )
1427 # Add CloudWatch agent permissions
1428 self.cloudwatch_role.add_managed_policy(
1429 iam.ManagedPolicy.from_aws_managed_policy_name("CloudWatchAgentServerPolicy")
1430 )
1431 self.cloudwatch_role.add_managed_policy(
1432 iam.ManagedPolicy.from_aws_managed_policy_name("AWSXrayWriteOnlyAccess")
1433 )
1435 # Create CloudWatch Observability add-on
1436 cw_addon = eks.Addon(
1437 self,
1438 "CloudWatchObservabilityAddon",
1439 cluster=self.cluster, # type: ignore[arg-type]
1440 addon_name="amazon-cloudwatch-observability",
1441 addon_version=EKS_ADDON_CLOUDWATCH_OBSERVABILITY,
1442 preserve_on_delete=False,
1443 configuration_values={
1444 "tolerations": self._ADDON_NODE_TOLERATIONS,
1445 # Enable Container Insights with application log collection
1446 # Logs are sent to /aws/containerinsights/{cluster}/application
1447 "containerLogs": {
1448 "enabled": True,
1449 },
1450 },
1451 )
1453 # Append the PassRole statement for the CloudWatch Observability
1454 # role to the shared AwsCustomResource execution role. See
1455 # _create_aws_custom_resource_role for the full rationale.
1456 self.aws_custom_resource_role.add_to_policy(
1457 iam.PolicyStatement(
1458 effect=iam.Effect.ALLOW,
1459 actions=["iam:PassRole"],
1460 resources=[self.cloudwatch_role.role_arn],
1461 )
1462 )
1464 # Update the add-on to use the IRSA role via custom resource
1465 update_cw_addon = cr.AwsCustomResource(
1466 self,
1467 "UpdateCloudWatchAddonRole",
1468 on_create=cr.AwsSdkCall(
1469 service="EKS",
1470 action="updateAddon",
1471 parameters={
1472 "clusterName": self.cluster.cluster_name,
1473 "addonName": "amazon-cloudwatch-observability",
1474 "serviceAccountRoleArn": self.cloudwatch_role.role_arn,
1475 },
1476 physical_resource_id=cr.PhysicalResourceId.of(
1477 f"{self.cluster.cluster_name}-cw-obs-role-update"
1478 ),
1479 ),
1480 on_update=cr.AwsSdkCall(
1481 service="EKS",
1482 action="updateAddon",
1483 parameters={
1484 "clusterName": self.cluster.cluster_name,
1485 "addonName": "amazon-cloudwatch-observability",
1486 "serviceAccountRoleArn": self.cloudwatch_role.role_arn,
1487 },
1488 ),
1489 role=self.aws_custom_resource_role,
1490 )
1492 # Ensure the update happens after the add-on is created. Depend on
1493 # the shared execution role so CFN has fully attached + replicated
1494 # its inline policy before the Lambda fires. No CR→CR dependency
1495 # chain needed anymore — the race it was serializing against is
1496 # eliminated by pre-creating the role.
1497 update_cw_addon.node.add_dependency(cw_addon)
1498 update_cw_addon.node.add_dependency(self.cloudwatch_role)
1499 update_cw_addon.node.add_dependency(self.aws_custom_resource_role)
1501 # Expose the update-addon resource so _apply_kubernetes_manifests can
1502 # make the kubectl Lambda wait for the IRSA annotation patch to land
1503 # before it rollout-restarts the cloudwatch-agent DaemonSet. See the
1504 # EFS CSI equivalent for the full rationale — same race, same fix.
1505 self._cloudwatch_addon_role_update = update_cw_addon
1507 def _create_service_account_role(self) -> None:
1508 """Create IAM role for Kubernetes service account using EKS Pod Identity.
1510 Pod Identity is the recommended mechanism for EKS Auto Mode. It's simpler
1511 and more reliable than IRSA — no OIDC provider, no webhook injection, no
1512 projected tokens. EKS manages the credential injection automatically.
1514 The general workload role is deliberately separate from the manifest
1515 processor role. Job and inference workload service accounts must never
1516 receive queue-table mutation privileges; only the platform API/worker
1517 identity can claim, fence, or transition centralized queue records.
1518 """
1519 self.service_account_role = GCORegionalStack._create_irsa_role(
1520 self,
1521 "ServiceAccountRole",
1522 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1523 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1524 service_account_names=[
1525 "gco-service-account",
1526 "gco-inference-monitor-sa",
1527 ],
1528 namespaces=["gco-system", "gco-jobs", "gco-inference"],
1529 )
1531 self.manifest_processor_role = GCORegionalStack._create_irsa_role(
1532 self,
1533 "ManifestProcessorRole",
1534 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1535 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1536 service_account_names=["gco-manifest-processor-sa"],
1537 namespaces=["gco-system"],
1538 )
1540 self.inference_proxy_role = GCORegionalStack._create_irsa_role(
1541 self,
1542 "InferenceProxyRole",
1543 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1544 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1545 service_account_names=["gco-inference-proxy-sa"],
1546 namespaces=["gco-system"],
1547 )
1549 self.health_monitor_role = GCORegionalStack._create_irsa_role(
1550 self,
1551 "HealthMonitorRole",
1552 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1553 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1554 service_account_names=["gco-health-monitor-sa"],
1555 namespaces=["gco-system"],
1556 )
1558 if self._cost_monitoring_active(): 1558 ↛ 1569line 1558 didn't jump to line 1569 because the condition on line 1558 was always true
1559 self.cost_monitor_role = GCORegionalStack._create_irsa_role(
1560 self,
1561 "CostMonitorRole",
1562 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1563 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1564 service_account_names=["gco-cost-monitor-sa"],
1565 namespaces=["gco-system"],
1566 )
1567 self._grant_cost_report_bucket_to_cost_monitor()
1569 self._create_aws_load_balancer_controller_role()
1571 # Grant permission to read the auth secret.
1572 #
1573 # The resource is built as a *deterministic* ARN from the known secret
1574 # name, the API Gateway region (where the secret lives), and this
1575 # stack's account — rather than from ``self.auth_secret_arn``, which is
1576 # ``api_gateway_stack.secret.secret_arn`` (a cross-stack reference
1577 # token). The token was the source of issue #125: it renders
1578 # differently depending on topology —
1579 # * cross-region -> a literal ARN, and
1580 # * same-region -> a native cross-stack export
1581 # (``gco-api-gateway:ExportsOutputRefGCOAuthSecret<hash>``)
1582 # so the trailing-``*`` IAM resource only matched the stack-level
1583 # AwsSolutions-IAM5 suppression in the cross-region (default) topology.
1584 # Collapsing every stack into one region left the export-token form
1585 # unsuppressed and failed ``cdk synth``. Building the ARN ourselves
1586 # makes the ``Resource`` render identically in both topologies so a
1587 # single deterministic suppression (see ``add_iam_suppressions``)
1588 # always matches.
1589 #
1590 # The trailing ``*`` matches the random 6-character suffix Secrets
1591 # Manager appends to secret ARNs (Secrets Manager accepts either the
1592 # full ARN with suffix or the partial ARN without it).
1593 auth_secret_resource = (
1594 f"arn:{self.partition}:secretsmanager:{self.config.get_api_gateway_region()}"
1595 f":{self.account}:secret:{api_gateway_auth_secret_name(self.config.get_project_name())}*"
1596 )
1597 self.manifest_processor_role.add_to_policy(
1598 iam.PolicyStatement(
1599 effect=iam.Effect.ALLOW,
1600 actions=[
1601 "secretsmanager:GetSecretValue",
1602 "secretsmanager:DescribeSecret",
1603 ],
1604 resources=[auth_secret_resource],
1605 )
1606 )
1607 self.inference_proxy_role.add_to_policy(
1608 iam.PolicyStatement(
1609 effect=iam.Effect.ALLOW,
1610 actions=[
1611 "secretsmanager:GetSecretValue",
1612 "secretsmanager:DescribeSecret",
1613 ],
1614 resources=[auth_secret_resource],
1615 )
1616 )
1617 self.health_monitor_role.add_to_policy(
1618 iam.PolicyStatement(
1619 effect=iam.Effect.ALLOW,
1620 actions=[
1621 "secretsmanager:GetSecretValue",
1622 "secretsmanager:DescribeSecret",
1623 ],
1624 resources=[auth_secret_resource],
1625 )
1626 )
1628 from gco.stacks.nag_suppressions import acknowledge_nag_findings
1630 # The SQS queue processor runs as gco-manifest-processor-sa. Keep
1631 # queue consumption on that dedicated platform identity; KEDA has its
1632 # own read-only queue role and general workload identities must not be
1633 # able to receive or delete submitted jobs.
1634 self.manifest_processor_role.add_to_policy(
1635 iam.PolicyStatement(
1636 effect=iam.Effect.ALLOW,
1637 actions=[
1638 "sqs:ReceiveMessage",
1639 "sqs:DeleteMessage",
1640 ],
1641 resources=[self.job_queue.queue_arn],
1642 )
1643 )
1645 # Manifest API/worker metrics are emitted only by the dedicated
1646 # platform identity, never by user workload service accounts.
1647 self.manifest_processor_role.add_to_policy(
1648 iam.PolicyStatement(
1649 effect=iam.Effect.ALLOW,
1650 actions=["cloudwatch:PutMetricData"],
1651 resources=["*"],
1652 conditions={"StringEquals": {"cloudwatch:namespace": "GCO/ManifestProcessor"}},
1653 )
1654 )
1656 # The central queue worker's spot price gate reads current spot
1657 # pricing before dispatching price-capped jobs.
1658 # ec2:DescribeSpotPriceHistory is a read-only Describe* action that
1659 # does not support resource-level scoping (Resource must be *).
1660 self.manifest_processor_role.add_to_policy(
1661 iam.PolicyStatement(
1662 effect=iam.Effect.ALLOW,
1663 actions=["ec2:DescribeSpotPriceHistory"],
1664 resources=["*"],
1665 )
1666 )
1668 # Add DynamoDB permissions for templates, webhooks, and job queue
1669 # Tables are created in the global stack and accessed from all regions
1670 project_name = self.config.get_project_name()
1671 global_region = self.config.get_global_region()
1673 # Health-monitor runtime grants are isolated from the shared workload
1674 # role. It can read/repair one endpoint-registry parameter, read webhook
1675 # subscriptions, publish only its metric namespace, and read the auth
1676 # secret granted above.
1677 self.health_monitor_role.add_to_policy(
1678 iam.PolicyStatement(
1679 effect=iam.Effect.ALLOW,
1680 actions=["ssm:GetParameter", "ssm:PutParameter"],
1681 resources=[
1682 f"arn:{self.partition}:ssm:{global_region}:{self.account}:"
1683 f"parameter/{project_name}/alb-hostname-{self.deployment_region}"
1684 ],
1685 )
1686 )
1687 self.health_monitor_role.add_to_policy(
1688 iam.PolicyStatement(
1689 effect=iam.Effect.ALLOW,
1690 actions=["dynamodb:Query", "dynamodb:Scan"],
1691 resources=[
1692 f"arn:{self.partition}:dynamodb:{global_region}:{self.account}:"
1693 f"table/{project_name}-webhooks",
1694 f"arn:{self.partition}:dynamodb:{global_region}:{self.account}:"
1695 f"table/{project_name}-webhooks/index/namespace-index",
1696 ],
1697 )
1698 )
1699 self.health_monitor_role.add_to_policy(
1700 iam.PolicyStatement(
1701 effect=iam.Effect.ALLOW,
1702 actions=["cloudwatch:PutMetricData"],
1703 resources=["*"],
1704 conditions={"StringEquals": {"cloudwatch:namespace": "GCO/HealthMonitor"}},
1705 )
1706 )
1707 acknowledge_nag_findings(
1708 self.health_monitor_role,
1709 [
1710 {
1711 "id": "AwsSolutions-IAM5",
1712 "reason": (
1713 "HealthMonitorRole has two unavoidable wildcard shapes: the "
1714 "Secrets Manager random ARN suffix and cloudwatch:PutMetricData's "
1715 "required Resource:*. PutMetricData is constrained to the exact "
1716 "GCO/HealthMonitor namespace; all SSM and DynamoDB resources are exact."
1717 ),
1718 "appliesTo": ["Resource::*"],
1719 }
1720 ],
1721 )
1723 # The manifest processor is the only identity that can mutate the
1724 # centralized queue. Workload identities receive no access to the jobs
1725 # table, preventing submitted pods from forging queue state.
1726 manifest_table_prefix = (
1727 f"arn:{self.partition}:dynamodb:{global_region}:{self.account}:table/{project_name}"
1728 )
1729 self.manifest_processor_role.add_to_policy(
1730 iam.PolicyStatement(
1731 effect=iam.Effect.ALLOW,
1732 actions=[
1733 "dynamodb:GetItem",
1734 "dynamodb:PutItem",
1735 "dynamodb:UpdateItem",
1736 "dynamodb:DeleteItem",
1737 "dynamodb:Query",
1738 "dynamodb:Scan",
1739 ],
1740 resources=[
1741 f"{manifest_table_prefix}-job-templates",
1742 f"{manifest_table_prefix}-job-templates/index/*",
1743 f"{manifest_table_prefix}-webhooks",
1744 f"{manifest_table_prefix}-webhooks/index/*",
1745 ],
1746 )
1747 )
1748 self.manifest_processor_role.add_to_policy(
1749 iam.PolicyStatement(
1750 effect=iam.Effect.ALLOW,
1751 actions=[
1752 "dynamodb:GetItem",
1753 "dynamodb:PutItem",
1754 "dynamodb:UpdateItem",
1755 "dynamodb:Query",
1756 "dynamodb:Scan",
1757 ],
1758 resources=[
1759 f"{manifest_table_prefix}-jobs",
1760 f"{manifest_table_prefix}-jobs/index/*",
1761 ],
1762 )
1763 )
1765 # The inference proxy needs only point reads of endpoint state. It has
1766 # no write, scan, index, S3, Kubernetes, or queue permissions.
1767 self.inference_proxy_role.add_to_policy(
1768 iam.PolicyStatement(
1769 effect=iam.Effect.ALLOW,
1770 actions=["dynamodb:GetItem"],
1771 resources=[f"{manifest_table_prefix}-inference-endpoints"],
1772 )
1773 )
1774 acknowledge_nag_findings(
1775 self.inference_proxy_role,
1776 [
1777 {
1778 "id": "AwsSolutions-IAM5",
1779 "reason": (
1780 "InferenceProxyRole uses one wildcard only for the random "
1781 "Secrets Manager ARN suffix. DynamoDB access is an exact-table "
1782 "GetItem grant, and the role has no Kubernetes, queue, or write access."
1783 ),
1784 "appliesTo": ["Resource::*"],
1785 }
1786 ],
1787 )
1789 # The inference monitor still owns desired-state reconciliation, but
1790 # this shared role intentionally has no jobs-table ARN.
1791 self.service_account_role.add_to_policy(
1792 iam.PolicyStatement(
1793 effect=iam.Effect.ALLOW,
1794 actions=[
1795 "dynamodb:GetItem",
1796 "dynamodb:PutItem",
1797 "dynamodb:UpdateItem",
1798 "dynamodb:DeleteItem",
1799 "dynamodb:Query",
1800 "dynamodb:Scan",
1801 ],
1802 resources=[
1803 f"{manifest_table_prefix}-inference-endpoints",
1804 f"{manifest_table_prefix}-inference-endpoints/index/*",
1805 ],
1806 )
1807 )
1808 acknowledge_nag_findings(
1809 self.manifest_processor_role,
1810 [
1811 {
1812 "id": "AwsSolutions-IAM5",
1813 "reason": (
1814 "ManifestProcessorRole has only four required wildcard shapes: "
1815 "DynamoDB secondary indexes, the Secrets Manager generated ARN "
1816 "suffix, cloudwatch:PutMetricData Resource:*, and the read-only "
1817 "ec2:DescribeSpotPriceHistory Resource:* (Describe* actions do "
1818 "not support resource-level scoping; the central queue worker's "
1819 "spot price gate needs current pricing). DynamoDB table names "
1820 "and the CloudWatch namespace are otherwise exact."
1821 ),
1822 "appliesTo": ["Resource::*"],
1823 }
1824 ],
1825 )
1827 # Add S3 permissions for model weights bucket (used by inference init containers)
1828 self.service_account_role.add_to_policy(
1829 iam.PolicyStatement(
1830 effect=iam.Effect.ALLOW,
1831 actions=[
1832 "s3:GetObject",
1833 "s3:ListBucket",
1834 ],
1835 resources=[
1836 f"arn:{self.partition}:s3:::{project_name}-*",
1837 f"arn:{self.partition}:s3:::{project_name}-*/*",
1838 ],
1839 )
1840 )
1842 # KMS decrypt for model weights bucket (S3-scoped)
1843 self.service_account_role.add_to_policy(
1844 iam.PolicyStatement(
1845 effect=iam.Effect.ALLOW,
1846 actions=["kms:Decrypt", "kms:GenerateDataKey"],
1847 resources=[f"arn:{self.partition}:kms:*:{self.account}:key/*"],
1848 conditions={
1849 "StringLike": {
1850 "kms:ViaService": f"s3.*.{self.url_suffix}",
1851 }
1852 },
1853 )
1854 )
1856 # Create KEDA operator IAM role for SQS access
1857 self._create_keda_operator_role()
1859 # Create Pod Identity Associations for all service accounts
1860 self._create_pod_identity_associations()
1862 def _create_aws_load_balancer_controller_role(self) -> None:
1863 """Create the controller's exact OIDC-only IRSA role and v3.4.2 policy."""
1864 self.aws_load_balancer_controller_role = GCORegionalStack._create_irsa_role(
1865 self,
1866 "AwsLoadBalancerControllerRole",
1867 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1868 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1869 service_account_names=["aws-load-balancer-controller"],
1870 namespaces=["kube-system"],
1871 include_pod_identity=False,
1872 )
1873 self.aws_load_balancer_controller_policy = iam.Policy(
1874 self,
1875 "AwsLoadBalancerControllerPolicy",
1876 document=iam.PolicyDocument.from_json(
1877 aws_load_balancer_controller_policy_document(self.partition)
1878 ),
1879 )
1880 self.aws_load_balancer_controller_role.attach_inline_policy(
1881 self.aws_load_balancer_controller_policy
1882 )
1884 from gco.stacks.nag_suppressions import acknowledge_nag_findings
1886 acknowledge_nag_findings(
1887 self.aws_load_balancer_controller_policy,
1888 [
1889 {
1890 "id": "AwsSolutions-IAM5",
1891 "reason": (
1892 "This is the exact upstream AWS Load Balancer Controller v3.4.2 "
1893 "IAM policy. Its Resource::* entries cover AWS APIs that cannot "
1894 "be resource-scoped, while its wildcard ARN segments are limited "
1895 "to the controller's supported EC2 and ELB resource types and "
1896 "constrained by upstream cluster ownership tag conditions. The "
1897 "role trust is restricted to kube-system/aws-load-balancer-controller."
1898 ),
1899 "appliesTo": [
1900 "Resource::*",
1901 "Resource::arn:<AWS::Partition>:ec2:*:*:security-group/*",
1902 (
1903 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
1904 "loadbalancer/app/*/*"
1905 ),
1906 (
1907 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
1908 "loadbalancer/net/*/*"
1909 ),
1910 ("Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:targetgroup/*/*"),
1911 (
1912 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
1913 "listener-rule/app/*/*/*"
1914 ),
1915 (
1916 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
1917 "listener-rule/net/*/*/*"
1918 ),
1919 (
1920 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
1921 "listener/app/*/*/*"
1922 ),
1923 (
1924 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
1925 "listener/net/*/*/*"
1926 ),
1927 ],
1928 }
1929 ],
1930 )
1932 def _create_keda_operator_role(self) -> None:
1933 """Create IAM role for KEDA operator service account using EKS Pod Identity.
1935 This role allows the KEDA operator to access SQS queues for scaling
1936 based on queue depth. The role is assumed by the keda-operator service
1937 account in the keda namespace.
1938 """
1939 # Create IAM role with IRSA (OIDC) trust + Pod Identity trust
1940 self.keda_operator_role = GCORegionalStack._create_irsa_role(
1941 self,
1942 "KedaOperatorRole",
1943 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1944 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1945 service_account_names=["keda-operator"],
1946 namespaces=["keda"],
1947 )
1949 # Add SQS permissions for KEDA to read queue metrics
1950 self.keda_operator_role.add_to_policy(
1951 iam.PolicyStatement(
1952 effect=iam.Effect.ALLOW,
1953 actions=[
1954 "sqs:GetQueueAttributes",
1955 "sqs:GetQueueUrl",
1956 ],
1957 resources=[
1958 self.job_queue.queue_arn,
1959 self.job_dlq.queue_arn,
1960 ],
1961 )
1962 )
1964 # CloudWatch read permissions for GPU-based autoscaling. The KEDA
1965 # aws-cloudwatch scaler reads ContainerInsights GPU utilization metrics
1966 # to scale inference roles that request GPUs — GPU is not a native HPA
1967 # resource metric, so this is the only path that can drive GPU scaling.
1968 # The CloudWatch read APIs do not support resource-level IAM scoping, so
1969 # they are granted account-wide (read-only).
1970 self.keda_operator_role.add_to_policy(
1971 iam.PolicyStatement(
1972 effect=iam.Effect.ALLOW,
1973 actions=[
1974 "cloudwatch:GetMetricData",
1975 "cloudwatch:GetMetricStatistics",
1976 "cloudwatch:ListMetrics",
1977 ],
1978 resources=["*"],
1979 )
1980 )
1982 # cdk-nag suppression: the CloudWatch metric-read APIs do not support
1983 # resource-level IAM scoping — Resource: * is the only valid form.
1984 from gco.stacks.nag_suppressions import acknowledge_nag_findings
1986 acknowledge_nag_findings(
1987 self.keda_operator_role,
1988 [
1989 {
1990 "id": "AwsSolutions-IAM5",
1991 "reason": (
1992 "The KEDA operator reads CloudWatch metrics "
1993 "(GetMetricData, GetMetricStatistics, ListMetrics) to "
1994 "drive GPU-based autoscaling. These APIs do not support "
1995 "resource-level IAM scoping — Resource: * is the only "
1996 "valid form. The grant is read-only."
1997 ),
1998 "appliesTo": ["Resource::*"],
1999 },
2000 ],
2001 )
2003 def _create_pod_identity_associations(self) -> None:
2004 """Create EKS Pod Identity Associations for all service accounts.
2006 Pod Identity is the recommended mechanism for EKS Auto Mode. Each
2007 association links an IAM role to a Kubernetes service account in a
2008 specific namespace. EKS manages credential injection automatically.
2010 Stores associations in self._pod_identity_associations so the
2011 kubectl-applier custom resource can declare an explicit dependency,
2012 ensuring credentials are available before workloads start.
2013 """
2014 self._pod_identity_associations: list[Any] = []
2016 # Health monitor — isolated write access for ALB hostname self-healing.
2017 health_assoc = eks_l1.CfnPodIdentityAssociation(
2018 self,
2019 "PodIdentity-health-monitor",
2020 cluster_name=self.cluster.cluster_name,
2021 namespace="gco-system",
2022 service_account="gco-health-monitor-sa",
2023 role_arn=self.health_monitor_role.role_arn,
2024 )
2025 self._pod_identity_associations.append(health_assoc)
2027 # Manifest API and central queue worker — dedicated queue mutation role.
2028 manifest_assoc = eks_l1.CfnPodIdentityAssociation(
2029 self,
2030 "PodIdentity-manifest-processor",
2031 cluster_name=self.cluster.cluster_name,
2032 namespace="gco-system",
2033 service_account="gco-manifest-processor-sa",
2034 role_arn=self.manifest_processor_role.role_arn,
2035 )
2036 self._pod_identity_associations.append(manifest_assoc)
2038 # Inference data plane — exact secret + endpoint-table read role, with
2039 # no Kubernetes RBAC binding.
2040 inference_proxy_assoc = eks_l1.CfnPodIdentityAssociation(
2041 self,
2042 "PodIdentity-inference-proxy",
2043 cluster_name=self.cluster.cluster_name,
2044 namespace="gco-system",
2045 service_account="gco-inference-proxy-sa",
2046 role_arn=self.inference_proxy_role.role_arn,
2047 )
2048 self._pod_identity_associations.append(inference_proxy_assoc)
2050 # Shared GCO service account for general platform/job workloads.
2051 for namespace in ["gco-system", "gco-jobs", "gco-inference"]:
2052 assoc = eks_l1.CfnPodIdentityAssociation(
2053 self,
2054 f"PodIdentity-gco-sa-{namespace}",
2055 cluster_name=self.cluster.cluster_name,
2056 namespace=namespace,
2057 service_account="gco-service-account",
2058 role_arn=self.service_account_role.role_arn,
2059 )
2060 self._pod_identity_associations.append(assoc)
2062 # KEDA operator — needs SQS access for queue-based scaling
2063 keda_assoc = eks_l1.CfnPodIdentityAssociation(
2064 self,
2065 "PodIdentity-keda-operator",
2066 cluster_name=self.cluster.cluster_name,
2067 namespace="keda",
2068 service_account="keda-operator",
2069 role_arn=self.keda_operator_role.role_arn,
2070 )
2071 self._pod_identity_associations.append(keda_assoc)
2073 # EFS CSI driver — needs EFS access for shared storage
2074 efs_assoc = eks_l1.CfnPodIdentityAssociation(
2075 self,
2076 "PodIdentity-efs-csi",
2077 cluster_name=self.cluster.cluster_name,
2078 namespace="kube-system",
2079 service_account="efs-csi-controller-sa",
2080 role_arn=self.efs_csi_role.role_arn,
2081 )
2082 self._pod_identity_associations.append(efs_assoc)
2084 # CloudWatch agent — needs CloudWatch access for observability
2085 cw_assoc = eks_l1.CfnPodIdentityAssociation(
2086 self,
2087 "PodIdentity-cloudwatch",
2088 cluster_name=self.cluster.cluster_name,
2089 namespace="amazon-cloudwatch",
2090 service_account="cloudwatch-agent",
2091 role_arn=self.cloudwatch_role.role_arn,
2092 )
2093 self._pod_identity_associations.append(cw_assoc)
2095 # FSx CSI driver — only when FSx is enabled (created later in _create_fsx_lustre)
2096 # The FSx Pod Identity association is added in _create_fsx_lustre instead
2098 def _resolve_cluster_shared_bucket_from_ssm(self) -> SharedBucketIdentity:
2099 """Resolve the ``Cluster_Shared_Bucket`` identity from cross-region SSM.
2101 ``GCOGlobalStack`` publishes three ``ssm.StringParameter``s in the
2102 global region at ``/gco/cluster-shared-bucket/{name,arn,region}``.
2103 This method reads them back from the regional stack via
2104 ``cr.AwsCustomResource`` with ``service="SSM"``,
2105 ``action="getParameter"``, and ``region=<global-region>`` — matching
2106 the cross-region read pattern already used in
2107 ``_create_ga_registration_lambda`` for the Global Accelerator
2108 endpoint group ARN.
2110 Runs unconditionally in ``__init__`` — no feature toggle, no
2111 conditional guard. The returned :class:`SharedBucketIdentity` feeds
2112 ``_grant_cluster_shared_bucket_to_job_role`` (IAM) and the
2113 ``image_replacements`` dict (ConfigMap) downstream.
2115 Returns:
2116 :class:`SharedBucketIdentity` with ``name``, ``arn``, and
2117 ``region`` populated as CDK tokens that resolve at deploy time.
2118 """
2119 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2121 global_region = self.config.get_global_region()
2122 cluster_shared_prefix = cluster_shared_ssm_parameter_prefix(self.config.get_project_name())
2123 resolved: dict[str, str] = {}
2125 for suffix in ("name", "arn", "region"):
2126 parameter_name = f"{cluster_shared_prefix}/{suffix}"
2127 read_cr = cr.AwsCustomResource(
2128 self,
2129 f"ReadClusterSharedBucket{suffix.capitalize()}",
2130 on_create=cr.AwsSdkCall(
2131 service="SSM",
2132 action="getParameter",
2133 parameters={"Name": parameter_name},
2134 region=global_region,
2135 physical_resource_id=cr.PhysicalResourceId.of(f"cluster-shared-{suffix}"),
2136 ),
2137 on_update=cr.AwsSdkCall(
2138 service="SSM",
2139 action="getParameter",
2140 parameters={"Name": parameter_name},
2141 region=global_region,
2142 physical_resource_id=cr.PhysicalResourceId.of(f"cluster-shared-{suffix}"),
2143 ),
2144 # Cross-region SSM GetParameter doesn't support resource-level
2145 # scoping cleanly — the principal evaluating the call lives in
2146 # this stack's region but the parameter lives in the global
2147 # region. ANY_RESOURCE is the AWS-documented escape hatch; the
2148 # resulting AwsSolutions-IAM5 nag finding is suppressed in a
2149 # scoped add_resource_suppressions call below.
2150 policy=cr.AwsCustomResourcePolicy.from_sdk_calls(
2151 resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE
2152 ),
2153 )
2155 # Scoped suppression: the CR policy is Resource::* because the
2156 # SSM parameter lives in the global region (cross-region calls
2157 # don't support resource-level scoping cleanly). The action is
2158 # fixed to ssm:GetParameter and the parameter Name is fixed to
2159 # a literal string, so the wildcard can only ever read one
2160 # parameter.
2161 acknowledge_nag_findings(
2162 read_cr,
2163 [
2164 {
2165 "id": "AwsSolutions-IAM5",
2166 "reason": (
2167 "Cross-region ssm:GetParameter for "
2168 f"{parameter_name} in the global region. The "
2169 "AwsCustomResource SDK-call policy is scoped to "
2170 "a single fixed action (ssm:GetParameter) with "
2171 "a fixed parameter Name — the Resource: * is "
2172 "the CDK-documented escape hatch because the "
2173 "parameter ARN is not known to the calling "
2174 "principal's region. Effective blast radius: "
2175 "one parameter."
2176 ),
2177 "appliesTo": ["Resource::*"],
2178 },
2179 ],
2180 )
2182 resolved[suffix] = read_cr.get_response_field("Parameter.Value")
2184 return SharedBucketIdentity(
2185 name=resolved["name"],
2186 arn=resolved["arn"],
2187 region=resolved["region"],
2188 )
2190 def _grant_cluster_shared_bucket_to_job_role(self, shared: SharedBucketIdentity) -> None:
2191 """Attach RW + KMS permissions on ``Cluster_Shared_Bucket`` to the job-pod role.
2193 Two ``iam.PolicyStatement``s are added to ``self.service_account_role``
2194 (the EKS Pod Identity role used by every pod in ``gco-jobs``,
2195 ``gco-system``, and ``gco-inference``):
2197 1. S3 object + bucket-level actions (``GetObject``, ``PutObject``,
2198 ``DeleteObject``, ``ListBucket``, ``GetBucketLocation``) scoped
2199 to ``<shared.arn>`` and ``<shared.arn>/*`` — the bucket-ARN
2200 shape uses the ``gco-cluster-shared-*`` prefix that IAM
2201 policies scope against.
2202 2. KMS ``Decrypt`` / ``GenerateDataKey`` scoped by the
2203 ``kms:ViaService=s3.<shared.region>.<AWS::URLSuffix>`` condition —
2204 ``resources=["*"]`` because the KMS key ARN is not known to this
2205 stack (it lives in the global region and is referenced indirectly
2206 through the S3 service). The condition is what actually restricts
2207 the grant to the cluster-shared bucket's key.
2209 Runs unconditionally — the grant is
2210 present on every regional cluster whether or not analytics is
2211 enabled.
2212 """
2213 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2215 self.service_account_role.add_to_policy(
2216 iam.PolicyStatement(
2217 effect=iam.Effect.ALLOW,
2218 actions=[
2219 "s3:GetObject",
2220 "s3:PutObject",
2221 "s3:DeleteObject",
2222 "s3:ListBucket",
2223 "s3:GetBucketLocation",
2224 ],
2225 resources=[shared.arn, f"{shared.arn}/*"],
2226 )
2227 )
2229 self.service_account_role.add_to_policy(
2230 iam.PolicyStatement(
2231 effect=iam.Effect.ALLOW,
2232 actions=["kms:Decrypt", "kms:GenerateDataKey"],
2233 resources=["*"],
2234 conditions={
2235 "StringEquals": {
2236 "kms:ViaService": f"s3.{shared.region}.{self.url_suffix}",
2237 }
2238 },
2239 )
2240 )
2242 # The grants contain two necessary wildcard shapes. The S3 bucket ARN
2243 # uses ``/*`` for object keys within the single resolved shared bucket.
2244 # KMS uses ``Resource::*`` because the global key ARN is not exported to
2245 # this stack; ``kms:ViaService`` confines its use to S3 in the bucket's
2246 # region, while the S3 statements separately scope accessible objects.
2247 acknowledge_nag_findings(
2248 self.service_account_role,
2249 [
2250 {
2251 "id": "AwsSolutions-IAM5",
2252 "reason": (
2253 "The Cluster_Shared_Bucket grants require two wildcard shapes: "
2254 "an <arn>/* object-key suffix on the single shared bucket resolved "
2255 "from SSM, and KMS Resource::* because the global key ARN is not "
2256 "exported. KMS use is constrained by kms:ViaService to S3 in the "
2257 "bucket's region, and S3 access is separately limited to the "
2258 "allowed bucket ARNs."
2259 ),
2260 "appliesTo": [
2261 "Resource::*",
2262 "Resource::<ReadClusterSharedBucketArn4B0BD291.Parameter.Value>/*",
2263 ],
2264 },
2265 ],
2266 )
2268 def _create_regional_shared_bucket(self) -> None:
2269 """Create the always-on general-purpose regional bucket for this region.
2271 Provisioned unconditionally — there is no ``cdk.json`` toggle and no
2272 feature flag that can suppress it — in addition to the central buckets
2273 owned by ``GCOGlobalStack`` (the model bucket and the cluster-shared
2274 bucket). The bucket is general purpose: any in-region workload may use
2275 it, and the per-region cold KV tier auto-targets it when cold-tier
2276 storage is requested. Its existence is independent of any endpoint's
2277 cold-tier choice.
2279 Three constructs are created, mirroring the cluster-shared bucket
2280 pattern in ``GCOGlobalStack``:
2282 1. ``regional_shared_kms_key`` — a customer-managed KMS key with annual
2283 rotation and a 7-day pending window on destroy. The key policy grants
2284 the ``s3.amazonaws.com`` and ``logs.<region>.amazonaws.com`` service
2285 principals encrypt/decrypt so S3 server-side encryption and access-log
2286 delivery work without role-side grants.
2287 2. ``regional_shared_access_logs_bucket`` — the dedicated S3 access-logs
2288 destination for the primary bucket.
2289 3. ``regional_shared_bucket`` — the primary bucket named
2290 ``<project_name>-regional-shared-<account>-<region>`` (the prefix
2291 from ``regional_shared_bucket_name_prefix(project_name)`` is the
2292 stable ARN prefix used by IAM policies and nag assertions).
2293 KMS-encrypted with
2294 ``regional_shared_kms_key``, block-public-access on, SSL enforced,
2295 versioned, destroy-on-teardown.
2297 An explicit ``Deny`` for ``aws:SecureTransport=false`` is added to the
2298 bucket policy independent of ``enforce_ssl=True`` so the deny is
2299 verifiable in the synthesized template under a known SID.
2300 """
2301 # KMS key for the regional bucket. Matches the cluster-shared key
2302 # posture: annual rotation, 7-day pending window, destroy-on-teardown.
2303 self.regional_shared_kms_key = kms.Key(
2304 self,
2305 "RegionalSharedKmsKey",
2306 description=(
2307 "Customer-managed KMS key for the always-on general-purpose "
2308 "regional bucket in this region's GCORegionalStack."
2309 ),
2310 enable_key_rotation=True,
2311 pending_window=Duration.days(7),
2312 removal_policy=RemovalPolicy.DESTROY,
2313 )
2315 # Key-policy grants for the service principals that encrypt/decrypt on
2316 # behalf of the bucket (S3 server-side encryption) and the access-logs
2317 # bucket (CloudWatch/S3 log delivery).
2318 kms_actions = [
2319 "kms:Encrypt",
2320 "kms:Decrypt",
2321 "kms:ReEncrypt*",
2322 "kms:GenerateDataKey*",
2323 "kms:DescribeKey",
2324 ]
2326 self.regional_shared_kms_key.add_to_resource_policy(
2327 iam.PolicyStatement(
2328 sid="AllowS3ServiceEncryptDecrypt",
2329 effect=iam.Effect.ALLOW,
2330 principals=[iam.ServicePrincipal("s3.amazonaws.com")],
2331 actions=kms_actions,
2332 resources=["*"],
2333 )
2334 )
2336 self.regional_shared_kms_key.add_to_resource_policy(
2337 iam.PolicyStatement(
2338 sid="AllowCloudWatchLogsEncryptDecrypt",
2339 effect=iam.Effect.ALLOW,
2340 principals=[iam.ServicePrincipal(f"logs.{self.deployment_region}.amazonaws.com")],
2341 actions=kms_actions,
2342 resources=["*"],
2343 )
2344 )
2346 # Retention for the access-logs bucket honors the same `s3_access_logs`
2347 # context field used by the central buckets (default 90 days).
2348 s3_access_logs_ctx = self.node.try_get_context("s3_access_logs") or {}
2349 access_logs_retention_days = int(s3_access_logs_ctx.get("retention_days", 90))
2351 # Dedicated access-logs bucket for the regional bucket, encrypted with
2352 # the regional KMS key (its key policy grants the logs service principal
2353 # encrypt/decrypt).
2354 self.regional_shared_access_logs_bucket = s3.Bucket(
2355 self,
2356 "RegionalSharedAccessLogsBucket",
2357 encryption=s3.BucketEncryption.KMS,
2358 encryption_key=self.regional_shared_kms_key,
2359 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
2360 enforce_ssl=True,
2361 versioned=True,
2362 removal_policy=RemovalPolicy.DESTROY,
2363 auto_delete_objects=True,
2364 lifecycle_rules=[
2365 s3.LifecycleRule(
2366 id="ExpireAccessLogs",
2367 enabled=True,
2368 expiration=Duration.days(access_logs_retention_days),
2369 )
2370 ],
2371 )
2373 # Primary general-purpose regional bucket. The name is derived from
2374 # ``project_name`` so the bucket and the IAM allow-list assertions
2375 # (arn:<partition>:s3:::<project_name>-regional-shared-*) stay in lockstep and
2376 # two deployments in the same account+region do not collide.
2377 # `bucket_key_enabled=True` mirrors the central-bucket pattern to
2378 # reduce per-object KMS request costs.
2379 project_name = self.config.get_project_name()
2380 regional_shared_prefix = regional_shared_ssm_parameter_prefix(project_name)
2381 self.regional_shared_bucket = s3.Bucket(
2382 self,
2383 "RegionalSharedBucket",
2384 bucket_name=(
2385 f"{regional_shared_bucket_name_prefix(project_name)}"
2386 f"-{self.account}-{self.deployment_region}"
2387 ),
2388 encryption=s3.BucketEncryption.KMS,
2389 encryption_key=self.regional_shared_kms_key,
2390 bucket_key_enabled=True,
2391 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
2392 enforce_ssl=True,
2393 versioned=True,
2394 removal_policy=RemovalPolicy.DESTROY,
2395 auto_delete_objects=True,
2396 server_access_logs_bucket=self.regional_shared_access_logs_bucket,
2397 server_access_logs_prefix="regional-shared/",
2398 )
2400 # Explicit Deny for insecure transport. `enforce_ssl=True` already adds
2401 # an equivalent statement, but duplicating it here makes the deny
2402 # verifiable in the synthesized template under a known SID.
2403 self.regional_shared_bucket.add_to_resource_policy(
2404 iam.PolicyStatement(
2405 sid="DenyInsecureTransport",
2406 effect=iam.Effect.DENY,
2407 principals=[iam.AnyPrincipal()],
2408 actions=["s3:*"],
2409 resources=[
2410 self.regional_shared_bucket.bucket_arn,
2411 f"{self.regional_shared_bucket.bucket_arn}/*",
2412 ],
2413 conditions={"Bool": {"aws:SecureTransport": "false"}},
2414 )
2415 )
2417 # Publish the bucket's identity as three SSM parameters in this
2418 # region's own parameter store, mirroring how the model bucket and
2419 # cluster-shared bucket publish theirs. In-region workloads and the
2420 # regional upload surface resolve the always-on regional bucket by
2421 # reading these back rather than reconstructing the name. Because the
2422 # bucket is unconditional, these parameters are always present once the
2423 # region's stack is deployed. The prefix from
2424 # ``regional_shared_ssm_parameter_prefix(project_name)`` is the single
2425 # source of truth for the namespace.
2426 ssm.StringParameter(
2427 self,
2428 "RegionalSharedBucketNameParam",
2429 parameter_name=f"{regional_shared_prefix}/name",
2430 string_value=self.regional_shared_bucket.bucket_name,
2431 description="Name of the always-on general-purpose regional bucket for this region.",
2432 )
2434 ssm.StringParameter(
2435 self,
2436 "RegionalSharedBucketArnParam",
2437 parameter_name=f"{regional_shared_prefix}/arn",
2438 string_value=self.regional_shared_bucket.bucket_arn,
2439 description="ARN of the always-on general-purpose regional bucket for this region.",
2440 )
2442 ssm.StringParameter(
2443 self,
2444 "RegionalSharedBucketRegionParam",
2445 parameter_name=f"{regional_shared_prefix}/region",
2446 string_value=self.deployment_region,
2447 description="Home region of the always-on general-purpose regional bucket.",
2448 )
2450 # CDK-nag suppressions scoped per-resource at the construct site,
2451 # mirroring the central bucket pattern. Every suppression carries an
2452 # explicit reason; no blanket bypasses.
2453 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2455 regional_replication_reason = (
2456 "The general-purpose regional bucket is a region-local store; "
2457 "in-region workloads publish to their own region's bucket and there "
2458 "is no durability requirement that warrants cross-region "
2459 "replication. Access logs do not require replication for the same "
2460 "reason."
2461 )
2463 acknowledge_nag_findings(
2464 self.regional_shared_bucket,
2465 [
2466 {
2467 "id": "HIPAA.Security-S3BucketReplicationEnabled",
2468 "reason": regional_replication_reason,
2469 },
2470 {
2471 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
2472 "reason": regional_replication_reason,
2473 },
2474 {
2475 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
2476 "reason": regional_replication_reason,
2477 },
2478 ],
2479 )
2481 access_logs_is_self_target_reason = (
2482 "This is the server access logs destination bucket for the "
2483 "general-purpose regional bucket."
2484 )
2485 acknowledge_nag_findings(
2486 self.regional_shared_access_logs_bucket,
2487 [
2488 {
2489 "id": "AwsSolutions-S1",
2490 "reason": access_logs_is_self_target_reason,
2491 },
2492 {
2493 "id": "HIPAA.Security-S3BucketLoggingEnabled",
2494 "reason": access_logs_is_self_target_reason,
2495 },
2496 {
2497 "id": "NIST.800.53.R5-S3BucketLoggingEnabled",
2498 "reason": access_logs_is_self_target_reason,
2499 },
2500 {
2501 "id": "PCI.DSS.321-S3BucketLoggingEnabled",
2502 "reason": access_logs_is_self_target_reason,
2503 },
2504 {
2505 "id": "HIPAA.Security-S3BucketReplicationEnabled",
2506 "reason": regional_replication_reason,
2507 },
2508 {
2509 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
2510 "reason": regional_replication_reason,
2511 },
2512 {
2513 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
2514 "reason": regional_replication_reason,
2515 },
2516 ],
2517 )
2519 # Grant the in-region pod role read/write on this bucket and use of its
2520 # KMS key — and nothing else. The grant lives next to the bucket it
2521 # scopes to, so the role's regional-bucket access stays exactly as wide
2522 # as this one bucket and its key.
2523 self._grant_regional_shared_bucket_to_service_account()
2525 def _grant_regional_shared_bucket_to_service_account(self) -> None:
2526 """Attach RW + KMS permissions on the regional bucket to the pod role.
2528 Two ``iam.PolicyStatement``s are added to ``self.service_account_role``
2529 (the EKS Pod Identity role used by every pod in ``gco-jobs``,
2530 ``gco-system``, and ``gco-inference``):
2532 1. S3 object + bucket-level actions (``GetObject``, ``PutObject``,
2533 ``DeleteObject``, ``ListBucket``, ``GetBucketLocation``) scoped to
2534 the literal ``regional_shared_bucket`` ARN and its ``<arn>/*``
2535 object-key space — and to no other bucket.
2536 2. KMS ``Decrypt`` / ``Encrypt`` / ``GenerateDataKey`` /
2537 ``DescribeKey`` scoped to the literal ``regional_shared_kms_key``
2538 ARN — and to no other key.
2540 Because both resources are local constructs in this stack, each ARN is
2541 a concrete reference rather than a wildcard, so the role gains access to
2542 precisely this bucket and this key. The grant runs unconditionally as
2543 part of provisioning the always-on regional bucket.
2544 """
2545 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2547 self.service_account_role.add_to_policy(
2548 iam.PolicyStatement(
2549 effect=iam.Effect.ALLOW,
2550 actions=[
2551 "s3:GetObject",
2552 "s3:PutObject",
2553 "s3:DeleteObject",
2554 "s3:ListBucket",
2555 "s3:GetBucketLocation",
2556 ],
2557 resources=[
2558 self.regional_shared_bucket.bucket_arn,
2559 f"{self.regional_shared_bucket.bucket_arn}/*",
2560 ],
2561 )
2562 )
2564 self.service_account_role.add_to_policy(
2565 iam.PolicyStatement(
2566 effect=iam.Effect.ALLOW,
2567 actions=[
2568 "kms:Decrypt",
2569 "kms:Encrypt",
2570 "kms:GenerateDataKey",
2571 "kms:DescribeKey",
2572 ],
2573 resources=[self.regional_shared_kms_key.key_arn],
2574 )
2575 )
2577 # The S3 bucket-ARN resource uses a ``<arn>/*`` object-key wildcard
2578 # which cdk-nag flags as a wildcard resource. The ARN itself is the
2579 # literal regional bucket ARN created in this stack — the ``/*`` covers
2580 # all object keys inside that single bucket, which is the intended
2581 # semantic for the RW grant. The KMS statement carries no wildcard.
2582 acknowledge_nag_findings(
2583 self.service_account_role,
2584 [
2585 {
2586 "id": "AwsSolutions-IAM5",
2587 "reason": (
2588 "The regional bucket RW grant uses an <arn>/* "
2589 "object-key wildcard on the literal "
2590 "gco-regional-shared-<account>-<region> bucket ARN "
2591 "created in this stack. The wildcard covers object "
2592 "keys within a single bucket — this is the standard "
2593 "shape for a bucket-scoped RW grant and is what the "
2594 "allow-list assertion is written against."
2595 ),
2596 "appliesTo": [
2597 "Resource::<RegionalSharedBucket3FF19783.Arn>/*",
2598 ],
2599 },
2600 ],
2601 )
2603 def _grant_cost_report_bucket_to_cost_monitor(self) -> None:
2604 """Grant the cost-monitor role write access to the cost report bucket.
2606 The bucket lives in ``GCOMonitoringStack`` in the monitoring region,
2607 which deploys *after* every regional stack — so no cross-stack
2608 reference or SSM read can resolve it here. Its physical name is fully
2609 deterministic (``cost_report_bucket_name``), which lets this grant use
2610 a literal ARN:
2612 1. S3 object + bucket-level actions (``PutObject``, ``GetObject``,
2613 ``ListBucket``, ``GetBucketLocation``) scoped to the literal cost
2614 report bucket ARN and its object-key space — and no other bucket.
2615 The service writes scheduled/ad-hoc Parquet reports and lists
2616 recent report objects for the API surface.
2617 2. KMS ``GenerateDataKey`` / ``Decrypt`` / ``DescribeKey`` restricted
2618 by ``kms:ViaService`` to S3 in the monitoring region. The bucket's
2619 customer-managed key ARN is not knowable from this stack, so the
2620 via-service condition provides the scoping — the same pattern the
2621 analytics stack uses for the cluster-shared bucket key.
2623 On a fresh ``deploy-all`` the bucket materializes only after the
2624 regional stacks; the cost-monitor service retries its next scheduled
2625 write, so the pipeline self-heals without ordering hacks.
2626 """
2627 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2629 monitoring_region = self.config.get_monitoring_region()
2630 bucket_arn = (
2631 f"arn:{self.partition}:s3:::"
2632 f"{cost_report_bucket_name(self.config.get_project_name(), self.account, monitoring_region)}"
2633 )
2635 self.cost_monitor_role.add_to_policy(
2636 iam.PolicyStatement(
2637 effect=iam.Effect.ALLOW,
2638 actions=[
2639 "s3:PutObject",
2640 "s3:GetObject",
2641 "s3:ListBucket",
2642 "s3:GetBucketLocation",
2643 ],
2644 resources=[bucket_arn, f"{bucket_arn}/*"],
2645 )
2646 )
2648 self.cost_monitor_role.add_to_policy(
2649 iam.PolicyStatement(
2650 effect=iam.Effect.ALLOW,
2651 actions=[
2652 "kms:GenerateDataKey",
2653 "kms:Decrypt",
2654 "kms:DescribeKey",
2655 ],
2656 resources=["*"],
2657 conditions={
2658 "StringEquals": {
2659 "kms:ViaService": f"s3.{monitoring_region}.{self.url_suffix}",
2660 }
2661 },
2662 )
2663 )
2665 acknowledge_nag_findings(
2666 self.cost_monitor_role,
2667 [
2668 {
2669 "id": "AwsSolutions-IAM5",
2670 "reason": (
2671 "The cost-monitor S3 grant uses an <arn>/* object-key "
2672 "wildcard on the literal deterministic cost report bucket "
2673 "ARN (one bucket). The KMS statement uses Resource::* "
2674 "because the bucket's customer-managed key is created by "
2675 "the monitoring stack, which deploys after this stack; the "
2676 "kms:ViaService condition restricts use to S3 in the "
2677 "monitoring region."
2678 ),
2679 "appliesTo": [
2680 "Resource::*",
2681 f"Resource::arn:<AWS::Partition>:s3:::{cost_report_bucket_name(self.config.get_project_name(), '<AWS::AccountId>', monitoring_region)}/*",
2682 ],
2683 },
2684 ],
2685 )
2687 def _create_kubectl_lambda(self) -> None:
2688 """Create Lambda function to apply Kubernetes manifests using Python client.
2690 Note: This creates the Lambda and provider but does NOT create the custom resource.
2691 The custom resource is created in _apply_kubernetes_manifests() after ALB is created,
2692 so that target group ARNs can be passed to the manifests.
2693 """
2694 project_name = self.config.get_project_name()
2696 # Create IAM role for kubectl Lambda
2697 kubectl_lambda_role = iam.Role(
2698 self,
2699 "KubectlLambdaRole",
2700 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
2701 managed_policies=[
2702 iam.ManagedPolicy.from_aws_managed_policy_name(
2703 "service-role/AWSLambdaVPCAccessExecutionRole"
2704 ),
2705 iam.ManagedPolicy.from_aws_managed_policy_name(
2706 "service-role/AWSLambdaBasicExecutionRole"
2707 ),
2708 ],
2709 )
2711 # Add EKS permissions
2712 kubectl_lambda_role.add_to_policy(
2713 iam.PolicyStatement(
2714 actions=[
2715 "eks:DescribeCluster",
2716 "eks:ListClusters",
2717 ],
2718 resources=[self.cluster.cluster_arn],
2719 )
2720 )
2722 # Add permissions to assume cluster admin role
2723 kubectl_lambda_role.add_to_policy(
2724 iam.PolicyStatement(actions=["sts:AssumeRole"], resources=["*"])
2725 )
2727 # Allow the convergence apply tasks to record per-phase status to SSM
2728 # (base-manifests / post-helm-manifests), mirroring the helm worker, so
2729 # `gco stacks addons status` surfaces the apply passes alongside charts.
2730 kubectl_lambda_role.add_to_policy(
2731 iam.PolicyStatement(
2732 actions=["ssm:PutParameter"],
2733 resources=[
2734 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
2735 f"parameter/{project_name}/addons/*"
2736 ],
2737 )
2738 )
2740 # Create security group for kubectl Lambda
2741 kubectl_lambda_sg = ec2.SecurityGroup(
2742 self,
2743 "KubectlLambdaSG",
2744 vpc=self.vpc,
2745 description="Security group for kubectl Lambda to access EKS cluster",
2746 security_group_name=f"{self.config.get_project_name()}-kubectl-lambda-sg-{self.deployment_region}",
2747 allow_all_outbound=True, # Lambda needs outbound access to EKS API
2748 )
2750 # Allow Lambda security group to access EKS cluster security group on port 443
2751 # The EKS cluster security group is automatically created by EKS
2752 self.cluster.cluster_security_group.add_ingress_rule(
2753 peer=kubectl_lambda_sg,
2754 connection=ec2.Port.tcp(443),
2755 description="Allow kubectl Lambda to access EKS API",
2756 )
2758 # Create Lambda function (Python-only, no Docker!)
2759 # Store function name as string attribute for cross-stack references
2760 # This avoids CDK cross-environment resolution issues when account is unresolved
2761 self.kubectl_lambda_function_name = f"{project_name}-kubectl-{self.deployment_region}"
2762 self.kubectl_lambda = lambda_.Function(
2763 self,
2764 "KubectlApplierFunction",
2765 function_name=self.kubectl_lambda_function_name,
2766 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
2767 handler="handler.lambda_handler",
2768 code=lambda_.Code.from_asset("lambda/kubectl-applier-simple-build"),
2769 timeout=Duration.minutes(15), # Max Lambda timeout
2770 memory_size=512,
2771 role=kubectl_lambda_role,
2772 vpc=self.vpc,
2773 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
2774 security_groups=[kubectl_lambda_sg], # Use the security group we created
2775 environment={
2776 "CLUSTER_NAME": self.cluster.cluster_name,
2777 "REGION": self.deployment_region,
2778 # Lets the convergence apply tasks record per-phase status to
2779 # SSM (/<project>/addons/<region>/{base,post-helm}-manifests).
2780 "PROJECT_NAME": project_name,
2781 },
2782 tracing=lambda_.Tracing.ACTIVE,
2783 )
2785 # Add EKS access entry for the Lambda role to authenticate with the cluster
2786 # This grants the Lambda role cluster admin permissions
2787 self.kubectl_lambda_access_entry = eks.AccessEntry(
2788 self,
2789 "KubectlLambdaAccessEntry",
2790 cluster=self.cluster, # type: ignore[arg-type]
2791 principal=kubectl_lambda_role.role_arn,
2792 access_policies=[
2793 eks.AccessPolicy.from_access_policy_name(
2794 "AmazonEKSClusterAdminPolicy", access_scope_type=eks.AccessScopeType.CLUSTER
2795 )
2796 ],
2797 )
2799 # No custom-resource provider needed: the kubectl-applier Lambda is now
2800 # invoked directly by the convergence state machine (the base and
2801 # post-Helm apply tasks), not through a CloudFormation custom resource.
2803 # cdk-nag suppression: the kubectl-applier Lambda requires broad
2804 # EKS and Kubernetes API access to apply arbitrary manifests.
2805 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2807 acknowledge_nag_findings(
2808 kubectl_lambda_role,
2809 [
2810 {
2811 "id": "AwsSolutions-IAM5",
2812 "reason": (
2813 "The kubectl-applier Lambda requires broad EKS and Kubernetes API "
2814 "access to apply arbitrary manifests (RBAC, ServiceAccounts, "
2815 "Deployments, Jobs, NetworkPolicies) across multiple namespaces. "
2816 "Resource: * is required because the set of Kubernetes resources "
2817 "is dynamic and not known at synth time."
2818 ),
2819 "appliesTo": ["Resource::*"],
2820 },
2821 ],
2822 )
2824 def _apply_kubernetes_manifests(self) -> None:
2825 """Build the complete base/Helm/post-Helm convergence pipeline.
2827 This is called after the Gateway certificate and shared storage exist.
2828 The post-Helm pass creates the internal ALB through Gateway API only
2829 after the mandatory AWS Load Balancer Controller is installed.
2830 """
2832 # Build image replacements dict
2833 # Include one deployment token to force pod rollouts and bind live
2834 # validation to this exact asynchronous convergence execution.
2835 deployment_timestamp = _deployment_timestamp()
2836 self.addon_deployment_token = deployment_timestamp
2838 # Get resource thresholds from config
2839 thresholds = self.config.get_resource_thresholds()
2841 # Get manifest processor resource quotas.
2842 # Resource quotas and the security/image policy now live under the
2843 # shared job_validation_policy section because both the REST
2844 # manifest_processor and the SQS queue_processor read them. Service-
2845 # specific knobs (replicas, validation_enabled, max_request_body_bytes,
2846 # etc.) stay under manifest_processor.
2847 mp_config = self.config.get_manifest_processor_config()
2848 job_policy = self.node.try_get_context("job_validation_policy") or {}
2849 job_quotas = job_policy.get("resource_quotas", {})
2850 allowed_kinds = job_policy.get(
2851 "allowed_kinds",
2852 [
2853 "Job",
2854 "CronJob",
2855 "Deployment",
2856 "StatefulSet",
2857 "DaemonSet",
2858 "Service",
2859 "ConfigMap",
2860 "Pod",
2861 ],
2862 )
2864 image_replacements = {
2865 "{{BACKEND_TLS_CERTIFICATE_ARN}}": self.backend_tls_certificate_arn,
2866 "{{HEALTH_MONITOR_IMAGE}}": self.health_monitor_image.image_uri,
2867 "{{MANIFEST_PROCESSOR_IMAGE}}": self.manifest_processor_image.image_uri,
2868 "{{INFERENCE_PROXY_IMAGE}}": self.inference_proxy_image.image_uri,
2869 "{{INFERENCE_MONITOR_IMAGE}}": self.inference_monitor_image.image_uri,
2870 # External, pinned upstream image for the shared Mooncake master
2871 # (bundles the mooncake_master binary). Same default as disaggregated
2872 # role pods; per-endpoint spec.mooncake.store.master_image overrides.
2873 "{{MOONCAKE_MASTER_IMAGE}}": MOONCAKE_MASTER_DEFAULT_IMAGE,
2874 "{{CLUSTER_NAME}}": self.cluster.cluster_name,
2875 "{{REGION}}": self.deployment_region,
2876 "{{AUTH_SECRET_ARN}}": self.auth_secret_arn,
2877 "{{SERVICE_ACCOUNT_ROLE_ARN}}": self.service_account_role.role_arn,
2878 "{{MANIFEST_PROCESSOR_ROLE_ARN}}": self.manifest_processor_role.role_arn,
2879 "{{INFERENCE_PROXY_ROLE_ARN}}": self.inference_proxy_role.role_arn,
2880 "{{HEALTH_MONITOR_ROLE_ARN}}": self.health_monitor_role.role_arn,
2881 "{{EFS_FILE_SYSTEM_ID}}": self.efs_file_system.file_system_id,
2882 "{{EFS_ACCESS_POINT_ID}}": self.efs_access_point.access_point_id,
2883 "{{JOB_QUEUE_URL}}": self.job_queue.queue_url,
2884 "{{JOB_QUEUE_ARN}}": self.job_queue.queue_arn,
2885 "{{DEPLOYMENT_TIMESTAMP}}": deployment_timestamp,
2886 # Resource thresholds
2887 "{{CPU_THRESHOLD}}": str(thresholds.cpu_threshold),
2888 "{{MEMORY_THRESHOLD}}": str(thresholds.memory_threshold),
2889 "{{GPU_THRESHOLD}}": str(thresholds.gpu_threshold),
2890 "{{PENDING_PODS_THRESHOLD}}": str(thresholds.pending_pods_threshold),
2891 "{{PENDING_REQUESTED_CPU_VCPUS}}": str(thresholds.pending_requested_cpu_vcpus),
2892 "{{PENDING_REQUESTED_MEMORY_GB}}": str(thresholds.pending_requested_memory_gb),
2893 "{{PENDING_REQUESTED_GPUS}}": str(thresholds.pending_requested_gpus),
2894 # Deployment prefix (#139). Injected so in-cluster services (the
2895 # inference monitor) resolve project-scoped SSM paths
2896 # (/<project>/regional-shared-bucket/*) instead of a hardcoded
2897 # /gco/ namespace, letting two deployments share an account+region.
2898 "{{PROJECT_NAME}}": self.config.get_project_name(),
2899 # DynamoDB table names (from global stack)
2900 "{{TEMPLATES_TABLE_NAME}}": f"{self.config.get_project_name()}-job-templates",
2901 "{{WEBHOOKS_TABLE_NAME}}": f"{self.config.get_project_name()}-webhooks",
2902 "{{JOBS_TABLE_NAME}}": f"{self.config.get_project_name()}-jobs",
2903 "{{INFERENCE_ENDPOINTS_TABLE_NAME}}": (
2904 f"{self.config.get_project_name()}-inference-endpoints"
2905 ),
2906 # DynamoDB region (global stack region, may differ from cluster region)
2907 "{{DYNAMODB_REGION}}": self.config.get_global_region(),
2908 # Global region for cross-region SSM reads/writes (e.g. the health
2909 # monitor's /<project>/alb-hostname-<region> sync in the global region).
2910 "{{GLOBAL_REGION}}": self.config.get_global_region(),
2911 # Manifest processor resource quotas (sourced from shared policy).
2912 "{{MP_MAX_CPU_PER_MANIFEST}}": str(job_quotas.get("max_cpu_per_manifest", "10")),
2913 "{{MP_MAX_MEMORY_PER_MANIFEST}}": str(
2914 job_quotas.get("max_memory_per_manifest", "32Gi")
2915 ),
2916 "{{MP_MAX_GPU_PER_MANIFEST}}": str(job_quotas.get("max_gpu_per_manifest", 4)),
2917 # Require accelerator (GPU/Neuron/EFA) jobs to carry a matching
2918 # toleration (shared policy). Mirrored on the SQS path via
2919 # {{QP_REQUIRE_ACCELERATOR_TOLERATION}} so neither path is a bypass.
2920 "{{MP_REQUIRE_ACCELERATOR_TOLERATION}}": (
2921 "true" if job_policy.get("require_accelerator_toleration", True) else "false"
2922 ),
2923 # Manifest processor namespace allowlist (sourced from shared policy).
2924 # Both the REST manifest processor and the SQS queue processor
2925 # read from job_validation_policy.allowed_namespaces so a single
2926 # edit takes effect on both submission paths at the next deploy.
2927 "{{MP_ALLOWED_NAMESPACES}}": ",".join(
2928 job_policy.get("allowed_namespaces", ["gco-jobs"])
2929 ),
2930 # Manifest processor Kubernetes resource kind allowlist (shared policy).
2931 "{{MP_ALLOWED_KINDS}}": ",".join(allowed_kinds),
2932 # Manifest processor image registry allowlist (sourced from shared
2933 # policy). Augmented with the project's own ECR registry hostnames
2934 # so jobs built via ``gco images build`` aren't rejected by the
2935 # REST submission path. Identical augmentation runs on the SQS
2936 # path below — see ``{{QP_TRUSTED_REGISTRIES}}``.
2937 "{{MP_TRUSTED_REGISTRIES}}": ",".join(
2938 _augment_trusted_registries_with_project_ecr(
2939 job_policy.get("trusted_registries", []),
2940 account=self.account,
2941 regions=self.config.get_regions(),
2942 global_region=self.config.get_global_region(),
2943 url_suffix=self.url_suffix,
2944 )
2945 ),
2946 "{{MP_TRUSTED_DOCKERHUB_ORGS}}": ",".join(job_policy.get("trusted_dockerhub_orgs", [])),
2947 # Manifest processor request body size cap (HTTP 413 middleware).
2948 # Lives at cdk.json::manifest_processor.max_request_body_bytes.
2949 "{{MP_MAX_REQUEST_BODY_BYTES}}": str(
2950 mp_config.get("max_request_body_bytes", 1_048_576)
2951 ),
2952 # Inference request bodies use the same operator-configured cap,
2953 # but retain a service-specific placeholder for future tuning.
2954 "{{INFERENCE_PROXY_MAX_REQUEST_BODY_BYTES}}": str(
2955 mp_config.get("max_request_body_bytes", 1_048_576)
2956 ),
2957 # Regional worker for the DynamoDB-backed global queue. Multiple API
2958 # replicas are safe because JobStore claims are conditional and
2959 # lease-backed; each replica also reconciles K8s status transitions.
2960 "{{CENTRAL_QUEUE_WORKER_ENABLED}}": (
2961 "true" if mp_config.get("central_queue_worker_enabled", True) else "false"
2962 ),
2963 "{{CENTRAL_QUEUE_POLL_INTERVAL_SECONDS}}": str(
2964 mp_config.get("central_queue_poll_interval_seconds", 10)
2965 ),
2966 "{{CENTRAL_QUEUE_BATCH_SIZE}}": str(mp_config.get("central_queue_batch_size", 5)),
2967 "{{CENTRAL_QUEUE_RECONCILE_LIMIT}}": str(
2968 mp_config.get("central_queue_reconcile_limit", 100)
2969 ),
2970 "{{CENTRAL_QUEUE_LEASE_SECONDS}}": str(
2971 mp_config.get("central_queue_lease_seconds", 300)
2972 ),
2973 "{{CENTRAL_QUEUE_LEASE_RENEWAL_SECONDS}}": str(
2974 mp_config.get("central_queue_lease_renewal_seconds", 60)
2975 ),
2976 "{{QUEUE_TARGET_REGIONS}}": ",".join(self.config.get_regions()),
2977 }
2979 # Always-on Cluster_Shared_Bucket replacements. Populated from the
2980 # SharedBucketIdentity resolved in __init__ via cross-region SSM
2981 # read from GCOGlobalStack. Never gated on the analytics toggle —
2982 # the gco-cluster-shared-bucket ConfigMap is applied to every
2983 # regional cluster.
2984 image_replacements.update(
2985 _compute_kubectl_cluster_shared_replacements(self.cluster_shared_identity)
2986 )
2988 # Cluster observability (on by default): gate the gp3 StorageClass and
2989 # the ServiceMonitors/dashboards on the toggle. When enabled the gating
2990 # placeholders resolve so those manifests apply; when disabled the keys
2991 # are absent, so the manifests keep an unreplaced placeholder and the
2992 # applier skips them (same mechanism FSx/Valkey use).
2993 _obs_config = self.config.get_cluster_observability_config()
2994 image_replacements.update(
2995 _compute_kubectl_observability_replacements(
2996 bool(_obs_config["enabled"]),
2997 grafana_admin_password_rotation_schedule=str(
2998 _obs_config["grafana"]["admin_password_rotation_schedule"]
2999 ),
3000 )
3001 )
3003 # Cost monitoring (on by default): gate the cost-monitor Deployment
3004 # and the Grafana cost dashboard on the toggle via the same
3005 # unreplaced-placeholder mechanism. The image/role placeholders exist
3006 # only when the pipeline is active, so a disabled deployment leaves
3007 # 34-cost-monitor.yaml and the cost dashboard unapplied.
3008 if self._cost_monitoring_active(): 3008 ↛ 3027line 3008 didn't jump to line 3027 because the condition on line 3008 was always true
3009 _cost_config = self.config.get_cost_monitoring_config()
3010 image_replacements.update(
3011 {
3012 "{{COST_MONITORING_ENABLED}}": "true",
3013 "{{COST_MONITOR_IMAGE}}": self.cost_monitor_image.image_uri,
3014 "{{COST_MONITOR_ROLE_ARN}}": self.cost_monitor_role.role_arn,
3015 "{{COST_REPORT_BUCKET}}": cost_report_bucket_name(
3016 self.config.get_project_name(),
3017 self.account,
3018 self.config.get_monitoring_region(),
3019 ),
3020 "{{COST_REPORT_INTERVAL_MINUTES}}": str(
3021 _cost_config["reports"]["interval_minutes"]
3022 ),
3023 }
3024 )
3026 # Add queue processor replacements if enabled
3027 qp_config = self.node.try_get_context("queue_processor") or {}
3029 # Add VPC endpoint CIDR replacements for network policy restrictions
3030 # Generates a YAML block of ipBlock entries from the vpc_endpoint_cidrs array.
3031 # The placeholder {{VPC_ENDPOINT_CIDR_BLOCKS}} sits at 8-space indentation in
3032 # the manifest, so the first entry needs no leading indent (the manifest provides
3033 # it) and subsequent entries are indented to align.
3034 vpc_endpoint_cidrs = self.node.try_get_context("vpc_endpoint_cidrs") or ["10.0.0.0/16"]
3035 cidr_lines = []
3036 for i, cidr in enumerate(vpc_endpoint_cidrs):
3037 prefix = "" if i == 0 else " "
3038 cidr_lines.append(f'{prefix}- ipBlock:\n cidr: "{cidr}"')
3039 image_replacements["{{VPC_ENDPOINT_CIDR_BLOCKS}}"] = "\n".join(cidr_lines)
3041 # Resource governance for gco-jobs namespace: ResourceQuota caps aggregate
3042 # resource consumption across the namespace, LimitRange caps per-container
3043 # maxima. Values come from cdk.json `resource_quota` context with defaults
3044 # sized for a modest multi-tenant dev cluster.
3045 resource_quota = self.node.try_get_context("resource_quota") or {}
3046 image_replacements["{{QUOTA_MAX_CPU}}"] = str(resource_quota.get("max_cpu", "100"))
3047 image_replacements["{{QUOTA_MAX_MEMORY}}"] = str(resource_quota.get("max_memory", "512Gi"))
3048 image_replacements["{{QUOTA_MAX_GPU}}"] = str(resource_quota.get("max_gpu", "32"))
3049 image_replacements["{{QUOTA_MAX_PODS}}"] = str(resource_quota.get("max_pods", "50"))
3050 image_replacements["{{LIMIT_MAX_CPU}}"] = str(resource_quota.get("container_max_cpu", "10"))
3051 image_replacements["{{LIMIT_MAX_MEMORY}}"] = str(
3052 resource_quota.get("container_max_memory", "64Gi")
3053 )
3054 image_replacements["{{LIMIT_MAX_GPU}}"] = str(resource_quota.get("container_max_gpu", "4"))
3056 if self.queue_processor_enabled:
3057 image_replacements["{{QUEUE_PROCESSOR_IMAGE}}"] = self.queue_processor_image.image_uri
3058 image_replacements["{{QP_POLLING_INTERVAL}}"] = str(
3059 qp_config.get("polling_interval", 10)
3060 )
3061 image_replacements["{{QP_MAX_CONCURRENT_JOBS}}"] = str(
3062 qp_config.get("max_concurrent_jobs", 10)
3063 )
3064 image_replacements["{{QP_MESSAGES_PER_JOB}}"] = str(
3065 qp_config.get("messages_per_job", 1)
3066 )
3067 image_replacements["{{QP_SUCCESSFUL_JOBS_HISTORY}}"] = str(
3068 qp_config.get("successful_jobs_history", 20)
3069 )
3070 image_replacements["{{QP_FAILED_JOBS_HISTORY}}"] = str(
3071 qp_config.get("failed_jobs_history", 10)
3072 )
3073 image_replacements["{{QP_ALLOWED_NAMESPACES}}"] = ",".join(
3074 job_policy.get("allowed_namespaces", ["gco-jobs"])
3075 )
3076 image_replacements["{{QP_ALLOWED_KINDS}}"] = ",".join(allowed_kinds)
3077 # Resource caps, image allowlist, and security policy are shared
3078 # with the REST manifest processor. Source them from the
3079 # job_validation_policy section so a single change in cdk.json
3080 # takes effect on both submission paths at the next deploy.
3081 image_replacements["{{QP_MAX_GPU_PER_MANIFEST}}"] = str(
3082 job_quotas.get("max_gpu_per_manifest", 4)
3083 )
3084 image_replacements["{{QP_MAX_CPU_PER_MANIFEST}}"] = str(
3085 job_quotas.get("max_cpu_per_manifest", "10")
3086 )
3087 image_replacements["{{QP_MAX_MEMORY_PER_MANIFEST}}"] = str(
3088 job_quotas.get("max_memory_per_manifest", "32Gi")
3089 )
3090 image_replacements["{{QP_TRUSTED_REGISTRIES}}"] = ",".join(
3091 _augment_trusted_registries_with_project_ecr(
3092 job_policy.get("trusted_registries", []),
3093 account=self.account,
3094 regions=self.config.get_regions(),
3095 global_region=self.config.get_global_region(),
3096 url_suffix=self.url_suffix,
3097 )
3098 )
3099 image_replacements["{{QP_TRUSTED_DOCKERHUB_ORGS}}"] = ",".join(
3100 job_policy.get("trusted_dockerhub_orgs", [])
3101 )
3103 # Security policy toggles — shared with the REST manifest_processor.
3104 # Both services read the same cdk.json section so a single policy
3105 # flip (e.g. block_run_as_root: true) takes effect on both paths.
3106 security_policy = job_policy.get("manifest_security_policy", {})
3108 def _policy_str(v: object) -> str:
3109 return "true" if v else "false"
3111 image_replacements["{{QP_BLOCK_PRIVILEGED}}"] = _policy_str(
3112 security_policy.get("block_privileged", True)
3113 )
3114 image_replacements["{{QP_BLOCK_PRIVILEGE_ESCALATION}}"] = _policy_str(
3115 security_policy.get("block_privilege_escalation", True)
3116 )
3117 image_replacements["{{QP_BLOCK_HOST_NETWORK}}"] = _policy_str(
3118 security_policy.get("block_host_network", True)
3119 )
3120 image_replacements["{{QP_BLOCK_HOST_PID}}"] = _policy_str(
3121 security_policy.get("block_host_pid", True)
3122 )
3123 image_replacements["{{QP_BLOCK_HOST_IPC}}"] = _policy_str(
3124 security_policy.get("block_host_ipc", True)
3125 )
3126 image_replacements["{{QP_BLOCK_HOST_PATH}}"] = _policy_str(
3127 security_policy.get("block_host_path", True)
3128 )
3129 image_replacements["{{QP_BLOCK_ADDED_CAPABILITIES}}"] = _policy_str(
3130 security_policy.get("block_added_capabilities", True)
3131 )
3132 image_replacements["{{QP_BLOCK_RUN_AS_ROOT}}"] = _policy_str(
3133 security_policy.get("block_run_as_root", False)
3134 )
3135 # Require accelerator (GPU/Neuron/EFA) jobs to carry a matching
3136 # toleration — shared with the REST manifest_processor via
3137 # {{MP_REQUIRE_ACCELERATOR_TOLERATION}}.
3138 image_replacements["{{QP_REQUIRE_ACCELERATOR_TOLERATION}}"] = _policy_str(
3139 job_policy.get("require_accelerator_toleration", True)
3140 )
3142 # Add Valkey endpoint if enabled
3143 if hasattr(self, "valkey_cache") and self.valkey_cache:
3144 image_replacements["{{VALKEY_ENDPOINT}}"] = self.valkey_cache.attr_endpoint_address
3145 image_replacements["{{VALKEY_PORT}}"] = self.valkey_cache.attr_endpoint_port
3147 # Add Aurora pgvector endpoint if enabled
3148 if hasattr(self, "aurora_cluster") and self.aurora_cluster:
3149 image_replacements["{{AURORA_PGVECTOR_ENDPOINT}}"] = (
3150 self.aurora_cluster.cluster_endpoint.hostname
3151 )
3152 image_replacements["{{AURORA_PGVECTOR_READER_ENDPOINT}}"] = (
3153 self.aurora_cluster.cluster_read_endpoint.hostname
3154 )
3155 image_replacements["{{AURORA_PGVECTOR_PORT}}"] = str(
3156 self.aurora_cluster.cluster_endpoint.port
3157 )
3158 if self.aurora_cluster.secret: 3158 ↛ 3164line 3158 didn't jump to line 3164 because the condition on line 3158 was always true
3159 image_replacements["{{AURORA_PGVECTOR_SECRET_ARN}}"] = (
3160 self.aurora_cluster.secret.secret_arn
3161 )
3163 # Add FSx replacements if enabled
3164 if self.fsx_file_system:
3165 image_replacements["{{FSX_FILE_SYSTEM_ID}}"] = self.fsx_file_system.ref
3166 image_replacements["{{FSX_DNS_NAME}}"] = self.fsx_file_system.attr_dns_name
3167 image_replacements["{{FSX_MOUNT_NAME}}"] = self.fsx_file_system.attr_lustre_mount_name
3168 image_replacements["{{PRIVATE_SUBNET_ID}}"] = self.vpc.private_subnets[0].subnet_id
3169 image_replacements["{{FSX_SECURITY_GROUP_ID}}"] = (
3170 self.fsx_security_group.security_group_id
3171 )
3173 # ── Trigger the convergence pipeline (fire-and-forget) ───────────────
3174 # A single custom resource starts the HelmInstallStateMachine, which now
3175 # owns the WHOLE cluster convergence: apply base manifests -> install
3176 # Helm charts -> apply post-Helm (CRD-dependent) manifests -> publish
3177 # the Gateway-created ALB and optionally register it with Global
3178 # Accelerator. The resource returns
3179 # as soon as the execution is *started* (no isComplete waiter), so the
3180 # cluster's CloudFormation lifecycle is never bound to the multi-minute
3181 # add-on convergence — a slow chart can't blow CloudFormation's ~1h
3182 # custom-resource ceiling and roll back (destroy) the freshly-created
3183 # cluster. Status lives in SSM and is surfaced via `gco stacks addons
3184 # status`; re-converge out-of-band with `gco stacks addons install`.
3185 #
3186 # The execution input carries everything the state-machine tasks need:
3187 # chart selection/overrides, the manifest ImageReplacements (for the base
3188 # and post-Helm kubectl passes), the endpoint-registry identity, and the
3189 # optional Global Accelerator EndpointGroupArn.
3190 convergence_properties: dict[str, Any] = {
3191 "ClusterName": self.cluster.cluster_name,
3192 "Region": self.deployment_region,
3193 # Helm chart selection + per-chart value overrides (e.g. Volcano
3194 # image_registry redirected to the ECR mirror when enabled).
3195 "EnabledCharts": self._get_enabled_helm_charts(),
3196 "Charts": self._helm_chart_value_overrides(),
3197 "KedaOperatorRoleArn": self.keda_operator_role.role_arn,
3198 # Template substitutions for the base + post-Helm kubectl passes.
3199 "ImageReplacements": image_replacements,
3200 # Project name lets the orchestrator persist the execution input
3201 # to SSM so `gco stacks addons install` can replay the whole
3202 # pipeline without reconstructing chart/manifest config.
3203 "ProjectName": self.config.get_project_name(),
3204 "RegistryRegion": self.config.get_global_region(),
3205 # Force re-invocation on every deployment (new charts.yaml,
3206 # manifest, or image) so convergence re-runs end to end.
3207 "DeploymentTimestamp": deployment_timestamp,
3208 }
3209 if self.global_accelerator_enabled:
3210 convergence_properties["EndpointGroupArn"] = self.endpoint_group_arn
3212 converge_trigger = CustomResource(
3213 self,
3214 "HelmInstallCharts",
3215 service_token=self.helm_installer_provider.service_token,
3216 properties=convergence_properties,
3217 )
3218 converge_trigger.node.add_dependency(self.helm_installer_provider_log_group)
3219 converge_trigger.node.add_dependency(self.aws_load_balancer_controller_policy)
3221 # The trigger (and therefore the whole convergence pipeline) must run
3222 # after the cluster, shared storage, managed-addon IRSA patches, and Pod
3223 # Identity associations exist: the base manifests reference their tokens,
3224 # and the rollout-restarts at the end of the base pass need the patched
3225 # service accounts (otherwise the mutating webhook can't inject
3226 # AWS_ROLE_ARN and the controllers fail with "no EC2 IMDS role found" —
3227 # PVCs stuck Pending, missing Container Insights metrics; see the
3228 # UpdateEfsCsiAddonRole resource in _create_efs_csi_driver_addon). These
3229 # gates previously sat on the synchronous KubectlApplyManifests custom
3230 # resource; the base apply now lives in the state machine, so the gate
3231 # moves to the trigger.
3232 converge_trigger.node.add_dependency(self.cluster)
3233 converge_trigger.node.add_dependency(self.efs_file_system)
3234 if self.fsx_file_system:
3235 converge_trigger.node.add_dependency(self.fsx_file_system)
3236 for attr in (
3237 "_efs_csi_addon_role_update",
3238 "_fsx_csi_addon_role_update",
3239 "_cloudwatch_addon_role_update",
3240 ):
3241 update_cr = getattr(self, attr, None)
3242 if update_cr is not None:
3243 converge_trigger.node.add_dependency(update_cr)
3244 # The trigger also needs both EKS access entries before it starts the
3245 # asynchronous pipeline. Keeping these explicit is essential on delete:
3246 # the ordered Helm teardown runs while its Kubernetes authentication is
3247 # still valid, then the trigger/access entries/cluster can disappear.
3248 for attr in (
3249 "kubectl_lambda_access_entry",
3250 "helm_installer_access_entry",
3251 "ga_registration_access_entry",
3252 ):
3253 access_entry = getattr(self, attr, None)
3254 if access_entry is not None:
3255 converge_trigger.node.add_dependency(access_entry)
3256 for assoc in self._pod_identity_associations:
3257 converge_trigger.node.add_dependency(assoc)
3259 # Deletion must run in the opposite safety order: synchronous Helm
3260 # teardown first (quiescing endpoint writers and removing Gateway
3261 # resources), then the unconditional endpoint deregistration guard,
3262 # then the convergence trigger and its EKS access entries. Build the
3263 # create-time chain as trigger -> endpoint guard -> Helm teardown so
3264 # CloudFormation reverses it during stack deletion.
3265 helm_teardown = getattr(self, "helm_teardown_resource", None)
3266 ga_deregistration = getattr(self, "ga_deregistration_resource", None)
3267 if helm_teardown is not None:
3268 if ga_deregistration is not None: 3268 ↛ 3272line 3268 didn't jump to line 3272 because the condition on line 3268 was always true
3269 helm_teardown.node.add_dependency(ga_deregistration)
3270 ga_deregistration.node.add_dependency(converge_trigger)
3271 else:
3272 helm_teardown.node.add_dependency(converge_trigger)
3273 elif ga_deregistration is not None: 3273 ↛ exitline 3273 didn't return from function '_apply_kubernetes_manifests' because the condition on line 3273 was always true
3274 ga_deregistration.node.add_dependency(converge_trigger)
3276 def _create_ga_registration_lambda(self) -> None:
3277 """Create the exact Gateway ALB discovery and endpoint-publication Lambda.
3279 Every partition uses this function to discover ``gco-system/gco-gateway``
3280 and publish its internal ALB hostname to the regional SSM registry.
3281 Commercial partitions additionally pass an endpoint-group ARN so the
3282 same exact ALB is registered with Global Accelerator.
3283 """
3284 project_name = self.config.get_project_name()
3286 # Create Lambda function for GA registration using external handler
3287 ga_registration_lambda = lambda_.Function(
3288 self,
3289 "GaRegistrationFunction",
3290 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
3291 handler="handler.lambda_handler",
3292 code=lambda_.Code.from_asset("lambda/ga-registration"),
3293 timeout=Duration.minutes(15), # Max Lambda timeout; handler uses 14 min budget
3294 memory_size=256,
3295 vpc=self.vpc,
3296 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
3297 environment={
3298 "CLUSTER_NAME": self.cluster.cluster_name,
3299 "REGION": self.deployment_region,
3300 },
3301 tracing=lambda_.Tracing.ACTIVE,
3302 )
3304 # Grant permissions
3305 ga_registration_lambda.add_to_role_policy(
3306 iam.PolicyStatement(
3307 effect=iam.Effect.ALLOW,
3308 actions=["eks:DescribeCluster"],
3309 resources=[self.cluster.cluster_arn],
3310 )
3311 )
3312 ga_registration_lambda.add_to_role_policy(
3313 iam.PolicyStatement(
3314 effect=iam.Effect.ALLOW,
3315 actions=[
3316 "elasticloadbalancing:DescribeLoadBalancers",
3317 "elasticloadbalancing:DescribeTags", # Required for tag-based ALB detection
3318 ],
3319 resources=["*"],
3320 )
3321 )
3322 if self.global_accelerator_enabled:
3323 ga_registration_lambda.add_to_role_policy(
3324 iam.PolicyStatement(
3325 effect=iam.Effect.ALLOW,
3326 actions=[
3327 "globalaccelerator:AddEndpoints",
3328 "globalaccelerator:RemoveEndpoints",
3329 "globalaccelerator:UpdateEndpointGroup",
3330 "globalaccelerator:DescribeEndpointGroup",
3331 # The teardown-time cleanup_gateway_endpoint task runs
3332 # on this Lambda and strictly waits for the accelerator
3333 # to reach DEPLOYED after endpoint removal.
3334 "globalaccelerator:DescribeAccelerator",
3335 ],
3336 resources=["*"],
3337 )
3338 )
3339 ga_registration_lambda.add_to_role_policy(
3340 iam.PolicyStatement(
3341 effect=iam.Effect.ALLOW,
3342 actions=["ssm:GetParameter", "ssm:PutParameter", "ssm:DeleteParameter"],
3343 resources=[
3344 f"arn:{self.partition}:ssm:{self.config.get_global_region()}:"
3345 f"{self.account}:parameter/{project_name}/*"
3346 ],
3347 )
3348 )
3350 # Retain the access entry so asynchronous convergence cannot start
3351 # until the endpoint publisher can read the exact Gateway object.
3352 self.ga_registration_access_entry = None
3353 if ga_registration_lambda.role is not None: 3353 ↛ 3367line 3353 didn't jump to line 3367 because the condition on line 3353 was always true
3354 self.ga_registration_access_entry = eks.AccessEntry(
3355 self,
3356 "GaRegistrationLambdaAccessEntry",
3357 cluster=self.cluster, # type: ignore[arg-type]
3358 principal=ga_registration_lambda.role.role_arn,
3359 access_policies=[
3360 eks.AccessPolicy.from_access_policy_name(
3361 "AmazonEKSClusterAdminPolicy", access_scope_type=eks.AccessScopeType.CLUSTER
3362 )
3363 ],
3364 )
3366 # Allow Lambda to access EKS API
3367 self.cluster.cluster_security_group.add_ingress_rule(
3368 peer=ec2.Peer.ipv4(self.vpc.vpc_cidr_block),
3369 connection=ec2.Port.tcp(443),
3370 description="Allow GA registration Lambda to access EKS API",
3371 )
3373 # Global Accelerator exists only in supported partitions. Resolve its
3374 # endpoint group lazily there; regional endpoint publication itself is
3375 # unconditional and needs no GA lookup.
3376 self.endpoint_group_arn: str | None = None
3377 if self.global_accelerator_enabled:
3378 global_region = self.config.get_global_region()
3379 get_endpoint_group_arn = cr.AwsCustomResource(
3380 self,
3381 "GetEndpointGroupArn",
3382 on_create=cr.AwsSdkCall(
3383 service="SSM",
3384 action="getParameter",
3385 parameters={
3386 "Name": f"/{project_name}/endpoint-group-{self.deployment_region}-arn"
3387 },
3388 region=global_region,
3389 physical_resource_id=cr.PhysicalResourceId.of(
3390 f"{project_name}-get-endpoint-group-arn-{self.deployment_region}"
3391 ),
3392 ),
3393 on_update=cr.AwsSdkCall(
3394 service="SSM",
3395 action="getParameter",
3396 parameters={
3397 "Name": f"/{project_name}/endpoint-group-{self.deployment_region}-arn"
3398 },
3399 region=global_region,
3400 ),
3401 role=self.aws_custom_resource_role,
3402 )
3403 get_endpoint_group_arn.node.add_dependency(self.aws_custom_resource_role)
3404 self.endpoint_group_arn = get_endpoint_group_arn.get_response_field("Parameter.Value")
3406 # Invoked directly by the convergence state machine's final task.
3407 self.ga_registration_lambda = ga_registration_lambda
3409 # cdk-nag suppression: the GA registration Lambda needs broad
3410 # Global Accelerator and ELB Describe access with Resource: *.
3411 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3413 acknowledge_nag_findings(
3414 ga_registration_lambda,
3415 [
3416 {
3417 "id": "AwsSolutions-IAM5",
3418 "reason": (
3419 "The endpoint-publication Lambda needs ELB Describe access to "
3420 "resolve the exact Gateway-owned ALB. In partitions with Global "
3421 "Accelerator it also needs the service's endpoint-group mutation "
3422 "APIs. These APIs do not support resource-level scoping."
3423 ),
3424 "appliesTo": ["Resource::*"],
3425 },
3426 ],
3427 )
3429 # Wire the delete-time teardown guard that deregisters this region's ALB
3430 # from Global Accelerator before its VPC subnets are deleted.
3431 self._create_ga_deregistration_resource()
3433 def _create_ga_deregistration_resource(self) -> None:
3434 """Create the unconditional endpoint-registry delete guard.
3436 On stack deletion the guard always removes this region's SSM hostname.
3437 When an endpoint group exists it first deregisters the ALB and waits for
3438 Global Accelerator to release its managed ENIs. The resource therefore
3439 exists in every partition even though the GA portion is optional.
3440 """
3441 project_name = self.config.get_project_name()
3443 # Dedicated Lambda built from the SAME asset as the registration Lambda
3444 # (it reuses the shared remove/wait helpers, entry point
3445 # handler.on_delete_event). Deliberately NOT in the VPC: it only calls
3446 # the public Global Accelerator API and must not create its own ENIs in
3447 # the VPC it is helping to tear down.
3448 ga_deregistration_lambda = lambda_.Function(
3449 self,
3450 "GaDeregistrationFunction",
3451 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
3452 handler="handler.on_delete_event",
3453 code=lambda_.Code.from_asset("lambda/ga-registration"),
3454 timeout=Duration.minutes(15), # covers the GA redeploy wait budget
3455 memory_size=256,
3456 tracing=lambda_.Tracing.ACTIVE,
3457 )
3458 if self.global_accelerator_enabled:
3459 ga_deregistration_lambda.add_to_role_policy(
3460 iam.PolicyStatement(
3461 effect=iam.Effect.ALLOW,
3462 actions=[
3463 "globalaccelerator:DescribeAccelerator",
3464 "globalaccelerator:DescribeEndpointGroup",
3465 "globalaccelerator:RemoveEndpoints",
3466 "globalaccelerator:UpdateEndpointGroup",
3467 ],
3468 resources=["*"],
3469 )
3470 )
3472 ga_deregistration_lambda.add_to_role_policy(
3473 iam.PolicyStatement(
3474 effect=iam.Effect.ALLOW,
3475 actions=["ssm:DeleteParameter"],
3476 resources=[
3477 f"arn:{self.partition}:ssm:{self.config.get_global_region()}:{self.account}:"
3478 f"parameter/{project_name}/alb-hostname-{self.deployment_region}"
3479 ],
3480 )
3481 )
3483 # Strict live validation retains the exact generation through the
3484 # provider's final delete invocation; its identity-fenced post-stack
3485 # cleanup removes it. Ordinary deployments retain DESTROY semantics.
3486 ga_deregistration_log_group = logs.LogGroup(
3487 self,
3488 "GaDeregistrationProviderLogGroup",
3489 retention=logs.RetentionDays.ONE_WEEK,
3490 removal_policy=self.provider_log_group_removal_policy,
3491 )
3492 ga_deregistration_provider = cr.Provider(
3493 self,
3494 "GaDeregistrationProvider",
3495 on_event_handler=ga_deregistration_lambda,
3496 log_group=ga_deregistration_log_group,
3497 )
3499 deregistration_properties: dict[str, Any] = {
3500 "Region": self.deployment_region,
3501 "RegistryRegion": self.config.get_global_region(),
3502 "ProjectName": project_name,
3503 }
3504 if self.endpoint_group_arn is not None:
3505 deregistration_properties["EndpointGroupArn"] = self.endpoint_group_arn
3507 ga_deregistration = CustomResource(
3508 self,
3509 "GaDeregistration",
3510 service_token=ga_deregistration_provider.service_token,
3511 properties=deregistration_properties,
3512 )
3514 # Teardown ordering: this deregistration must run BEFORE the VPC (and its
3515 # public subnets, where Global Accelerator pins its managed ENIs) is
3516 # deleted. Depending on the VPC means CloudFormation creates the VPC
3517 # first and — critically — deletes this custom resource first on
3518 # teardown, releasing the GA ENIs so the subnets can be removed cleanly.
3519 self.ga_deregistration_resource = ga_deregistration
3520 ga_deregistration.node.add_dependency(self.vpc)
3521 ga_deregistration.node.add_dependency(ga_deregistration_log_group)
3523 # cdk-nag: the deregistration Lambda needs globalaccelerator Describe*/
3524 # RemoveEndpoints with Resource: * (these Global Accelerator APIs do not
3525 # support resource-level IAM scoping), mirroring the registration Lambda.
3526 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3528 acknowledge_nag_findings(
3529 ga_deregistration_lambda,
3530 [
3531 {
3532 "id": "AwsSolutions-IAM5",
3533 "reason": (
3534 "Where Global Accelerator is enabled, the delete guard needs its "
3535 "Describe and endpoint-group mutation APIs to release managed ENIs. "
3536 "Those APIs do not support resource-level IAM scoping; non-GA "
3537 "partitions receive no Global Accelerator actions."
3538 ),
3539 "appliesTo": ["Resource::*"],
3540 },
3541 ],
3542 )
3544 def _get_volcano_image_mirror_config(self) -> dict[str, Any]:
3545 """Parse the ``volcano_image_mirror`` block from cdk.json.
3547 Returns a normalized dict ``{enabled, ecr_namespace}``. Validation is
3548 strict so a misconfiguration fails at synth rather than silently leaving
3549 Volcano pointed at docker.io.
3551 - ``enabled`` (default False) — master toggle.
3552 - ``ecr_namespace`` (default ``"<project_name>/dockerhub"``, i.e.
3553 ``gco/dockerhub`` for the stock project) — the ECR repository
3554 namespace the mirrored Volcano images live under. Must start with
3555 ``<project_name>/`` so it inherits the project's existing
3556 ``<project_name>/*`` machinery (node pull access, replication rule,
3557 trusted-registry allow-list) with no extra IAM, and must be a valid
3558 (possibly nested) ECR repository path.
3559 """
3560 raw = self.node.try_get_context("volcano_image_mirror") or {}
3561 if not isinstance(raw, dict): 3561 ↛ 3562line 3561 didn't jump to line 3562 because the condition on line 3561 was never true
3562 raise ValueError(f"volcano_image_mirror must be a mapping, got {type(raw).__name__}")
3564 # The mirror namespace lives under this deployment's project prefix
3565 # (``<project_name>/``) so it inherits the project's ECR access,
3566 # replication rule, and trusted-registry allow-list (#139). Defaults to
3567 # ``<project_name>/dockerhub`` — ``gco/dockerhub`` for the stock project.
3568 project_prefix = f"{self.config.get_project_name()}/"
3569 enabled = bool(raw.get("enabled", False))
3570 ecr_namespace = (
3571 str(raw.get("ecr_namespace", f"{project_prefix}dockerhub")).strip().strip("/")
3572 )
3574 if not enabled:
3575 return {"enabled": False, "ecr_namespace": ecr_namespace}
3577 # Must live under the project prefix and be a valid nested ECR repo
3578 # path (lowercase alphanumerics + . _ - per segment, slash-separated).
3579 if not ecr_namespace.startswith(project_prefix):
3580 raise ValueError(
3581 f"volcano_image_mirror.ecr_namespace must start with {project_prefix!r} so it "
3582 f"inherits the project's {project_prefix}* ECR access/replication, got "
3583 f"{ecr_namespace!r}"
3584 )
3585 segment = r"[a-z0-9]+(?:[._-][a-z0-9]+)*"
3586 if not re.fullmatch(rf"{segment}(?:/{segment})+", ecr_namespace):
3587 raise ValueError(
3588 "volcano_image_mirror.ecr_namespace must be a valid ECR repository "
3589 f"path (lowercase alphanumerics + . _ - per slash-separated segment), "
3590 f"got {ecr_namespace!r}"
3591 )
3593 return {"enabled": True, "ecr_namespace": ecr_namespace}
3595 def _configure_volcano_image_mirror(self) -> None:
3596 """Resolve the optional Volcano image-mirror registry (no infra).
3598 Volcano is the only default chart whose images live exclusively on
3599 docker.io (``volcanosh/vc-*``). On a cold EKS Auto Mode cluster those
3600 anonymous pulls are slow / rate-limited, so Volcano's blocking
3601 ``helm --wait`` could never finish inside the installer Lambda's
3602 wall-clock guard and the whole add-on batch looped on it.
3604 The fix is to mirror Volcano's pinned images into the project's own ECR
3605 under ``gco/*`` and point Volcano's ``basic.image_registry`` there, so
3606 the cluster makes fast, same-account ECR pulls with the pull-only node
3607 role it already has — no Docker Hub credential, no pull-through cache
3608 rule, and no registry permissions policy. The mirror itself is populated
3609 out-of-band (``gco images mirror``) before the add-ons
3610 converge; this method only computes the registry override and creates no
3611 CloudFormation resources.
3613 Sets ``self.volcano_mirror_registry`` to
3614 ``<account>.dkr.ecr.<region>.<url-suffix>/<ecr_namespace>`` that
3615 ``_helm_chart_value_overrides`` feeds into Volcano's
3616 ``basic.image_registry``; left ``None`` when disabled.
3617 """
3618 # Always define the attribute so downstream code can branch on it.
3619 self.volcano_mirror_registry: str | None = None
3621 cfg = self._get_volcano_image_mirror_config()
3622 if not cfg["enabled"]:
3623 return
3625 ecr_namespace = cfg["ecr_namespace"]
3626 self.volcano_mirror_registry = (
3627 f"{self.account}.dkr.ecr.{self.deployment_region}.{self.url_suffix}/{ecr_namespace}"
3628 )
3630 def _helm_chart_value_overrides(self) -> dict[str, Any]:
3631 """Per-chart helm value overrides injected into the install payload.
3633 Returned dict is forwarded verbatim as the ``Charts`` property of the
3634 ``HelmInstallCharts`` custom resource; the installer deep-merges each
3635 chart's ``values`` over ``charts.yaml``. The mandatory
3636 ``aws-load-balancer-controller`` chart always receives the cluster,
3637 region, VPC, and dedicated IRSA role values. Optional overrides are:
3639 - ``volcano``: point ``basic.image_registry`` at the project's ECR
3640 image mirror when enabled, so every Volcano image (controller,
3641 scheduler, admission webhook, and the pre-install admission-init
3642 hook) resolves from ECR instead of docker.io. The upstream names
3643 (``volcanosh/vc-*``) are preserved, so each resolves to
3644 ``<mirror_registry>/volcanosh/vc-*``.
3645 - ``kube-prometheus-stack``: inject the ``cdk.json``-derived dynamic
3646 values (Grafana/Prometheus/Alertmanager persistence sizes, Prometheus
3647 retention, the gp3 ``storageClassName``, and the GPU/Neuron/EFA
3648 node-exporter tolerations) over the static hardening values in
3649 ``charts.yaml`` when ``cluster_observability.enabled`` is true.
3651 The result is never empty because Gateway API requires the controller.
3652 """
3653 overrides: dict[str, Any] = {
3654 "aws-load-balancer-controller": {
3655 "values": {
3656 "clusterName": self.cluster.cluster_name,
3657 "region": self.deployment_region,
3658 "vpcId": self.vpc.vpc_id,
3659 "serviceAccount": {
3660 "annotations": {
3661 "eks.amazonaws.com/role-arn": (
3662 self.aws_load_balancer_controller_role.role_arn
3663 )
3664 }
3665 },
3666 }
3667 }
3668 }
3670 if getattr(self, "volcano_mirror_registry", None):
3671 overrides["volcano"] = {
3672 "values": {
3673 "basic": {
3674 "image_registry": self.volcano_mirror_registry,
3675 }
3676 }
3677 }
3679 if self.config.get_cluster_observability_enabled():
3680 overrides["kube-prometheus-stack"] = self._observability_chart_values()
3682 if self._cost_monitoring_active():
3683 overrides["opencost"] = self._opencost_chart_values()
3685 return overrides
3687 def _cost_monitoring_active(self) -> bool:
3688 """Return whether the per-region cost monitoring pipeline deploys here.
3690 Delegates to ``ConfigLoader.get_cost_monitoring_enabled``, which is
3691 already the conjunction of the ``cost_monitoring`` toggle and its
3692 ``cluster_observability`` data-source dependency — disabling either
3693 switches OpenCost, the cost-monitor service, and the cost dashboard
3694 off together.
3695 """
3696 return self.config.get_cost_monitoring_enabled()
3698 def _opencost_chart_values(self) -> dict[str, Any]:
3699 """Build the OpenCost value overrides that carry deployment tokens.
3701 Only the cluster identity is dynamic — every static hardening value
3702 (Prometheus wiring, ServiceMonitor, resource limits, security
3703 contexts) lives in ``charts.yaml``. The identity is
3704 ``opencost.exporter.defaultClusterId`` — the value OpenCost stamps on
3705 every allocation row, which is what lets the multi-region Athena data
3706 distinguish clusters. The chart's root-level ``clusterName`` is NOT
3707 set here: that value is the Kubernetes DNS zone (``cluster.local``)
3708 used to build the Prometheus URL, and overriding it with the EKS
3709 cluster name breaks in-cluster DNS resolution.
3710 """
3711 return {
3712 "values": {
3713 "opencost": {
3714 "exporter": {
3715 "defaultClusterId": self.cluster.cluster_name,
3716 },
3717 },
3718 }
3719 }
3721 def _observability_chart_values(self) -> dict[str, Any]:
3722 """Build the kube-prometheus-stack value overrides from cdk.json.
3724 Sizes/retention come from ``cluster_observability`` in ``cdk.json``; the
3725 gp3 ``storageClassName`` is the shared ``_OBSERVABILITY_STORAGE_CLASS``
3726 (also the name of the gated StorageClass manifest), and the
3727 node-exporter tolerations reuse the shared accelerator-node tolerations
3728 so the DaemonSet schedules on tainted GPU/Neuron/EFA nodes. Deep-merged
3729 by the installer over the static hardening values in ``charts.yaml``.
3730 """
3731 obs = self.config.get_cluster_observability_config()
3732 storage_class = _OBSERVABILITY_STORAGE_CLASS
3733 return {
3734 "values": {
3735 "grafana": {
3736 "persistence": {
3737 "storageClassName": storage_class,
3738 "size": obs["grafana"]["persistence_size"],
3739 },
3740 },
3741 "prometheus": {
3742 "prometheusSpec": {
3743 "retention": obs["prometheus"]["retention"],
3744 "storageSpec": {
3745 "volumeClaimTemplate": {
3746 "spec": {
3747 "storageClassName": storage_class,
3748 "resources": {
3749 "requests": {
3750 "storage": obs["prometheus"]["persistence_size"],
3751 },
3752 },
3753 },
3754 },
3755 },
3756 },
3757 },
3758 "alertmanager": {
3759 "enabled": obs["alertmanager"]["enabled"],
3760 "alertmanagerSpec": {
3761 "storage": {
3762 "volumeClaimTemplate": {
3763 "spec": {
3764 "storageClassName": storage_class,
3765 "resources": {
3766 "requests": {
3767 "storage": obs["alertmanager"]["persistence_size"],
3768 },
3769 },
3770 },
3771 },
3772 },
3773 },
3774 },
3775 "prometheus-node-exporter": {
3776 "tolerations": GCORegionalStack._ADDON_NODE_TOLERATIONS,
3777 },
3778 }
3779 }
3781 def _get_enabled_helm_charts(self) -> list[str]:
3782 """Return the list of Helm charts to install based on cdk.json helm config.
3784 Reads the 'helm' section from cdk.json context. Each key maps to one or
3785 more Helm chart names. Charts are returned in dependency order with Kueue
3786 last (its webhook intercepts all Job/Deployment mutations).
3787 """
3788 helm_config = self.node.try_get_context("helm") or {}
3790 # Mapping from cdk.json helm key → Helm chart name(s) in charts.yaml
3791 # Order matters: dependencies first, Kueue last
3792 chart_map: list[tuple[str, list[str]]] = [
3793 ("aws_load_balancer_controller", ["aws-load-balancer-controller"]),
3794 ("keda", ["keda"]),
3795 ("aws_efa_device_plugin", ["aws-efa-device-plugin"]),
3796 ("aws_neuron_device_plugin", ["aws-neuron-device-plugin"]),
3797 ("volcano", ["volcano"]),
3798 ("kuberay", ["kuberay-operator"]),
3799 ("cert_manager", ["cert-manager"]),
3800 ("slurm", ["slinky-slurm-operator", "slinky-slurm"]),
3801 ("yunikorn", ["yunikorn"]),
3802 ("kueue", ["kueue"]), # Must be last
3803 ]
3805 # Charts that are mandatory platform components and cannot be disabled
3806 # via cdk.json. KEDA is always installed: it backs the built-in SQS
3807 # queue processor (a ScaledJob) and is the only metrics bridge that lets
3808 # autoscalers consume GPU/CloudWatch metrics (the keda-metrics-apiserver
3809 # serves external.metrics.k8s.io). Disabling it would silently break
3810 # both, so the cdk.json toggle is ignored for KEDA.
3811 mandatory_chart_keys = {"aws_load_balancer_controller", "keda"}
3813 enabled_charts = []
3814 for config_key, chart_names in chart_map:
3815 chart_config = helm_config.get(config_key, {})
3816 if config_key in mandatory_chart_keys or chart_config.get("enabled", True):
3817 enabled_charts.extend(chart_names)
3819 # kube-prometheus-stack is driven by the separate on-by-default
3820 # cluster_observability toggle (not the helm block), so include it here
3821 # when enabled. Its install order comes from its file position in
3822 # charts.yaml (before kueue), not from where it sits in this list — the
3823 # installer runs one task per chart in charts.yaml order and skips any
3824 # task whose chart is absent from this enabled set.
3825 if self.config.get_cluster_observability_enabled():
3826 enabled_charts.append("kube-prometheus-stack")
3828 # OpenCost is driven by the on-by-default cost_monitoring toggle and
3829 # additionally requires observability (its Prometheus data source).
3830 # charts.yaml places it after kube-prometheus-stack so the Prometheus
3831 # Operator CRDs exist before its ServiceMonitor renders.
3832 if self._cost_monitoring_active():
3833 enabled_charts.append("opencost")
3835 return enabled_charts
3837 def _create_helm_installer_lambda(self) -> None:
3838 """Create Lambda function to install Helm charts (KEDA, NVIDIA DRA, etc.).
3840 This Lambda uses Helm to install charts that require complex setup
3841 (TLS certificates, CRDs, etc.) that are difficult to manage via raw manifests.
3843 Charts installed:
3844 - KEDA: Kubernetes Event-Driven Autoscaling (mandatory, always installed)
3845 - Volcano, KubeRay, Kueue, cert-manager, and other schedulers (toggle via cdk.json)
3846 """
3847 project_name = self.config.get_project_name()
3849 # Create IAM role for Helm installer Lambda
3850 helm_lambda_role = iam.Role(
3851 self,
3852 "HelmInstallerLambdaRole",
3853 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
3854 managed_policies=[
3855 iam.ManagedPolicy.from_aws_managed_policy_name(
3856 "service-role/AWSLambdaVPCAccessExecutionRole"
3857 ),
3858 iam.ManagedPolicy.from_aws_managed_policy_name(
3859 "service-role/AWSLambdaBasicExecutionRole"
3860 ),
3861 ],
3862 )
3864 # Add EKS permissions
3865 helm_lambda_role.add_to_policy(
3866 iam.PolicyStatement(
3867 actions=["eks:DescribeCluster", "eks:ListClusters"],
3868 resources=[self.cluster.cluster_arn],
3869 )
3870 )
3872 # Create security group for Helm installer Lambda
3873 helm_lambda_sg = ec2.SecurityGroup(
3874 self,
3875 "HelmInstallerLambdaSG",
3876 vpc=self.vpc,
3877 description="Security group for Helm installer Lambda to access EKS cluster",
3878 security_group_name=f"{project_name}-helm-lambda-sg-{self.deployment_region}",
3879 allow_all_outbound=True,
3880 )
3882 # Allow Lambda to access EKS cluster API
3883 self.cluster.cluster_security_group.add_ingress_rule(
3884 peer=helm_lambda_sg,
3885 connection=ec2.Port.tcp(443),
3886 description="Allow Helm installer Lambda to access EKS API",
3887 )
3889 # Build Docker image for Helm installer Lambda
3890 # Points at helm-installer-build/ which is rebuilt fresh every deploy
3891 # by _build_helm_installer_lambda() in cli/stacks.py
3892 ecr_assets.DockerImageAsset(
3893 self,
3894 "HelmInstallerImage",
3895 directory="lambda/helm-installer-build",
3896 platform=ecr_assets.Platform.LINUX_AMD64,
3897 )
3899 # Create Lambda function using Docker image
3900 # Store function name as string attribute for cross-stack references
3901 # This avoids CDK cross-environment resolution issues when account is unresolved
3902 self.helm_installer_lambda_function_name = f"{project_name}-helm-{self.deployment_region}"
3903 self.helm_installer_lambda = lambda_.DockerImageFunction(
3904 self,
3905 "HelmInstallerFunction",
3906 function_name=self.helm_installer_lambda_function_name,
3907 code=lambda_.DockerImageCode.from_image_asset(
3908 directory="lambda/helm-installer-build",
3909 platform=ecr_assets.Platform.LINUX_AMD64,
3910 ),
3911 timeout=Duration.minutes(15),
3912 memory_size=1024,
3913 architecture=lambda_.Architecture.X86_64,
3914 role=helm_lambda_role,
3915 vpc=self.vpc,
3916 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
3917 security_groups=[helm_lambda_sg],
3918 environment={
3919 "CLUSTER_NAME": self.cluster.cluster_name,
3920 "REGION": self.deployment_region,
3921 "PROJECT_NAME": project_name,
3922 },
3923 tracing=lambda_.Tracing.ACTIVE,
3924 )
3926 # Allow the installer to record per-chart add-on status to SSM so the
3927 # add-on layer's health is observable out-of-band (decoupled from the
3928 # CloudFormation rollback path). Read back via `gco stacks addons-status`.
3929 helm_lambda_role.add_to_policy(
3930 iam.PolicyStatement(
3931 actions=["ssm:PutParameter"],
3932 resources=[
3933 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
3934 f"parameter/{project_name}/addons/*"
3935 ],
3936 )
3937 )
3939 # Add EKS access entry for the Lambda role
3940 self.helm_installer_access_entry = eks.AccessEntry(
3941 self,
3942 "HelmInstallerLambdaAccessEntry",
3943 cluster=self.cluster, # type: ignore[arg-type]
3944 principal=helm_lambda_role.role_arn,
3945 access_policies=[
3946 eks.AccessPolicy.from_access_policy_name(
3947 "AmazonEKSClusterAdminPolicy", access_scope_type=eks.AccessScopeType.CLUSTER
3948 )
3949 ],
3950 )
3952 # ------------------------------------------------------------------
3953 # Step Functions state machine: one task per chart, in charts.yaml
3954 # order. Each chart gets its own retry + Step Functions console
3955 # visibility, and — critically — no single Lambda invocation is bound
3956 # by the 15-minute Lambda limit, so a slow operator (e.g. a cold NVIDIA
3957 # image pull) just costs extra retries instead of failing the deploy.
3958 # ------------------------------------------------------------------
3959 chart_order = _load_helm_chart_order()
3961 def _chart_task(chart_name: str) -> sfn_tasks.LambdaInvoke:
3962 task = sfn_tasks.LambdaInvoke(
3963 self,
3964 f"HelmChart-{chart_name}",
3965 lambda_function=self.helm_installer_lambda,
3966 payload=sfn.TaskInput.from_object(
3967 {
3968 "Action": "install_chart",
3969 "Chart": chart_name,
3970 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
3971 "Region": sfn.JsonPath.string_at("$.Region"),
3972 "EnabledCharts": sfn.JsonPath.list_at("$.EnabledCharts"),
3973 "Charts": sfn.JsonPath.object_at("$.Charts"),
3974 "KedaOperatorRoleArn": sfn.JsonPath.string_at("$.KedaOperatorRoleArn"),
3975 }
3976 ),
3977 payload_response_only=True,
3978 # Keep the execution input intact so the next chart task can
3979 # still read $.ClusterName, $.EnabledCharts, etc.
3980 result_path="$.lastChart",
3981 task_timeout=sfn.Timeout.duration(Duration.minutes(16)),
3982 )
3983 # Per-chart retry with backoff. A cold image pull or a webhook race
3984 # clears on a later attempt; only after exhausting these does the
3985 # chart (and the deploy) fail.
3986 task.add_retry(
3987 errors=["States.ALL"],
3988 max_attempts=4,
3989 interval=Duration.seconds(30),
3990 backoff_rate=2.0,
3991 max_delay=Duration.minutes(5),
3992 )
3993 return task
3995 def _kubectl_task(task_id: str, *, post_helm: bool) -> sfn_tasks.LambdaInvoke:
3996 """One kubectl-apply pass (base or post-Helm) as a state-machine task.
3998 Reads ClusterName / Region / ImageReplacements from the execution
3999 input; the handler raises on any manifest failure so the task's
4000 Retry/Catch can react.
4001 """
4002 task = sfn_tasks.LambdaInvoke(
4003 self,
4004 task_id,
4005 lambda_function=self.kubectl_lambda,
4006 payload=sfn.TaskInput.from_object(
4007 {
4008 "Action": "apply_manifests",
4009 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4010 "Region": sfn.JsonPath.string_at("$.Region"),
4011 "ImageReplacements": sfn.JsonPath.object_at("$.ImageReplacements"),
4012 "PostHelm": "true" if post_helm else "false",
4013 }
4014 ),
4015 payload_response_only=True,
4016 # Keep the execution input intact so later tasks still read
4017 # $.ClusterName, $.ImageReplacements, $.EndpointGroupArn, etc.
4018 result_path="$.lastApply",
4019 task_timeout=sfn.Timeout.duration(Duration.minutes(15)),
4020 )
4021 task.add_retry(
4022 errors=["States.ALL"],
4023 max_attempts=3,
4024 interval=Duration.seconds(30),
4025 backoff_rate=2.0,
4026 max_delay=Duration.minutes(3),
4027 )
4028 return task
4030 def _manifest_validation_task() -> sfn_tasks.LambdaInvoke:
4031 """Require every effective raw manifest object to exist and be ready."""
4032 task = sfn_tasks.LambdaInvoke(
4033 self,
4034 "ValidateKubernetesManifests",
4035 lambda_function=self.kubectl_lambda,
4036 payload=sfn.TaskInput.from_object(
4037 {
4038 "Action": "validate_manifests",
4039 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4040 "Region": sfn.JsonPath.string_at("$.Region"),
4041 "ImageReplacements": sfn.JsonPath.object_at("$.ImageReplacements"),
4042 "DeploymentToken": sfn.JsonPath.string_at("$.DeploymentToken"),
4043 }
4044 ),
4045 payload_response_only=True,
4046 result_path="$.manifestValidation",
4047 task_timeout=sfn.Timeout.duration(Duration.minutes(15)),
4048 )
4049 task.add_retry(
4050 errors=["States.ALL"],
4051 max_attempts=4,
4052 interval=Duration.minutes(1),
4053 backoff_rate=2.0,
4054 max_delay=Duration.minutes(3),
4055 )
4056 return task
4058 def _helm_validation_task() -> sfn_tasks.LambdaInvoke:
4059 """Require every configured Helm release and rendered object to be ready."""
4060 task = sfn_tasks.LambdaInvoke(
4061 self,
4062 "ValidateHelmReleases",
4063 lambda_function=self.helm_installer_lambda,
4064 payload=sfn.TaskInput.from_object(
4065 {
4066 "Action": "validate_releases",
4067 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4068 "Region": sfn.JsonPath.string_at("$.Region"),
4069 "EnabledCharts": sfn.JsonPath.list_at("$.EnabledCharts"),
4070 "Charts": sfn.JsonPath.object_at("$.Charts"),
4071 "DeploymentToken": sfn.JsonPath.string_at("$.DeploymentToken"),
4072 }
4073 ),
4074 payload_response_only=True,
4075 result_path="$.helmValidation",
4076 task_timeout=sfn.Timeout.duration(Duration.minutes(16)),
4077 )
4078 task.add_retry(
4079 errors=["States.ALL"],
4080 max_attempts=4,
4081 interval=Duration.minutes(1),
4082 backoff_rate=2.0,
4083 max_delay=Duration.minutes(3),
4084 )
4085 return task
4087 def _endpoint_publication_task() -> sfn_tasks.LambdaInvoke:
4088 """Publish the exact Gateway ALB and optionally register it with GA."""
4089 payload: dict[str, Any] = {
4090 "Action": "publish_gateway_endpoint",
4091 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4092 "Region": sfn.JsonPath.string_at("$.Region"),
4093 "RegistryRegion": sfn.JsonPath.string_at("$.RegistryRegion"),
4094 "ProjectName": sfn.JsonPath.string_at("$.ProjectName"),
4095 }
4096 if self.global_accelerator_enabled:
4097 payload["EndpointGroupArn"] = sfn.JsonPath.string_at("$.EndpointGroupArn")
4099 task = sfn_tasks.LambdaInvoke(
4100 self,
4101 "PublishGatewayEndpoint",
4102 lambda_function=self.ga_registration_lambda,
4103 payload=sfn.TaskInput.from_object(payload),
4104 payload_response_only=True,
4105 result_path="$.endpointPublication",
4106 task_timeout=sfn.Timeout.duration(Duration.minutes(16)),
4107 )
4108 task.add_retry(
4109 errors=["States.ALL"],
4110 max_attempts=3,
4111 interval=Duration.seconds(30),
4112 backoff_rate=2.0,
4113 max_delay=Duration.minutes(3),
4114 )
4115 return task
4117 chart_tasks = [_chart_task(name) for name in chart_order]
4119 # The state machine owns the full convergence pipeline:
4120 # base apply -> Helm charts -> post-Helm apply -> exhaustive raw
4121 # manifest validation -> exhaustive Helm/rendered-object validation
4122 # -> unconditional Gateway endpoint publication, with optional Global
4123 # Accelerator registration.
4124 #
4125 # Individual chart failures still continue so every release gets an
4126 # install attempt and diagnostic. The terminal validators then make the
4127 # overall execution fail unless every expected object and release is
4128 # present and ready. Post-Helm apply, both validators, and endpoint
4129 # publication deliberately have no catch-to-success path: topology may
4130 # trust only an exact SUCCEEDED execution for the current deployment
4131 # token.
4132 done = sfn.Succeed(self, "HelmInstallComplete")
4133 base_apply = _kubectl_task("ApplyBaseManifests", post_helm=False)
4134 post_apply = _kubectl_task("ApplyPostHelmManifests", post_helm=True)
4135 manifest_validation = _manifest_validation_task()
4136 helm_validation = _helm_validation_task()
4138 endpoint_publication = _endpoint_publication_task()
4139 post_apply.next(manifest_validation)
4140 manifest_validation.next(helm_validation)
4141 helm_validation.next(endpoint_publication)
4142 endpoint_publication.next(done)
4144 if chart_tasks:
4145 for i, task in enumerate(chart_tasks):
4146 next_state: sfn.IChainable = (
4147 chart_tasks[i + 1] if i + 1 < len(chart_tasks) else post_apply
4148 )
4149 task.add_catch(
4150 next_state,
4151 errors=["States.ALL"],
4152 result_path="$.lastChartError",
4153 )
4154 task.next(next_state)
4155 base_apply.next(chart_tasks[0])
4156 else: # pragma: no cover - charts.yaml is always present in the repo
4157 base_apply.next(post_apply)
4159 # base apply has NO catch on purpose: a persistent base-manifest failure
4160 # fails the execution rather than converging onto an incomplete base.
4161 start_state: sfn.IChainable = base_apply
4163 helm_sm_log_group = logs.LogGroup(
4164 self,
4165 "HelmInstallStateMachineLogGroup",
4166 retention=logs.RetentionDays.ONE_WEEK,
4167 removal_policy=RemovalPolicy.DESTROY,
4168 )
4170 self.helm_install_state_machine = sfn.StateMachine(
4171 self,
4172 "HelmInstallStateMachine",
4173 definition_body=sfn.DefinitionBody.from_chainable(start_state),
4174 state_machine_type=sfn.StateMachineType.STANDARD,
4175 timeout=Duration.hours(2),
4176 tracing_enabled=True,
4177 logs=sfn.LogOptions(destination=helm_sm_log_group, level=sfn.LogLevel.ALL),
4178 )
4180 # Thin fire-and-forget provider: onEvent starts the execution and
4181 # returns immediately. It does no Helm/Kubernetes work, so it never
4182 # approaches the Lambda timeout — all the heavy lifting lives in the
4183 # state machine, which converges charts in the background.
4184 helm_orchestrator_on_event = lambda_.Function(
4185 self,
4186 "HelmOrchestratorOnEvent",
4187 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
4188 handler="handler.on_event",
4189 code=lambda_.Code.from_asset("lambda/helm-orchestrator"),
4190 timeout=Duration.minutes(1),
4191 memory_size=256,
4192 environment={
4193 "STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
4194 },
4195 tracing=lambda_.Tracing.ACTIVE,
4196 )
4197 self.helm_install_state_machine.grant_start_execution(helm_orchestrator_on_event)
4198 self.helm_install_state_machine.grant_execution(
4199 helm_orchestrator_on_event,
4200 "states:StopExecution",
4201 "states:DescribeExecution",
4202 )
4204 # Let on_event persist the execution input to SSM so the add-on install
4205 # can be replayed out-of-band (gco stacks addons install) without the
4206 # CLI reconstructing chart config or the KEDA role ARN.
4207 helm_orchestrator_on_event.add_to_role_policy(
4208 iam.PolicyStatement(
4209 actions=[
4210 "ssm:PutParameter",
4211 "ssm:GetParameter",
4212 "ssm:DeleteParameter",
4213 ],
4214 resources=[
4215 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
4216 f"parameter/{project_name}/addons/*"
4217 ],
4218 )
4219 )
4221 # HelmInstallCharts depends on this explicit bounded-retention group,
4222 # forcing the trigger's final provider invocation to finish before
4223 # CloudFormation removes the group.
4224 self.helm_installer_provider = cr.Provider(
4225 self,
4226 "HelmInstallerProvider",
4227 on_event_handler=helm_orchestrator_on_event,
4228 log_group=self.helm_installer_provider_log_group,
4229 )
4231 # Unlike create/update convergence, stack deletion must be synchronous:
4232 # Helm releases can own admission webhooks, load balancers, and CRs that
4233 # have to disappear while the Kubernetes API and installer AccessEntry
4234 # still exist. A delete-only provider waits on a reverse-order state
4235 # machine and fails CloudFormation if any real uninstall fails.
4236 self._create_helm_teardown(chart_order)
4238 # cdk-nag suppressions for the install path.
4239 from gco.stacks.nag_suppressions import acknowledge_nag_findings
4241 acknowledge_nag_findings(
4242 helm_lambda_role,
4243 [
4244 {
4245 "id": "AwsSolutions-IAM5",
4246 "reason": (
4247 "The Helm installer Lambda requires broad EKS and Kubernetes API "
4248 "access to install Helm charts (KEDA, NVIDIA DRA, etc.) that create "
4249 "CRDs, RBAC rules, and workloads across multiple namespaces. "
4250 "Resource: * is required because the set of Kubernetes resources "
4251 "is dynamic and not known at synth time."
4252 ),
4253 "appliesTo": ["Resource::*"],
4254 },
4255 ],
4256 )
4257 # The state machine role (auto-generated) invokes the worker Lambda
4258 # across versions using the AWS-standard ``:*`` qualifier that cannot be
4259 # enumerated at synth time.
4260 acknowledge_nag_findings(
4261 self.helm_install_state_machine,
4262 [
4263 {
4264 "id": "AwsSolutions-IAM5",
4265 "reason": (
4266 "The state machine invokes the helm worker Lambda; CDK grants "
4267 "lambda:InvokeFunction with the :* version qualifier, which is the "
4268 "standard form and cannot be narrowed at synth time."
4269 ),
4270 "appliesTo": ["Resource::*"],
4271 },
4272 ],
4273 )
4274 acknowledge_nag_findings(
4275 helm_orchestrator_on_event,
4276 [
4277 {
4278 "id": "AwsSolutions-IAM5",
4279 "reason": (
4280 "Lambda active tracing requires X-Ray write APIs against "
4281 "Resource::*, as X-Ray does not expose resource-level "
4282 "permissions for these telemetry calls."
4283 ),
4284 "appliesTo": ["Resource::*"],
4285 },
4286 {
4287 "id": "AwsSolutions-IAM5",
4288 "reason": (
4289 "If execution metadata persistence fails, the orchestrator must "
4290 "stop the just-started convergence execution before CloudFormation "
4291 "can roll back. This grant is limited to executions of the single "
4292 "regional Helm install state machine."
4293 ),
4294 "appliesTo": [
4295 "Resource::arn:<AWS::Partition>:states:<AWS::Region>:"
4296 "<AWS::AccountId>:execution:"
4297 '{"Fn::Select":[6,{"Fn::Split":[":",'
4298 '{"Ref":"HelmInstallStateMachine7DB71CDC"}]}]}:*'
4299 ],
4300 },
4301 ],
4302 )
4304 # The cr.Provider framework auto-generates a framework-onEvent Lambda and
4305 # its role (and, were an is_complete_handler set, a waiter state machine —
4306 # which this fire-and-forget provider does NOT create). None of these are
4307 # configurable by us: the framework role invokes our handler Lambda via
4308 # the standard ``<lambda-arn>:*`` version qualifier that cannot be narrowed
4309 # at synth time. Suppress the relevant rules across the whole provider
4310 # subtree; appliesTo is omitted because the findings are granted on
4311 # CDK-managed resources we do not author. The SF1/SF2/X-Ray entries are
4312 # retained defensively to cover any helper state machine the framework may
4313 # emit across CDK versions; they are harmless no-ops when none exists.
4314 acknowledge_nag_findings(
4315 self.helm_installer_provider,
4316 [
4317 {
4318 "id": "AwsSolutions-IAM5",
4319 "reason": (
4320 "CDK custom-resource provider framework roles invoke the "
4321 "orchestrator Lambdas via the standard '<lambda-arn>:*' version "
4322 "qualifier, which cannot be enumerated at synth time."
4323 ),
4324 "appliesTo": [
4325 "Resource::<HelmOrchestratorOnEventD0D51D9B.Arn>:*",
4326 ],
4327 },
4328 {
4329 "id": "AwsSolutions-SF1",
4330 "reason": (
4331 "The waiter state machine is auto-generated by the CDK "
4332 "cr.Provider framework and does not expose log configuration; "
4333 "ALL-event logging cannot be enabled on it."
4334 ),
4335 },
4336 {
4337 "id": "AwsSolutions-SF2",
4338 "reason": (
4339 "The waiter state machine is auto-generated by the CDK "
4340 "cr.Provider framework and does not expose tracing "
4341 "configuration; X-Ray cannot be enabled on it."
4342 ),
4343 },
4344 {
4345 "id": "Serverless-StepFunctionStateMachineXray",
4346 "reason": (
4347 "The waiter state machine is auto-generated by the CDK "
4348 "cr.Provider framework and does not expose tracing "
4349 "configuration; X-Ray cannot be enabled on it."
4350 ),
4351 },
4352 ],
4353 )
4355 def _create_helm_teardown(self, chart_order: list[str]) -> None:
4356 """Create the synchronous, reverse-order Helm stack-delete path.
4358 Create/update remain fire-and-forget through ``HelmInstallCharts``. This
4359 separate custom resource is a no-op for those events, but on Delete it
4360 starts a state machine whose per-chart tasks call ``uninstall_chart`` in
4361 reverse install order. The provider polls to terminal state, so a failed
4362 release blocks deletion before EKS authentication or the API disappears.
4363 """
4365 lbc_chart = "aws-load-balancer-controller"
4366 if not chart_order or chart_order[0] != lbc_chart: 4366 ↛ 4367line 4366 didn't jump to line 4367 because the condition on line 4366 was never true
4367 raise RuntimeError(
4368 "aws-load-balancer-controller must be the first Helm chart for safe teardown"
4369 )
4371 provider_code = lambda_.Code.from_asset("lambda/helm-installer")
4372 drain_checker = lambda_.Function(
4373 self,
4374 "HelmTeardownDrainChecker",
4375 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
4376 handler="teardown_provider.drain_install_executions",
4377 code=provider_code,
4378 timeout=Duration.minutes(1),
4379 memory_size=256,
4380 environment={
4381 "INSTALL_STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
4382 },
4383 tracing=lambda_.Tracing.ACTIVE,
4384 )
4386 def _uninstall_task(chart_name: str) -> sfn_tasks.LambdaInvoke:
4387 timeout_minutes = 5 if chart_name == lbc_chart else 4 if chart_name == "keda" else 2
4388 task = sfn_tasks.LambdaInvoke(
4389 self,
4390 f"HelmUninstallChart-{chart_name}",
4391 lambda_function=self.helm_installer_lambda,
4392 payload=sfn.TaskInput.from_object(
4393 {
4394 "Action": "uninstall_chart",
4395 "Chart": chart_name,
4396 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4397 "Region": sfn.JsonPath.string_at("$.Region"),
4398 "EnabledCharts": sfn.JsonPath.list_at("$.EnabledCharts"),
4399 "Charts": sfn.JsonPath.object_at("$.Charts"),
4400 "KedaOperatorRoleArn": sfn.JsonPath.string_at("$.KedaOperatorRoleArn"),
4401 }
4402 ),
4403 payload_response_only=True,
4404 result_path="$.lastChart",
4405 task_timeout=sfn.Timeout.duration(Duration.minutes(timeout_minutes)),
4406 )
4407 return task
4409 def _drain_check_task() -> sfn_tasks.LambdaInvoke:
4410 return sfn_tasks.LambdaInvoke(
4411 self,
4412 "CheckRunningConvergence",
4413 lambda_function=drain_checker,
4414 payload=sfn.TaskInput.from_object({}),
4415 payload_response_only=True,
4416 result_path="$.drainCheck",
4417 task_timeout=sfn.Timeout.duration(Duration.minutes(1)),
4418 )
4420 def _quiesce_task() -> sfn_tasks.LambdaInvoke:
4421 task = sfn_tasks.LambdaInvoke(
4422 self,
4423 "QuiesceHealthMonitor",
4424 lambda_function=self.helm_installer_lambda,
4425 payload=sfn.TaskInput.from_object(
4426 {
4427 "Action": "quiesce_health_monitor",
4428 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4429 "Region": sfn.JsonPath.string_at("$.Region"),
4430 }
4431 ),
4432 payload_response_only=True,
4433 result_path="$.healthMonitorQuiesce",
4434 task_timeout=sfn.Timeout.duration(Duration.minutes(3)),
4435 )
4436 return task
4438 def _endpoint_cleanup_task() -> sfn_tasks.LambdaInvoke:
4439 """Fence SSM/GA publication after all endpoint writers are quiesced."""
4440 payload: dict[str, Any] = {
4441 "Action": "cleanup_gateway_endpoint",
4442 "Region": sfn.JsonPath.string_at("$.Region"),
4443 "RegistryRegion": sfn.JsonPath.string_at("$.RegistryRegion"),
4444 "ProjectName": sfn.JsonPath.string_at("$.ProjectName"),
4445 }
4446 if self.global_accelerator_enabled:
4447 payload["EndpointGroupArn"] = sfn.JsonPath.string_at("$.EndpointGroupArn")
4448 return sfn_tasks.LambdaInvoke(
4449 self,
4450 "CleanupGatewayEndpoint",
4451 lambda_function=self.ga_registration_lambda,
4452 payload=sfn.TaskInput.from_object(payload),
4453 payload_response_only=True,
4454 result_path="$.endpointCleanup",
4455 task_timeout=sfn.Timeout.duration(Duration.minutes(15)),
4456 )
4458 def _gateway_cleanup_task() -> sfn_tasks.LambdaInvoke:
4459 return sfn_tasks.LambdaInvoke(
4460 self,
4461 "DeleteGatewayResources",
4462 lambda_function=self.kubectl_lambda,
4463 payload=sfn.TaskInput.from_object(
4464 {
4465 "Action": "delete_gateway_resources",
4466 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4467 "Region": sfn.JsonPath.string_at("$.Region"),
4468 }
4469 ),
4470 payload_response_only=True,
4471 result_path="$.gatewayCleanup",
4472 task_timeout=sfn.Timeout.duration(Duration.minutes(5)),
4473 )
4475 non_lbc_tasks = [
4476 _uninstall_task(name) for name in reversed(chart_order) if name != lbc_chart
4477 ]
4478 endpoint_cleanup = _endpoint_cleanup_task()
4479 pre_gateway_cleanup = sfn.Parallel(
4480 self,
4481 "CleanupEndpointAndCharts",
4482 result_path="$.preGatewayCleanup",
4483 )
4484 pre_gateway_cleanup.branch(endpoint_cleanup)
4485 if non_lbc_tasks: 4485 ↛ 4490line 4485 didn't jump to line 4490 because the condition on line 4485 was always true
4486 for index, task in enumerate(non_lbc_tasks[:-1]):
4487 task.next(non_lbc_tasks[index + 1])
4488 pre_gateway_cleanup.branch(non_lbc_tasks[0])
4490 lbc_uninstall = _uninstall_task(lbc_chart)
4491 gateway_cleanup = _gateway_cleanup_task()
4492 done = sfn.Succeed(self, "HelmTeardownComplete")
4493 quiesce = _quiesce_task()
4495 quiesce.next(pre_gateway_cleanup)
4496 pre_gateway_cleanup.next(gateway_cleanup)
4497 gateway_cleanup.next(lbc_uninstall)
4498 lbc_uninstall.next(done)
4500 # StopExecution cannot cancel a Lambda invocation already in flight,
4501 # and ListExecutions is eventually consistent. The provider stops the
4502 # initially visible executions before this unconditional 16-minute
4503 # drain. The checker then stops any late-visible work and loops through
4504 # another complete drain interval before quiescence. The SSM teardown
4505 # fence blocks supported convergence entrypoints from creating new work.
4506 drain_in_flight = sfn.Wait(
4507 self,
4508 "DrainInFlightConvergence",
4509 time=sfn.WaitTime.seconds_path("$.WaitForInFlightSeconds"),
4510 )
4511 drain_check = _drain_check_task()
4512 late_work = sfn.Choice(self, "LateConvergenceFound")
4513 drain_in_flight.next(drain_check)
4514 drain_check.next(late_work)
4515 late_work.when(
4516 sfn.Condition.number_greater_than("$.drainCheck.StoppedExecutions", 0),
4517 drain_in_flight,
4518 ).otherwise(quiesce)
4519 start_state: sfn.IChainable = drain_in_flight
4521 teardown_log_group = logs.LogGroup(
4522 self,
4523 "HelmTeardownStateMachineLogGroup",
4524 retention=logs.RetentionDays.ONE_WEEK,
4525 removal_policy=RemovalPolicy.DESTROY,
4526 )
4527 self.helm_teardown_state_machine = sfn.StateMachine(
4528 self,
4529 "HelmTeardownStateMachine",
4530 definition_body=sfn.DefinitionBody.from_chainable(start_state),
4531 state_machine_type=sfn.StateMachineType.STANDARD,
4532 # 16m drain + 3m quiesce + max(15m endpoint cleanup, 24m ordinary
4533 # chart cleanup) + 5m Gateway deletion + 5m LBC uninstall = 53m.
4534 # Three minutes of workflow margin leave another three minutes for
4535 # the provider's final poll inside CloudFormation's one-hour ceiling.
4536 timeout=Duration.minutes(56),
4537 tracing_enabled=True,
4538 logs=sfn.LogOptions(destination=teardown_log_group, level=sfn.LogLevel.ALL),
4539 )
4541 teardown_on_event = lambda_.Function(
4542 self,
4543 "HelmTeardownOnEvent",
4544 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
4545 handler="teardown_provider.on_event",
4546 code=provider_code,
4547 timeout=Duration.minutes(1),
4548 memory_size=256,
4549 environment={
4550 "TEARDOWN_STATE_MACHINE_ARN": self.helm_teardown_state_machine.state_machine_arn,
4551 "INSTALL_STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
4552 },
4553 tracing=lambda_.Tracing.ACTIVE,
4554 )
4555 teardown_is_complete = lambda_.Function(
4556 self,
4557 "HelmTeardownIsComplete",
4558 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
4559 handler="teardown_provider.is_complete",
4560 code=provider_code,
4561 timeout=Duration.minutes(1),
4562 memory_size=256,
4563 environment={
4564 "TEARDOWN_STATE_MACHINE_ARN": self.helm_teardown_state_machine.state_machine_arn,
4565 "INSTALL_STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
4566 },
4567 tracing=lambda_.Tracing.ACTIVE,
4568 )
4569 self.helm_teardown_state_machine.grant_start_execution(teardown_on_event)
4570 self.helm_teardown_state_machine.grant_read(teardown_is_complete)
4571 install_execution_detail = (
4572 "Resource::arn:<AWS::Partition>:states:<AWS::Region>:<AWS::AccountId>:execution:"
4573 '{"Fn::Select":[6,{"Fn::Split":[":",'
4574 '{"Ref":"HelmInstallStateMachine7DB71CDC"}]}]}:*'
4575 )
4576 teardown_execution_detail = (
4577 "Resource::arn:<AWS::Partition>:states:<AWS::Region>:<AWS::AccountId>:execution:"
4578 '{"Fn::Select":[6,{"Fn::Split":[":",'
4579 '{"Ref":"HelmTeardownStateMachine1C15895F"}]}]}:*'
4580 )
4581 for handler in (teardown_on_event, drain_checker):
4582 self.helm_install_state_machine.grant(
4583 handler,
4584 "states:ListExecutions",
4585 )
4586 self.helm_install_state_machine.grant_execution(
4587 handler,
4588 "states:StopExecution",
4589 "states:DescribeExecution",
4590 )
4591 teardown_on_event.add_to_role_policy(
4592 iam.PolicyStatement(
4593 actions=["ssm:PutParameter"],
4594 resources=[
4595 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
4596 f"parameter/{self.config.get_project_name()}/addons/"
4597 f"{self.deployment_region}/_teardown"
4598 ],
4599 )
4600 )
4602 # Strict live validation preserves this generation until exact
4603 # post-stack cleanup. Ordinary deployments retain DESTROY semantics.
4604 provider_log_group = logs.LogGroup(
4605 self,
4606 "HelmTeardownProviderLogGroup",
4607 retention=logs.RetentionDays.ONE_WEEK,
4608 removal_policy=self.provider_log_group_removal_policy,
4609 )
4610 self.helm_teardown_provider = cr.Provider(
4611 self,
4612 "HelmTeardownProvider",
4613 on_event_handler=teardown_on_event,
4614 is_complete_handler=teardown_is_complete,
4615 query_interval=Duration.seconds(15),
4616 total_timeout=Duration.minutes(59),
4617 log_group=provider_log_group,
4618 )
4619 teardown_properties: dict[str, Any] = {
4620 "ClusterName": self.cluster.cluster_name,
4621 "Region": self.deployment_region,
4622 "RegistryRegion": self.config.get_global_region(),
4623 "ProjectName": self.config.get_project_name(),
4624 "EnabledCharts": self._get_enabled_helm_charts(),
4625 "Charts": self._helm_chart_value_overrides(),
4626 "KedaOperatorRoleArn": self.keda_operator_role.role_arn,
4627 }
4628 if self.endpoint_group_arn is not None:
4629 teardown_properties["EndpointGroupArn"] = self.endpoint_group_arn
4630 self.helm_teardown_resource = CustomResource(
4631 self,
4632 "HelmTeardown",
4633 service_token=self.helm_teardown_provider.service_token,
4634 properties=teardown_properties,
4635 )
4636 self.helm_teardown_resource.node.add_dependency(self.cluster)
4637 self.helm_teardown_resource.node.add_dependency(self.helm_installer_access_entry)
4638 self.helm_teardown_resource.node.add_dependency(self.kubectl_lambda_access_entry)
4639 self.helm_teardown_resource.node.add_dependency(self.helm_install_state_machine)
4640 self.helm_teardown_resource.node.add_dependency(self.aws_load_balancer_controller_policy)
4641 self.helm_teardown_resource.node.add_dependency(provider_log_group)
4643 from gco.stacks.nag_suppressions import acknowledge_nag_findings
4645 acknowledge_nag_findings(
4646 self.helm_teardown_state_machine,
4647 [
4648 {
4649 "id": "AwsSolutions-IAM5",
4650 "reason": (
4651 "The teardown state machine's X-Ray integration requires "
4652 "Resource::*, and its Lambda task grants use CDK's required :* "
4653 "version qualifier. The drain-checker detail names the single "
4654 "dedicated function created by this stack."
4655 ),
4656 "appliesTo": [
4657 "Resource::*",
4658 "Resource::<HelmTeardownDrainCheckerCCF8D9D1.Arn>:*",
4659 ],
4660 },
4661 ],
4662 )
4663 acknowledge_nag_findings(
4664 drain_checker,
4665 [
4666 {
4667 "id": "AwsSolutions-IAM5",
4668 "reason": (
4669 "The drain checker uses X-Ray Resource::* APIs and stops only "
4670 "runtime-generated executions of the single regional Helm install "
4671 "state machine before Kubernetes teardown."
4672 ),
4673 "appliesTo": ["Resource::*", install_execution_detail],
4674 }
4675 ],
4676 )
4677 for handler in (teardown_on_event, teardown_is_complete):
4678 acknowledge_nag_findings(
4679 handler,
4680 [
4681 {
4682 "id": "AwsSolutions-IAM5",
4683 "reason": (
4684 "X-Ray write APIs require Resource::*. StopExecution and "
4685 "DescribeExecution are otherwise limited to runtime-generated "
4686 "execution ARNs belonging to the two regional Helm state machines."
4687 ),
4688 "appliesTo": [
4689 "Resource::*",
4690 install_execution_detail,
4691 teardown_execution_detail,
4692 ],
4693 }
4694 ],
4695 )
4696 acknowledge_nag_findings(
4697 self.helm_teardown_provider,
4698 [
4699 {
4700 "id": "AwsSolutions-IAM5",
4701 "reason": (
4702 "The CDK provider framework invokes only versioned onEvent/isComplete "
4703 "handlers and its generated waiter invokes only its versioned timeout "
4704 "and completion handlers; every wildcard is a Lambda qualifier."
4705 ),
4706 "appliesTo": [
4707 "Resource::<HelmTeardownIsComplete5ECB4605.Arn>:*",
4708 "Resource::<HelmTeardownOnEvent3DB6F756.Arn>:*",
4709 ("Resource::<HelmTeardownProviderframeworkisComplete3D7339F4.Arn>:*"),
4710 ("Resource::<HelmTeardownProviderframeworkonTimeout3415E5E9.Arn>:*"),
4711 ],
4712 },
4713 {
4714 "id": "AwsSolutions-SF1",
4715 "reason": (
4716 "The provider waiter state machine is generated by CDK and does not "
4717 "expose logging configuration."
4718 ),
4719 },
4720 {
4721 "id": "AwsSolutions-SF2",
4722 "reason": (
4723 "The provider waiter state machine is generated by CDK and does not "
4724 "expose tracing configuration."
4725 ),
4726 },
4727 {
4728 "id": "Serverless-StepFunctionStateMachineXray",
4729 "reason": (
4730 "The provider waiter state machine is generated by CDK and does not "
4731 "expose tracing configuration."
4732 ),
4733 },
4734 ],
4735 )
4737 def _create_efs(self) -> None:
4738 """Create EFS file system for shared storage across jobs.
4740 Creates an EFS file system with mount targets in each private subnet,
4741 allowing pods to share data and persist outputs. The EFS is configured
4742 with:
4743 - Encryption at rest
4744 - Automatic backups
4745 - General Purpose performance mode (suitable for most workloads)
4746 - Bursting throughput mode
4748 Kubernetes resources (StorageClass, PV, PVC) are created via manifests.
4749 """
4750 project_name = self.config.get_project_name()
4752 # Create security group for EFS
4753 self.efs_security_group = ec2.SecurityGroup(
4754 self,
4755 "EfsSecurityGroup",
4756 vpc=self.vpc,
4757 description=f"Security group for {project_name} EFS in {self.deployment_region}",
4758 security_group_name=f"{project_name}-efs-sg-{self.deployment_region}",
4759 allow_all_outbound=False, # EFS doesn't need outbound
4760 )
4762 # Allow NFS traffic from EKS cluster security group
4763 self.efs_security_group.add_ingress_rule(
4764 peer=self.cluster.cluster_security_group,
4765 connection=ec2.Port.tcp(2049),
4766 description="Allow NFS from EKS cluster",
4767 )
4769 # Create EFS file system
4770 self.efs_file_system = efs.FileSystem(
4771 self,
4772 "GCOEfs",
4773 vpc=self.vpc,
4774 file_system_name=f"{project_name}-efs-{self.deployment_region}",
4775 security_group=self.efs_security_group,
4776 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
4777 encrypted=True,
4778 performance_mode=efs.PerformanceMode.GENERAL_PURPOSE,
4779 throughput_mode=efs.ThroughputMode.BURSTING,
4780 removal_policy=RemovalPolicy.DESTROY, # For dev/test; use RETAIN for production
4781 enable_automatic_backups=True,
4782 )
4784 # Add file system policy to allow mounting without IAM authorization
4785 # This allows any client that can reach the mount target to mount the file system
4786 self.efs_file_system.add_to_resource_policy(
4787 iam.PolicyStatement(
4788 effect=iam.Effect.ALLOW,
4789 principals=[iam.AnyPrincipal()],
4790 actions=[
4791 "elasticfilesystem:ClientMount",
4792 "elasticfilesystem:ClientWrite",
4793 "elasticfilesystem:ClientRootAccess",
4794 ],
4795 conditions={"Bool": {"elasticfilesystem:AccessedViaMountTarget": "true"}},
4796 )
4797 )
4799 # Create access point for the gco-jobs directory
4800 self.efs_access_point = self.efs_file_system.add_access_point(
4801 "JobsAccessPoint",
4802 path="/gco-jobs",
4803 create_acl=efs.Acl(owner_uid="1000", owner_gid="1000", permissions="755"),
4804 posix_user=efs.PosixUser(uid="1000", gid="1000"),
4805 )
4807 # Output EFS information
4808 CfnOutput(
4809 self,
4810 "EfsFileSystemId",
4811 value=self.efs_file_system.file_system_id,
4812 description="EFS File System ID for shared job storage",
4813 )
4815 CfnOutput(
4816 self,
4817 "EfsAccessPointId",
4818 value=self.efs_access_point.access_point_id,
4819 description="EFS Access Point ID for job outputs",
4820 )
4822 def _create_fsx_lustre(self) -> None:
4823 """Create FSx for Lustre file system for high-performance storage.
4825 FSx for Lustre provides high-performance parallel file system storage
4826 ideal for ML training workloads that require high throughput and low latency.
4828 This is optional and controlled by the fsx_lustre.enabled config setting.
4830 Supported deployment types:
4831 - SCRATCH_1: Temporary storage, no data replication
4832 - SCRATCH_2: Temporary storage with better burst performance
4833 - PERSISTENT_1: Persistent storage with data replication
4834 - PERSISTENT_2: Latest persistent storage with higher throughput
4835 """
4836 fsx_config = self.config.get_fsx_lustre_config(self.deployment_region)
4838 if not fsx_config.get("enabled", False):
4839 self.fsx_file_system = None
4840 return
4842 project_name = self.config.get_project_name()
4844 # Create security group for FSx
4845 self.fsx_security_group = ec2.SecurityGroup(
4846 self,
4847 "FsxSecurityGroup",
4848 vpc=self.vpc,
4849 description=f"Security group for {project_name} FSx Lustre in {self.deployment_region}",
4850 security_group_name=f"{project_name}-fsx-sg-{self.deployment_region}",
4851 allow_all_outbound=False,
4852 )
4854 # Allow Lustre traffic from EKS cluster security group
4855 # Lustre uses ports 988 (control) and 1021-1023 (data)
4856 self.fsx_security_group.add_ingress_rule(
4857 peer=self.cluster.cluster_security_group,
4858 connection=ec2.Port.tcp(988),
4859 description="Allow Lustre control traffic from EKS cluster",
4860 )
4861 self.fsx_security_group.add_ingress_rule(
4862 peer=self.cluster.cluster_security_group,
4863 connection=ec2.Port.tcp_range(1021, 1023),
4864 description="Allow Lustre data traffic from EKS cluster",
4865 )
4867 # Allow self-referencing traffic for FSx Lustre internal communication
4868 # FSx Lustre nodes need to communicate with each other on port 988
4869 self.fsx_security_group.add_ingress_rule(
4870 peer=self.fsx_security_group,
4871 connection=ec2.Port.tcp(988),
4872 description="Allow Lustre internal traffic on port 988",
4873 )
4874 self.fsx_security_group.add_ingress_rule(
4875 peer=self.fsx_security_group,
4876 connection=ec2.Port.tcp_range(1021, 1023),
4877 description="Allow Lustre internal traffic on ports 1021-1023",
4878 )
4880 # Get deployment type
4881 deployment_type = fsx_config.get("deployment_type", "SCRATCH_2")
4882 storage_capacity = fsx_config.get("storage_capacity_gib", 1200)
4884 # Build Lustre configuration based on deployment type
4885 lustre_config = {
4886 "deploymentType": deployment_type,
4887 "dataCompressionType": fsx_config.get("data_compression_type", "LZ4"),
4888 }
4890 # Add throughput for PERSISTENT types
4891 if deployment_type.startswith("PERSISTENT"):
4892 lustre_config["perUnitStorageThroughput"] = fsx_config.get(
4893 "per_unit_storage_throughput", 200
4894 )
4896 # Add S3 import/export if configured
4897 import_path = fsx_config.get("import_path")
4898 export_path = fsx_config.get("export_path")
4900 if import_path:
4901 lustre_config["importPath"] = import_path
4902 lustre_config["autoImportPolicy"] = fsx_config.get(
4903 "auto_import_policy", "NEW_CHANGED_DELETED"
4904 )
4906 if export_path:
4907 lustre_config["exportPath"] = export_path
4909 # Get file system type version (default to 2.15 for kernel 6.x compatibility)
4910 # IMPORTANT: Lustre 2.10 is NOT compatible with kernel 6.x (AL2023, Bottlerocket 1.19+)
4911 # See: https://docs.aws.amazon.com/fsx/latest/LustreGuide/lustre-client-matrix.html
4912 file_system_type_version = fsx_config.get("file_system_type_version", "2.15")
4914 # Create FSx for Lustre file system
4915 self.fsx_file_system = fsx.CfnFileSystem(
4916 self,
4917 "GCOFsxLustre",
4918 file_system_type="LUSTRE",
4919 file_system_type_version=file_system_type_version,
4920 storage_capacity=storage_capacity,
4921 subnet_ids=[self.vpc.private_subnets[0].subnet_id],
4922 security_group_ids=[self.fsx_security_group.security_group_id],
4923 lustre_configuration=lustre_config,
4924 tags=[
4925 {"key": "Name", "value": f"{project_name}-fsx-{self.deployment_region}"},
4926 {"key": "Project", "value": project_name},
4927 ],
4928 )
4930 # Ensure FSx file system waits for security group ingress rules to be created
4931 # This prevents "security group does not permit Lustre LNET traffic" errors
4932 self.fsx_file_system.node.add_dependency(self.fsx_security_group)
4934 # Create FSx CSI Driver add-on for Kubernetes integration
4935 self._create_fsx_csi_driver_addon()
4937 # Output FSx information
4938 CfnOutput(
4939 self,
4940 "FsxFileSystemId",
4941 value=self.fsx_file_system.ref,
4942 description="FSx for Lustre File System ID",
4943 )
4945 CfnOutput(
4946 self,
4947 "FsxDnsName",
4948 value=self.fsx_file_system.attr_dns_name,
4949 description="FSx for Lustre DNS Name",
4950 )
4952 CfnOutput(
4953 self,
4954 "FsxMountName",
4955 value=self.fsx_file_system.attr_lustre_mount_name,
4956 description="FSx for Lustre Mount Name",
4957 )
4959 def _create_valkey_cache(self) -> None:
4960 """Create an ElastiCache Serverless Valkey cache for K/V caching.
4962 Provides a low-latency key-value store that inference endpoints and
4963 jobs can use for prompt caching, session state, feature stores, or
4964 any shared state across pods. Valkey Serverless auto-scales and
4965 requires no node management.
4967 The cache is placed in the VPC private subnets and accessible from
4968 any pod via the cluster security group.
4969 """
4970 valkey_config = self.config.get_valkey_config()
4971 if not valkey_config.get("enabled", False):
4972 return
4974 from aws_cdk import aws_elasticache as elasticache
4976 # Security group for Valkey (allow access from EKS cluster)
4977 valkey_sg = ec2.SecurityGroup(
4978 self,
4979 "ValkeySG",
4980 vpc=self.vpc,
4981 description="Security group for Valkey Serverless cache",
4982 allow_all_outbound=False,
4983 )
4984 valkey_sg.add_ingress_rule(
4985 ec2.Peer.ipv4(self.vpc.vpc_cidr_block),
4986 ec2.Port.tcp(6379),
4987 "Allow Valkey access from VPC",
4988 )
4990 # The Valkey SG ingress allows 6379 from the VPC CIDR (an ``Fn::GetAtt``
4991 # token cdk-nag can't resolve), so the SG-ingress rules throw. Scope the
4992 # acknowledgment to the Valkey SG construct itself.
4993 from gco.stacks.nag_suppressions import acknowledge_security_group_cidr_findings
4995 acknowledge_security_group_cidr_findings(
4996 valkey_sg,
4997 reason=(
4998 "The Valkey Serverless cache security group allows the Valkey "
4999 "port (6379) from the VPC CIDR only, referenced via an "
5000 "``Fn::GetAtt`` token that cdk-nag cannot resolve at synth "
5001 "time. Ingress is restricted to intra-VPC traffic from the "
5002 "job pods that use the cache."
5003 ),
5004 )
5006 private_subnet_ids = [s.subnet_id for s in self.vpc.private_subnets]
5008 self.valkey_cache = elasticache.CfnServerlessCache(
5009 self,
5010 "ValkeyCache",
5011 engine="valkey",
5012 serverless_cache_name=f"{self.config.get_project_name()}-{self.deployment_region}",
5013 description=f"GCO K/V cache for {self.deployment_region}",
5014 major_engine_version="8",
5015 security_group_ids=[valkey_sg.security_group_id],
5016 subnet_ids=private_subnet_ids,
5017 cache_usage_limits=elasticache.CfnServerlessCache.CacheUsageLimitsProperty(
5018 data_storage=elasticache.CfnServerlessCache.DataStorageProperty(
5019 maximum=valkey_config.get("max_data_storage_gb", 5),
5020 minimum=1,
5021 unit="GB",
5022 ),
5023 ecpu_per_second=elasticache.CfnServerlessCache.ECPUPerSecondProperty(
5024 maximum=valkey_config.get("max_ecpu_per_second", 5000),
5025 minimum=1000,
5026 ),
5027 ),
5028 snapshot_retention_limit=valkey_config.get("snapshot_retention_limit", 1),
5029 tags=[
5030 CfnTag(key="Project", value=self.config.get_project_name()),
5031 CfnTag(key="gco:project", value=self.config.get_project_name()),
5032 CfnTag(key="Region", value=self.deployment_region),
5033 ],
5034 )
5036 CfnOutput(
5037 self,
5038 "ValkeyEndpoint",
5039 value=self.valkey_cache.attr_endpoint_address,
5040 description="Valkey Serverless cache endpoint",
5041 )
5042 CfnOutput(
5043 self,
5044 "ValkeyPort",
5045 value=self.valkey_cache.attr_endpoint_port,
5046 description="Valkey Serverless cache port",
5047 )
5049 # Store endpoint in SSM for discovery by pods
5050 ssm.StringParameter(
5051 self,
5052 "ValkeyEndpointParam",
5053 parameter_name=f"/{self.config.get_project_name()}/valkey-endpoint-{self.deployment_region}",
5054 string_value=self.valkey_cache.attr_endpoint_address,
5055 description=f"Valkey endpoint for {self.deployment_region}",
5056 )
5058 def _create_aurora_pgvector(self) -> None:
5059 """Create an Aurora Serverless v2 PostgreSQL cluster with pgvector.
5061 Provides a fully managed vector database that inference endpoints and
5062 jobs can use for RAG (retrieval-augmented generation), semantic search,
5063 embedding storage, and similarity queries. Aurora Serverless v2
5064 auto-scales capacity and requires no instance management.
5066 The cluster is placed in the VPC private subnets and accessible from
5067 any pod via the cluster security group. Credentials are stored in
5068 Secrets Manager and the endpoint is published to SSM + a K8s ConfigMap
5069 for automatic discovery.
5071 See: https://aws.amazon.com/blogs/database/accelerate-generative-ai-workloads-on-amazon-aurora-with-optimized-reads-and-pgvector/
5072 """
5073 aurora_config = self.config.get_aurora_pgvector_config()
5074 if not aurora_config.get("enabled", False):
5075 return
5077 from aws_cdk import aws_rds as rds
5079 project_name = self.config.get_project_name()
5081 # Security group for Aurora (allow PostgreSQL access from EKS cluster only)
5082 aurora_sg = ec2.SecurityGroup(
5083 self,
5084 "AuroraPgvectorSG",
5085 vpc=self.vpc,
5086 description="Security group for Aurora Serverless v2 pgvector",
5087 allow_all_outbound=False,
5088 )
5089 aurora_sg.add_ingress_rule(
5090 self.cluster.cluster_security_group,
5091 ec2.Port.tcp(5432),
5092 "Allow PostgreSQL access from EKS cluster",
5093 )
5095 # Subnet group for Aurora (private subnets only)
5096 subnet_group = rds.SubnetGroup(
5097 self,
5098 "AuroraPgvectorSubnetGroup",
5099 description=f"Subnet group for GCO Aurora pgvector in {self.deployment_region}",
5100 vpc=self.vpc,
5101 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
5102 )
5104 # Aurora Serverless v2 cluster with PostgreSQL 16 + pgvector
5105 self.aurora_cluster = rds.DatabaseCluster(
5106 self,
5107 "AuroraPgvectorCluster",
5108 engine=rds.DatabaseClusterEngine.aurora_postgres(
5109 version=getattr(rds.AuroraPostgresEngineVersion, AURORA_POSTGRES_VERSION),
5110 ),
5111 serverless_v2_min_capacity=aurora_config.get("min_acu", 0),
5112 serverless_v2_max_capacity=aurora_config.get("max_acu", 16),
5113 writer=rds.ClusterInstance.serverless_v2(
5114 "Writer",
5115 auto_minor_version_upgrade=True,
5116 ),
5117 readers=[
5118 rds.ClusterInstance.serverless_v2(
5119 "Reader",
5120 auto_minor_version_upgrade=True,
5121 scale_with_writer=True,
5122 ),
5123 ],
5124 vpc=self.vpc,
5125 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
5126 subnet_group=subnet_group,
5127 security_groups=[aurora_sg],
5128 default_database_name="gco_vectors",
5129 backup=rds.BackupProps(
5130 retention=Duration.days(aurora_config.get("backup_retention_days", 7)),
5131 ),
5132 deletion_protection=aurora_config.get("deletion_protection", False),
5133 removal_policy=RemovalPolicy.DESTROY,
5134 storage_encrypted=True,
5135 iam_authentication=True,
5136 cloudwatch_logs_exports=["postgresql"],
5137 monitoring_interval=Duration.seconds(60),
5138 cluster_identifier=f"{project_name}-pgvector-{self.deployment_region}",
5139 )
5141 # aws-cdk-lib >= 2.262 ships a built-in "CloudFormation Validate"
5142 # pack whose W9008 wants StorageEncrypted on every CfnDBInstance. The
5143 # cluster above sets storage_encrypted=True, and Aurora cluster
5144 # members inherit the cluster's storage encryption — the
5145 # instance-level property is not applicable to Aurora members, so the
5146 # finding cannot be satisfied at the instance. ``Validations.acknowledge``
5147 # is the API that feeds the validation report's suppression pass
5148 # (``Annotations.acknowledge_warning`` only silences the console
5149 # annotation). Note: the current CDK implementation collects these
5150 # acknowledgments app-wide per rule ID, so this quiets W9008
5151 # everywhere — attaching it here records this cluster as the
5152 # provenance in the report, and the five cdk-nag packs' own
5153 # RDS storage-encryption rules remain scoped and would still fail
5154 # a genuinely unencrypted instance elsewhere.
5155 Validations.of(self.aurora_cluster).acknowledge(
5156 Acknowledgment(
5157 id="CloudFormation-Validate::W9008",
5158 reason=(
5159 "Aurora cluster members inherit the cluster's "
5160 "storage_encrypted=True; StorageEncrypted is not applicable "
5161 "on Aurora member DBInstances."
5162 ),
5163 )
5164 )
5166 # Construct-level cdk-nag suppressions for Aurora pgvector
5167 from gco.stacks.nag_suppressions import NagSuppression, acknowledge_nag_findings
5169 acknowledge_nag_findings(
5170 self.aurora_cluster,
5171 [
5172 NagSuppression(
5173 id="AwsSolutions-RDS10",
5174 reason=(
5175 "Deletion protection is intentionally disabled for dev/test deployments. "
5176 "Production deployments should set aurora_pgvector.deletion_protection=true "
5177 "in cdk.json."
5178 ),
5179 ),
5180 NagSuppression(
5181 id="AwsSolutions-SMG4",
5182 reason=(
5183 "Aurora manages credential rotation via the RDS integration with Secrets "
5184 "Manager. Manual Secrets Manager rotation is not required. "
5185 "See: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/rds-secrets-manager.html"
5186 ),
5187 ),
5188 NagSuppression(
5189 id="HIPAA.Security-RDSInstanceDeletionProtectionEnabled",
5190 reason=(
5191 "Deletion protection is intentionally disabled for dev/test deployments. "
5192 "Production deployments should set aurora_pgvector.deletion_protection=true "
5193 "in cdk.json."
5194 ),
5195 ),
5196 NagSuppression(
5197 id="NIST.800.53.R5-RDSInstanceDeletionProtectionEnabled",
5198 reason=(
5199 "Deletion protection is intentionally disabled for dev/test deployments. "
5200 "Production deployments should set aurora_pgvector.deletion_protection=true "
5201 "in cdk.json."
5202 ),
5203 ),
5204 NagSuppression(
5205 id="PCI.DSS.321-SecretsManagerUsingKMSKey",
5206 reason=(
5207 "Aurora Serverless v2 credentials in Secrets Manager are encrypted with "
5208 "AWS-managed keys by default. Customer-managed KMS can be enabled if "
5209 "required for PCI compliance."
5210 ),
5211 ),
5212 ],
5213 )
5215 # Outputs
5216 CfnOutput(
5217 self,
5218 "AuroraPgvectorEndpoint",
5219 value=self.aurora_cluster.cluster_endpoint.hostname,
5220 description="Aurora pgvector cluster writer endpoint",
5221 )
5222 CfnOutput(
5223 self,
5224 "AuroraPgvectorReaderEndpoint",
5225 value=self.aurora_cluster.cluster_read_endpoint.hostname,
5226 description="Aurora pgvector cluster reader endpoint",
5227 )
5228 CfnOutput(
5229 self,
5230 "AuroraPgvectorPort",
5231 value=str(self.aurora_cluster.cluster_endpoint.port),
5232 description="Aurora pgvector cluster port",
5233 )
5234 CfnOutput(
5235 self,
5236 "AuroraPgvectorSecretArn",
5237 value=self.aurora_cluster.secret.secret_arn if self.aurora_cluster.secret else "",
5238 description="Aurora pgvector credentials secret ARN",
5239 )
5241 # Store endpoint in SSM for discovery by pods and external tools
5242 ssm.StringParameter(
5243 self,
5244 "AuroraPgvectorEndpointParam",
5245 parameter_name=f"/{project_name}/aurora-pgvector-endpoint-{self.deployment_region}",
5246 string_value=self.aurora_cluster.cluster_endpoint.hostname,
5247 description=f"Aurora pgvector endpoint for {self.deployment_region}",
5248 )
5250 # Grant the ServiceAccountRole read access to the Aurora secret
5251 # so pods can retrieve credentials via the ConfigMap + Secrets Manager.
5252 if self.aurora_cluster.secret: 5252 ↛ exitline 5252 didn't return from function '_create_aurora_pgvector' because the condition on line 5252 was always true
5253 self.aurora_cluster.secret.grant_read(self.service_account_role)
5255 def _create_fsx_csi_driver_addon(self) -> None:
5256 """Create FSx CSI Driver add-on for Kubernetes integration.
5258 The FSx CSI driver enables Kubernetes pods to mount FSx for Lustre
5259 file systems as persistent volumes.
5260 """
5261 # Create IAM role for FSx CSI Driver using IRSA + Pod Identity
5262 self.fsx_csi_role = GCORegionalStack._create_irsa_role(
5263 self,
5264 "FsxCsiDriverRole",
5265 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
5266 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
5267 service_account_names=["fsx-csi-controller-sa"],
5268 namespaces=["kube-system"],
5269 )
5271 # Add FSx CSI driver permissions
5272 self.fsx_csi_role.add_to_policy(
5273 iam.PolicyStatement(
5274 effect=iam.Effect.ALLOW,
5275 actions=[
5276 "fsx:DescribeFileSystems",
5277 "fsx:DescribeVolumes",
5278 "fsx:CreateVolume",
5279 "fsx:DeleteVolume",
5280 "fsx:TagResource",
5281 ],
5282 resources=["*"],
5283 )
5284 )
5286 self.fsx_csi_role.add_to_policy(
5287 iam.PolicyStatement(
5288 effect=iam.Effect.ALLOW,
5289 actions=[
5290 "ec2:DescribeInstances",
5291 "ec2:DescribeVolumes",
5292 "ec2:DescribeVpcs",
5293 "ec2:DescribeSubnets",
5294 "ec2:DescribeSecurityGroups",
5295 ],
5296 resources=["*"],
5297 )
5298 )
5300 # cdk-nag suppression: the FSx CSI driver role grants
5301 # ec2:Describe* APIs that don't support resource-level scoping.
5302 from gco.stacks.nag_suppressions import acknowledge_nag_findings
5304 acknowledge_nag_findings(
5305 self.fsx_csi_role,
5306 [
5307 {
5308 "id": "AwsSolutions-IAM5",
5309 "reason": (
5310 "The FSx CSI driver role grants ec2:Describe* for volume "
5311 "and network discovery. These AWS APIs do not support "
5312 "resource-level IAM scoping — Resource: * is the only "
5313 "valid form."
5314 ),
5315 "appliesTo": ["Resource::*"],
5316 },
5317 ],
5318 )
5320 # Create FSx CSI Driver add-on
5321 fsx_addon = eks.Addon(
5322 self,
5323 "FsxCsiDriverAddon",
5324 cluster=self.cluster, # type: ignore[arg-type]
5325 addon_name="aws-fsx-csi-driver",
5326 addon_version=EKS_ADDON_FSX_CSI_DRIVER,
5327 preserve_on_delete=False,
5328 configuration_values={
5329 "node": {
5330 "tolerations": self._ADDON_NODE_TOLERATIONS,
5331 },
5332 "controller": {
5333 "tolerations": self._ADDON_NODE_TOLERATIONS,
5334 },
5335 },
5336 )
5338 # Append the PassRole statement for the FSx CSI role to the shared
5339 # AwsCustomResource execution role. See
5340 # _create_aws_custom_resource_role for the full rationale.
5341 self.aws_custom_resource_role.add_to_policy(
5342 iam.PolicyStatement(
5343 effect=iam.Effect.ALLOW,
5344 actions=["iam:PassRole"],
5345 resources=[self.fsx_csi_role.role_arn],
5346 )
5347 )
5349 # Update the add-on to use the IRSA role
5350 update_fsx_addon = cr.AwsCustomResource(
5351 self,
5352 "UpdateFsxCsiAddonRole",
5353 on_create=cr.AwsSdkCall(
5354 service="EKS",
5355 action="updateAddon",
5356 parameters={
5357 "clusterName": self.cluster.cluster_name,
5358 "addonName": "aws-fsx-csi-driver",
5359 "serviceAccountRoleArn": self.fsx_csi_role.role_arn,
5360 },
5361 physical_resource_id=cr.PhysicalResourceId.of(
5362 f"{self.cluster.cluster_name}-fsx-csi-role-update"
5363 ),
5364 ),
5365 on_update=cr.AwsSdkCall(
5366 service="EKS",
5367 action="updateAddon",
5368 parameters={
5369 "clusterName": self.cluster.cluster_name,
5370 "addonName": "aws-fsx-csi-driver",
5371 "serviceAccountRoleArn": self.fsx_csi_role.role_arn,
5372 },
5373 ),
5374 role=self.aws_custom_resource_role,
5375 )
5377 update_fsx_addon.node.add_dependency(fsx_addon)
5378 update_fsx_addon.node.add_dependency(self.fsx_csi_role)
5379 update_fsx_addon.node.add_dependency(self.aws_custom_resource_role)
5381 # Expose the update-addon resource so _apply_kubernetes_manifests can
5382 # make the kubectl Lambda wait for the IRSA annotation patch to land
5383 # before it rollout-restarts the fsx-csi-controller. See the EFS CSI
5384 # equivalent for the full rationale — same race, same fix, same
5385 # symptom (PVCs stuck Pending with "no EC2 IMDS role found").
5386 self._fsx_csi_addon_role_update = update_fsx_addon
5388 # Create Pod Identity Association for FSx CSI driver
5389 eks_l1.CfnPodIdentityAssociation(
5390 self,
5391 "PodIdentity-fsx-csi",
5392 cluster_name=self.cluster.cluster_name,
5393 namespace="kube-system",
5394 service_account="fsx-csi-controller-sa",
5395 role_arn=self.fsx_csi_role.role_arn,
5396 )
5398 def _create_drift_detection(self) -> None:
5399 """Create CloudFormation drift detection on a daily schedule.
5401 Creates:
5402 - SNS topic (KMS-encrypted) for drift alerts
5403 - Lambda function that initiates drift detection on this stack, polls
5404 until detection completes, and publishes to SNS if drift is found
5405 - EventBridge rule on a daily schedule (configurable via cdk.json
5406 ``drift_detection.schedule_hours``) that invokes the Lambda
5408 Operators can disable drift detection entirely by setting
5409 ``drift_detection.enabled`` to ``false`` in cdk.json. When disabled,
5410 no resources are created.
5411 """
5412 drift_config = self.node.try_get_context("drift_detection") or {}
5413 if not drift_config.get("enabled", True):
5414 return
5416 schedule_hours = int(drift_config.get("schedule_hours", 24))
5418 # KMS key for SNS topic encryption. SNS with AWS-managed keys doesn't
5419 # allow CloudFormation/Lambda to publish, so we use a customer-managed
5420 # key we can grant publish access on.
5421 drift_topic_key = kms.Key(
5422 self,
5423 "DriftDetectionTopicKey",
5424 description="KMS key for GCO drift detection SNS topic",
5425 enable_key_rotation=True,
5426 removal_policy=RemovalPolicy.DESTROY,
5427 )
5429 self.drift_detection_topic = sns.Topic(
5430 self,
5431 "DriftDetectionTopic",
5432 display_name="GCO CloudFormation Drift Alerts",
5433 master_key=drift_topic_key,
5434 )
5436 # IAM role for the drift detection Lambda
5437 drift_lambda_role = iam.Role(
5438 self,
5439 "DriftDetectionLambdaRole",
5440 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
5441 managed_policies=[
5442 iam.ManagedPolicy.from_aws_managed_policy_name(
5443 "service-role/AWSLambdaBasicExecutionRole"
5444 ),
5445 ],
5446 )
5448 # CloudFormation drift APIs operate at the stack level; the API does
5449 # not support resource-level ARN scoping for these actions, so we scope
5450 # to this stack's ARN where supported and accept "*" where not.
5451 drift_lambda_role.add_to_policy(
5452 iam.PolicyStatement(
5453 effect=iam.Effect.ALLOW,
5454 actions=[
5455 "cloudformation:DetectStackDrift",
5456 "cloudformation:DescribeStackDriftDetectionStatus",
5457 "cloudformation:DescribeStackResourceDrifts",
5458 "cloudformation:DescribeStackResource",
5459 "cloudformation:DescribeStackResources",
5460 ],
5461 resources=["*"],
5462 )
5463 )
5465 self.drift_detection_topic.grant_publish(drift_lambda_role)
5467 # Lambda function — one per stack; stack name is baked into env vars
5468 drift_lambda = lambda_.Function(
5469 self,
5470 "DriftDetectionFunction",
5471 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
5472 handler="handler.lambda_handler",
5473 code=lambda_.Code.from_asset("lambda/drift-detection"),
5474 timeout=Duration.minutes(14), # Leave headroom under Lambda 15-min cap
5475 memory_size=256,
5476 role=drift_lambda_role,
5477 environment={
5478 "STACK_NAME": self.stack_name,
5479 "SNS_TOPIC_ARN": self.drift_detection_topic.topic_arn,
5480 "REGION": self.deployment_region,
5481 },
5482 tracing=lambda_.Tracing.ACTIVE,
5483 )
5485 # Dead-letter queue for EventBridge → Lambda target failures.
5486 # Captures events that fail to reach the Lambda (e.g. due to
5487 # throttling or permission issues) so operators can retry or
5488 # investigate. Required by Serverless-EventBusDLQ cdk-nag rule.
5489 drift_rule_dlq = sqs.Queue(
5490 self,
5491 "DriftDetectionRuleDlq",
5492 retention_period=Duration.days(14),
5493 enforce_ssl=True,
5494 encryption=sqs.QueueEncryption.SQS_MANAGED,
5495 removal_policy=RemovalPolicy.DESTROY,
5496 )
5498 # DLQs themselves are terminal — they don't need their own DLQ.
5499 # Suppress the circular AwsSolutions-SQS3 nag finding.
5500 from gco.stacks.nag_suppressions import acknowledge_nag_findings
5502 acknowledge_nag_findings(
5503 drift_rule_dlq,
5504 [
5505 {
5506 "id": "AwsSolutions-SQS3",
5507 "reason": (
5508 "This queue IS the dead-letter queue for the "
5509 "DriftDetectionSchedule EventBridge rule. A DLQ for a "
5510 "DLQ is circular; if events fail to reach this queue "
5511 "they are captured by EventBridge's own retry metrics "
5512 "(CloudWatch FailedInvocations)."
5513 ),
5514 },
5515 ],
5516 )
5518 # EventBridge rule — daily schedule by default
5519 events.Rule(
5520 self,
5521 "DriftDetectionSchedule",
5522 description=(f"Daily CloudFormation drift detection for {self.stack_name}"),
5523 schedule=events.Schedule.rate(Duration.hours(schedule_hours)),
5524 targets=[
5525 events_targets.LambdaFunction(
5526 drift_lambda,
5527 dead_letter_queue=drift_rule_dlq,
5528 retry_attempts=2,
5529 )
5530 ],
5531 )
5533 # Outputs for operators to subscribe to the topic
5534 CfnOutput(
5535 self,
5536 "DriftDetectionTopicArn",
5537 value=self.drift_detection_topic.topic_arn,
5538 description=(
5539 f"SNS topic ARN for CloudFormation drift alerts in "
5540 f"{self.deployment_region}. Subscribe an endpoint (email, "
5541 f"Slack, PagerDuty) to receive drift notifications."
5542 ),
5543 )
5545 # cdk-nag suppressions for this component
5546 from gco.stacks.nag_suppressions import acknowledge_nag_findings
5548 acknowledge_nag_findings(
5549 drift_lambda_role,
5550 [
5551 {
5552 "id": "AwsSolutions-IAM4",
5553 "reason": (
5554 "AWSLambdaBasicExecutionRole provides standard "
5555 "CloudWatch Logs permissions required for Lambda "
5556 "logging. This is the AWS-recommended managed policy."
5557 ),
5558 },
5559 {
5560 "id": "AwsSolutions-IAM5",
5561 "reason": (
5562 "CloudFormation drift detection APIs (DetectStackDrift, "
5563 "DescribeStackDriftDetectionStatus, "
5564 "DescribeStackResourceDrifts) cannot be scoped to a "
5565 "specific stack resource via IAM; the action-level "
5566 "scoping requires wildcard resources. The Lambda's "
5567 "environment pins it to a single stack name, so the "
5568 "effective blast radius is limited. The "
5569 "``kms:GenerateDataKey*`` action wildcard is the "
5570 "AWS-recommended grant for publishing to the "
5571 "KMS-encrypted drift-detection SNS topic."
5572 ),
5573 "appliesTo": [
5574 "Resource::*",
5575 "Action::kms:GenerateDataKey*",
5576 ],
5577 },
5578 ],
5579 )
5581 def _create_mcp_role(self) -> None:
5582 """Create dedicated IAM role for the MCP server.
5584 The MCP server exposes GCO CLI tools to LLM agents. Without a dedicated
5585 role, the server would inherit the full ambient credentials of the user
5586 who launches it (often an administrator). This method creates a
5587 least-privilege role that the MCP server can assume at startup via
5588 ``GCO_MCP_ROLE_ARN``.
5590 Permissions are scoped to the minimum needed by the tools exposed:
5592 - ``eks:DescribeCluster`` on this regional EKS cluster ARN only.
5593 - ``s3:GetObject`` on model weights buckets. The model bucket lives in
5594 the global stack, so we scope to the same name pattern used by the
5595 service account role (``{project_name}-*``). This is a deliberate
5596 compromise: a precise cross-stack ARN export would force a tight
5597 dependency on the global stack, and cdk-nag will flag it anyway
5598 because the bucket name is auto-generated.
5599 - ``cloudwatch:GetMetricData`` / ``cloudwatch:ListMetrics``. These APIs
5600 do not support resource-level IAM, so wildcard is required. Read-only.
5601 - ``sqs:SendMessage`` scoped to this region's job queue ARN only.
5603 The trust policy uses ``AccountRootPrincipal`` so any IAM user/role in
5604 the account can assume it (gated by an explicit sts:AssumeRole
5605 permission on the caller — standard AWS behavior). Operators who want
5606 to restrict assumption further should add an external-id or principal
5607 condition to the trust policy after deployment.
5609 Operators can disable this component entirely by setting
5610 ``mcp_server.enabled`` to ``false`` in cdk.json.
5611 """
5612 mcp_config = self.node.try_get_context("mcp_server") or {}
5613 if not mcp_config.get("enabled", True):
5614 return
5616 project_name = self.config.get_project_name()
5618 self.mcp_server_role = iam.Role(
5619 self,
5620 "McpServerRole",
5621 assumed_by=iam.AccountRootPrincipal(),
5622 description=(
5623 "Least-privilege role assumed by the GCO MCP server at startup. "
5624 "Grants only the permissions needed by MCP tools: eks:DescribeCluster, "
5625 "s3:GetObject on model buckets, cloudwatch read-only metrics, and "
5626 "sqs:SendMessage to the regional job queue."
5627 ),
5628 max_session_duration=Duration.hours(12),
5629 )
5631 # eks:DescribeCluster on this region's cluster only
5632 self.mcp_server_role.add_to_policy(
5633 iam.PolicyStatement(
5634 effect=iam.Effect.ALLOW,
5635 actions=["eks:DescribeCluster"],
5636 resources=[self.cluster.cluster_arn],
5637 )
5638 )
5640 # s3:GetObject on model weights buckets. Bucket name is auto-generated
5641 # in the global stack, so we match the same prefix pattern used by the
5642 # service account role.
5643 self.mcp_server_role.add_to_policy(
5644 iam.PolicyStatement(
5645 effect=iam.Effect.ALLOW,
5646 actions=["s3:GetObject", "s3:ListBucket"],
5647 resources=[
5648 f"arn:{self.partition}:s3:::{project_name}-*",
5649 f"arn:{self.partition}:s3:::{project_name}-*/*",
5650 ],
5651 )
5652 )
5654 # CloudWatch read-only metrics APIs. These APIs do not support
5655 # resource-level IAM so wildcard is required.
5656 self.mcp_server_role.add_to_policy(
5657 iam.PolicyStatement(
5658 effect=iam.Effect.ALLOW,
5659 actions=[
5660 "cloudwatch:GetMetricData",
5661 "cloudwatch:GetMetricStatistics",
5662 "cloudwatch:ListMetrics",
5663 ],
5664 resources=["*"],
5665 )
5666 )
5668 # sqs:SendMessage scoped to the regional job queue only
5669 self.mcp_server_role.add_to_policy(
5670 iam.PolicyStatement(
5671 effect=iam.Effect.ALLOW,
5672 actions=["sqs:SendMessage", "sqs:GetQueueUrl", "sqs:GetQueueAttributes"],
5673 resources=[self.job_queue.queue_arn],
5674 )
5675 )
5677 # Export the role ARN so operators can set GCO_MCP_ROLE_ARN in their
5678 # MCP server environment.
5679 CfnOutput(
5680 self,
5681 "McpServerRoleArn",
5682 value=self.mcp_server_role.role_arn,
5683 description=(
5684 "IAM role ARN for the GCO MCP server. Set GCO_MCP_ROLE_ARN to "
5685 "this value when launching the MCP server so it assumes a "
5686 "least-privilege role instead of ambient credentials."
5687 ),
5688 export_name=f"{project_name}-mcp-server-role-arn-{self.deployment_region}",
5689 )
5691 # cdk-nag suppressions: CloudWatch metrics APIs cannot be scoped.
5692 from gco.stacks.nag_suppressions import acknowledge_nag_findings
5694 acknowledge_nag_findings(
5695 self.mcp_server_role,
5696 [
5697 {
5698 "id": "AwsSolutions-IAM5",
5699 "reason": (
5700 "The CloudWatch metrics APIs (GetMetricData, "
5701 "GetMetricStatistics, ListMetrics) do not support "
5702 "resource-level IAM; wildcard resource is required. "
5703 "The S3 permissions use the {project_name}-* prefix "
5704 "pattern because the model weights bucket name is "
5705 "auto-generated by CDK in the global stack and a "
5706 "cross-stack ARN export would create tight stack "
5707 "coupling. All actions are read-only or scoped "
5708 "send-only (SQS)."
5709 ),
5710 "appliesTo": [
5711 "Resource::*",
5712 ],
5713 },
5714 ],
5715 )
5717 def _create_outputs(self) -> None:
5718 """Create CloudFormation outputs for cluster information"""
5719 project_name = self.config.get_project_name()
5721 # Export cluster information
5722 CfnOutput(
5723 self,
5724 "ClusterName",
5725 value=self.cluster.cluster_name,
5726 description=f"EKS cluster name for {self.deployment_region}",
5727 export_name=f"{project_name}-cluster-name-{self.deployment_region}",
5728 )
5730 CfnOutput(
5731 self,
5732 "AddonDeploymentToken",
5733 value=self.addon_deployment_token,
5734 description=(
5735 "Exact token for the asynchronous Kubernetes and Helm convergence execution"
5736 ),
5737 )
5739 CfnOutput(
5740 self,
5741 "ClusterArn",
5742 value=self.cluster.cluster_arn,
5743 description=f"EKS cluster ARN for {self.deployment_region}",
5744 export_name=f"{project_name}-cluster-arn-{self.deployment_region}",
5745 )
5747 CfnOutput(
5748 self,
5749 "ClusterEndpoint",
5750 value=self.cluster.cluster_endpoint,
5751 description=f"EKS cluster endpoint for {self.deployment_region}",
5752 export_name=f"{project_name}-cluster-endpoint-{self.deployment_region}",
5753 )
5755 CfnOutput(
5756 self,
5757 "ClusterSecurityGroupId",
5758 value=self.cluster.cluster_security_group_id,
5759 description=f"EKS cluster security group ID for {self.deployment_region}",
5760 export_name=f"{project_name}-cluster-sg-{self.deployment_region}",
5761 )
5763 CfnOutput(
5764 self,
5765 "VpcId",
5766 value=self.vpc.vpc_id,
5767 description=f"VPC ID for {self.deployment_region}",
5768 export_name=f"{project_name}-vpc-id-{self.deployment_region}",
5769 )
5771 # Export public subnet IDs for ALB
5772 public_subnet_ids = [subnet.subnet_id for subnet in self.vpc.public_subnets]
5773 CfnOutput(
5774 self,
5775 "PublicSubnetIds",
5776 value=Fn.join(",", public_subnet_ids),
5777 description=f"Public subnet IDs for ALB in {self.deployment_region}",
5778 export_name=f"{project_name}-public-subnets-{self.deployment_region}",
5779 )
5781 # Note: the ALB is created by the AWS Load Balancer Controller from the
5782 # gco-system/gco-gateway Gateway API resources; the GA registration
5783 # Lambda registers its ARN with Global Accelerator
5785 def get_cluster(self) -> eks.Cluster:
5786 """Get the EKS cluster"""
5787 return self.cluster
5789 def get_vpc(self) -> ec2.Vpc:
5790 """Get the VPC"""
5791 return self.vpc