Coverage for cli/ephemeral_bastion.py: 97.76%

1"""Ephemeral SSM bastion lifecycle for reaching a private EKS API endpoint.

3``gco monitoring open --via-ssm <instance-id>`` tunnels to a private EKS

4endpoint through an *existing* SSM-managed instance. This module lets the CLI

5**create that instance on demand** — a minimal, self-terminating ``t3.micro`` in

6the cluster VPC — and tear it down when the port-forward session ends, so an

7operator doesn't have to keep a standing bastion around just to view Grafana.

9Orphan safeguards (defence in depth — a crash, ``Ctrl-C``, or a forgotten

10teardown must never leave a paid instance running):

12* launched with ``--instance-initiated-shutdown-behavior terminate``;

13* user-data schedules ``shutdown -h +<ttl>`` as an unconditional backstop;

14* IMDSv2 required (``HttpTokens=required``);

15* tagged ``gco:ephemeral=true`` + ``gco:purpose`` so any orphan is greppable;

16* the CLI tears it down in a ``finally:`` block.

18Network posture: the instance reuses the cluster's own security group (which is

19self-referencing, so it can reach the private API endpoint) and is placed in one

20of the cluster's private subnets **without a public IP**. GCO private subnets

21have NAT egress, which lets the SSM agent reach Systems Manager over HTTPS while

22keeping the instance unreachable from the internet. No inbound ports are opened;

23SSM is agent-initiated outbound only.

25Style matches :mod:`cli.ssm_tunnel`: pure, validated argv builders (list form,

26never a shell string) that are fully unit-testable, plus thin runtime wrappers

27that shell out to the AWS CLI (the ``cli`` package does not depend on boto3).

28"""

30from __future__ import annotations

32import json

33import logging

34import re

35import subprocess

36import time

37from collections.abc import Iterator

38from contextlib import contextmanager

39from dataclasses import dataclass

41from ._image_uri import aws_partition, aws_url_suffix

43logger = logging.getLogger(__name__)

45# --------------------------------------------------------------------------

46# Constants — the fixed identity of the ephemeral bastion.

47# --------------------------------------------------------------------------

49# The bastion's IAM role, instance profile, and Name tag are scoped to the

50# deployment's project key (cdk.json context.project_name, default "gco" — the

51# same key cli/config.py derives cluster and stack names from), so a non-default

52# deployment addresses its own resources and two differently named deployments in

53# one account don't collide on a shared name. See bastion_role_name() below.

54DEFAULT_PROJECT_NAME = "gco"

55_ROLE_NAME_SUFFIX = "-ephemeral-bastion-role"

56_PROFILE_NAME_SUFFIX = "-ephemeral-bastion-profile"

57_INSTANCE_NAME_SUFFIX = "-ephemeral-ssm-bastion"

60# AmazonSSMManagedInstanceCore is AWS-managed and therefore not project-scoped,

61# but its ARN partition must follow the deployment Region.

62def ssm_managed_policy_arn(region: str) -> str:

63 return f"arn:{aws_partition(region)}:iam::aws:policy/AmazonSSMManagedInstanceCore"

66def bastion_trust_policy(region: str) -> dict[str, object]:

67 """Return an EC2 trust policy using the partition's service DNS suffix."""

68 return {

69 "Version": "2012-10-17",

70 "Statement": [

71 {

72 "Effect": "Allow",

73 "Principal": {"Service": f"ec2.{aws_url_suffix(region)}"},

74 "Action": "sts:AssumeRole",

75 }

76 ],

77 }

80# Back-compatible commercial defaults for callers that imported the constants.

81SSM_MANAGED_POLICY_ARN = ssm_managed_policy_arn("us-east-1")

83BASTION_INSTANCE_TYPE = "t3.micro"

85# Public SSM parameter that always resolves to the latest Amazon Linux 2023

86# x86_64 AMI in the target region (t3.micro is x86_64).

87AL2023_AMI_SSM_PARAMETER = "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64"

89# Tags stamped on every ephemeral instance so orphans are trivially discoverable:

90# aws ec2 describe-instances \

91# --filters Name=tag:gco:ephemeral,Values=true \

92# Name=instance-state-name,Values=running,pending

93TAG_EPHEMERAL_KEY = "gco:ephemeral"

94TAG_PURPOSE_KEY = "gco:purpose"

95TAG_PROJECT_KEY = "gco:project"

96TAG_TTL_KEY = "gco:ttl-minutes"

97BASTION_PURPOSE = "cluster-observability"

99DEFAULT_TTL_MINUTES = 120

100

101# EC2 instance-profile propagation to the RunInstances API is eventually

102# consistent; retry the launch for a short window on the "not found" error.

103_PROFILE_PROPAGATION_RETRIES = 6

104_PROFILE_PROPAGATION_WAIT_SECONDS = 5.0

105

106# EC2 trust policy default; builders derive the principal from their target Region.

107BASTION_TRUST_POLICY: dict[str, object] = bastion_trust_policy("us-east-1")

108

109# --------------------------------------------------------------------------

110# Validators — every AWS-supplied id is re-validated before it enters an argv.

111# --------------------------------------------------------------------------

112

113_REGION_RE = re.compile(r"^[a-z]{2,4}(?:-[a-z0-9]+)+-[0-9]+$")

114_CLUSTER_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9\-]{0,99}$")

115_INSTANCE_RE = re.compile(r"^i-[0-9a-f]{8}([0-9a-f]{9})?$")

116_VPC_RE = re.compile(r"^vpc-[0-9a-f]{8}([0-9a-f]{9})?$")

117_SUBNET_RE = re.compile(r"^subnet-[0-9a-f]{8}([0-9a-f]{9})?$")

118_SG_RE = re.compile(r"^sg-[0-9a-f]{8}([0-9a-f]{9})?$")

119_AMI_RE = re.compile(r"^ami-[0-9a-f]{8}([0-9a-f]{9})?$")

120_INSTANCE_TYPE_RE = re.compile(r"^[a-z0-9]+\.[a-z0-9]+$")

121

122

123def _validate(value: str, pattern: re.Pattern[str], what: str) -> str:

124 if not isinstance(value, str) or not pattern.match(value):

125 raise ValueError(f"Invalid {what}: {value!r}")

126 return value

127

128

129def _validate_ttl(ttl_minutes: int) -> int:

130 try:

131 value = int(ttl_minutes)

132 except (TypeError, ValueError) as exc:

133 raise ValueError(f"Invalid ttl-minutes {ttl_minutes!r}: must be an integer") from exc

134 if not 5 <= value <= 1440:

135 raise ValueError(f"Invalid ttl-minutes {value}: must be between 5 and 1440")

136 return value

137

138

139# Project keys follow the cdk.json context.project_name shape (alnum + hyphen).

140_PROJECT_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9\-]{0,40}$")

141

142

143def _validate_project(project_name: str) -> str:

144 return _validate(project_name, _PROJECT_RE, "project name")

145

146

147def bastion_role_name(project_name: str = DEFAULT_PROJECT_NAME) -> str:

148 """IAM role name for ``project_name``'s ephemeral bastion."""

149 return f"{_validate_project(project_name)}{_ROLE_NAME_SUFFIX}"

150

151

152def bastion_profile_name(project_name: str = DEFAULT_PROJECT_NAME) -> str:

153 """Instance-profile name for ``project_name``'s ephemeral bastion."""

154 return f"{_validate_project(project_name)}{_PROFILE_NAME_SUFFIX}"

155

156

157def bastion_instance_name(project_name: str = DEFAULT_PROJECT_NAME) -> str:

158 """EC2 ``Name`` tag for ``project_name``'s ephemeral bastion."""

159 return f"{_validate_project(project_name)}{_INSTANCE_NAME_SUFFIX}"

160

161

162# Back-compat convenience constants for the default project. The helpers above

163# are the source of truth; these are the default-project values that the builder

164# defaults (and tests) reference.

165BASTION_ROLE_NAME = bastion_role_name()

166BASTION_PROFILE_NAME = bastion_profile_name()

167BASTION_NAME = bastion_instance_name()

168

169

170@dataclass(frozen=True)

171class BastionNetwork:

172 """Private VPC placement for an ephemeral bastion."""

173

174 vpc_id: str

175 subnet_id: str

176 security_group_id: str

177

178

179# --------------------------------------------------------------------------

180# Pure argv / payload builders (unit-tested; list form, no shell).

181# --------------------------------------------------------------------------

182

183

184def render_user_data(ttl_minutes: int = DEFAULT_TTL_MINUTES) -> str:

185 """Return the bastion boot script: an unconditional self-terminate backstop."""

186 ttl = _validate_ttl(ttl_minutes)

187 return (

188 "#!/bin/bash\n"

189 "# GCO ephemeral SSM bastion — self-terminate backstop.\n"

190 "# The instance is launched with --instance-initiated-shutdown-behavior\n"

191 "# terminate, so this scheduled halt terminates it even if no explicit\n"

192 "# teardown ever runs.\n"

193 f'shutdown -h +{ttl} "gco ephemeral bastion self-terminate backstop"\n'

194 )

195

196

197def build_get_ami_command(region: str) -> list[str]:

198 """``aws ssm get-parameter`` argv resolving the latest AL2023 x86_64 AMI."""

199 _validate(region, _REGION_RE, "region")

200 return [

201 "aws",

202 "ssm",

203 "get-parameter",

204 "--name",

205 AL2023_AMI_SSM_PARAMETER,

206 "--region",

207 region,

208 "--query",

209 "Parameter.Value",

210 "--output",

211 "text",

212 ]

213

214

215def build_describe_cluster_network_command(cluster: str, region: str) -> list[str]:

216 """``aws eks describe-cluster`` argv projecting VPC, cluster SG, and subnets."""

217 _validate(cluster, _CLUSTER_RE, "cluster name")

218 _validate(region, _REGION_RE, "region")

219 return [

220 "aws",

221 "eks",

222 "describe-cluster",

223 "--name",

224 cluster,

225 "--region",

226 region,

227 "--query",

228 ("cluster.resourcesVpcConfig.{vpc:vpcId,sg:clusterSecurityGroupId,subnets:subnetIds}"),

229 "--output",

230 "json",

231 ]

232

233

234def build_describe_private_cluster_subnet_command(subnet_ids: list[str], region: str) -> list[str]:

235 """Find the first private subnet among the EKS control-plane subnets."""

236 if not subnet_ids: 236 ↛ 237line 236 didn't jump to line 237 because the condition on line 236 was never true

237 raise ValueError("At least one cluster subnet id is required")

238 validated_subnets = [

239 _validate(subnet_id, _SUBNET_RE, "cluster subnet id") for subnet_id in subnet_ids

240 ]

241 _validate(region, _REGION_RE, "region")

242 return [

243 "aws",

244 "ec2",

245 "describe-subnets",

246 "--region",

247 region,

248 "--subnet-ids",

249 *validated_subnets,

250 "--filters",

251 "Name=map-public-ip-on-launch,Values=false",

252 "--query",

253 "Subnets[0].SubnetId",

254 "--output",

255 "text",

256 ]

257

258

259def build_create_role_command(

260 role_name: str = BASTION_ROLE_NAME,

261 region: str = "us-east-1",

262) -> list[str]:

263 """``aws iam create-role`` argv with the partition-correct EC2 trust."""

264 _validate(role_name, _CLUSTER_RE, "role name")

265 _validate(region, _REGION_RE, "region")

266 return [

267 "aws",

268 "iam",

269 "create-role",

270 "--region",

271 region,

272 "--role-name",

273 role_name,

274 "--assume-role-policy-document",

275 json.dumps(bastion_trust_policy(region)),

276 "--description",

277 "GCO ephemeral SSM bastion (cluster-observability); safe to delete.",

278 "--tags",

279 f"Key={TAG_EPHEMERAL_KEY},Value=true",

280 f"Key={TAG_PURPOSE_KEY},Value={BASTION_PURPOSE}",

281 ]

282

283

284def build_attach_role_policy_command(

285 role_name: str = BASTION_ROLE_NAME,

286 region: str = "us-east-1",

287) -> list[str]:

288 """``aws iam attach-role-policy`` argv attaching the partition policy."""

289 _validate(role_name, _CLUSTER_RE, "role name")

290 _validate(region, _REGION_RE, "region")

291 return [

292 "aws",

293 "iam",

294 "attach-role-policy",

295 "--region",

296 region,

297 "--role-name",

298 role_name,

299 "--policy-arn",

300 ssm_managed_policy_arn(region),

301 ]

302

303

304def build_create_instance_profile_command(

305 profile_name: str = BASTION_PROFILE_NAME,

306 region: str = "us-east-1",

307) -> list[str]:

308 """``aws iam create-instance-profile`` argv."""

309 _validate(profile_name, _CLUSTER_RE, "instance-profile name")

310 _validate(region, _REGION_RE, "region")

311 return [

312 "aws",

313 "iam",

314 "create-instance-profile",

315 "--region",

316 region,

317 "--instance-profile-name",

318 profile_name,

319 ]

320

321

322def build_add_role_to_profile_command(

323 role_name: str = BASTION_ROLE_NAME,

324 profile_name: str = BASTION_PROFILE_NAME,

325 region: str = "us-east-1",

326) -> list[str]:

327 """``aws iam add-role-to-instance-profile`` argv."""

328 _validate(role_name, _CLUSTER_RE, "role name")

329 _validate(profile_name, _CLUSTER_RE, "instance-profile name")

330 _validate(region, _REGION_RE, "region")

331 return [

332 "aws",

333 "iam",

334 "add-role-to-instance-profile",

335 "--region",

336 region,

337 "--instance-profile-name",

338 profile_name,

339 "--role-name",

340 role_name,

341 ]

342

343

344def _tag_specification(

345 ttl_minutes: int,

346 instance_name: str = BASTION_NAME,

347 project_name: str = DEFAULT_PROJECT_NAME,

348) -> str:

349 """Build the ``--tag-specifications`` value stamping the ephemeral markers."""

350 project = _validate_project(project_name)

351 tags = (

352 f"{{Key=Name,Value={instance_name}}},"

353 f"{{Key={TAG_EPHEMERAL_KEY},Value=true}},"

354 f"{{Key={TAG_PURPOSE_KEY},Value={BASTION_PURPOSE}}},"

355 f"{{Key={TAG_PROJECT_KEY},Value={project}}},"

356 f"{{Key={TAG_TTL_KEY},Value={ttl_minutes}}}"

357 )

358 return f"ResourceType=instance,Tags=[{tags}]"

359

360

361def build_run_instances_command(

362 *,

363 ami_id: str,

364 instance_type: str,

365 subnet_id: str,

366 security_group_id: str,

367 profile_name: str,

368 region: str,

369 user_data: str,

370 ttl_minutes: int = DEFAULT_TTL_MINUTES,

371 instance_name: str = BASTION_NAME,

372 project_name: str = DEFAULT_PROJECT_NAME,

373) -> list[str]:

374 """Build the validated ``aws ec2 run-instances`` argv, safeguards included.

375

376 The safeguards (IMDSv2, shutdown-behaviour terminate, TTL user-data, and the

377 ``gco:ephemeral`` tag set) are not optional — they are the contract that

378 keeps a forgotten teardown from turning into a paid orphan.

379 """

380 _validate(ami_id, _AMI_RE, "AMI id")

381 _validate(subnet_id, _SUBNET_RE, "subnet id")

382 _validate(security_group_id, _SG_RE, "security group id")

383 _validate(profile_name, _CLUSTER_RE, "instance-profile name")

384 _validate(region, _REGION_RE, "region")

385 ttl = _validate_ttl(ttl_minutes)

386 if not _INSTANCE_TYPE_RE.match(instance_type):

387 raise ValueError(f"Invalid instance type {instance_type!r}")

388

389 cmd = [

390 "aws",

391 "ec2",

392 "run-instances",

393 "--region",

394 region,

395 "--image-id",

396 ami_id,

397 "--instance-type",

398 instance_type,

399 "--count",

400 "1",

401 "--subnet-id",

402 subnet_id,

403 "--security-group-ids",

404 security_group_id,

405 "--iam-instance-profile",

406 f"Name={profile_name}",

407 # IMDSv2 required.

408 "--metadata-options",

409 "HttpTokens=required,HttpEndpoint=enabled",

410 # Orphan safeguard #1: an OS-initiated shutdown terminates the instance.

411 "--instance-initiated-shutdown-behavior",

412 "terminate",

413 # Orphan safeguard #2: schedule that shutdown from boot.

414 "--user-data",

415 user_data,

416 # Orphan safeguard #3: greppable ephemeral tags.

417 "--tag-specifications",

418 _tag_specification(ttl, instance_name, project_name),

419 ]

420 cmd.append("--no-associate-public-ip-address")

421 # Ask only for the instance id back.

422 cmd += ["--query", "Instances[0].InstanceId", "--output", "text"]

423 return cmd

424

425

426def build_describe_ssm_ping_command(instance_id: str, region: str) -> list[str]:

427 """``aws ssm describe-instance-information`` argv projecting the PingStatus."""

428 _validate(instance_id, _INSTANCE_RE, "instance id")

429 _validate(region, _REGION_RE, "region")

430 return [

431 "aws",

432 "ssm",

433 "describe-instance-information",

434 "--region",

435 region,

436 "--filters",

437 f"Key=InstanceIds,Values={instance_id}",

438 "--query",

439 "InstanceInformationList[0].PingStatus",

440 "--output",

441 "text",

442 ]

443

444

445def build_terminate_instances_command(instance_id: str, region: str) -> list[str]:

446 """``aws ec2 terminate-instances`` argv."""

447 _validate(instance_id, _INSTANCE_RE, "instance id")

448 _validate(region, _REGION_RE, "region")

449 return [

450 "aws",

451 "ec2",

452 "terminate-instances",

453 "--region",

454 region,

455 "--instance-ids",

456 instance_id,

457 "--query",

458 "TerminatingInstances[0].CurrentState.Name",

459 "--output",

460 "text",

461 ]

462

463

464def build_iam_teardown_commands(

465 role_name: str = BASTION_ROLE_NAME,

466 profile_name: str = BASTION_PROFILE_NAME,

467 region: str = "us-east-1",

468) -> list[list[str]]:

469 """Return ordered, partition-aware IAM teardown argvs."""

470 _validate(role_name, _CLUSTER_RE, "role name")

471 _validate(profile_name, _CLUSTER_RE, "instance-profile name")

472 _validate(region, _REGION_RE, "region")

473 policy_arn = ssm_managed_policy_arn(region)

474 region_args = ["--region", region]

475 return [

476 [

477 "aws",

478 "iam",

479 "remove-role-from-instance-profile",

480 *region_args,

481 "--instance-profile-name",

482 profile_name,

483 "--role-name",

484 role_name,

485 ],

486 [

487 "aws",

488 "iam",

489 "delete-instance-profile",

490 *region_args,

491 "--instance-profile-name",

492 profile_name,

493 ],

494 [

495 "aws",

496 "iam",

497 "detach-role-policy",

498 *region_args,

499 "--role-name",

500 role_name,

501 "--policy-arn",

502 policy_arn,

503 ],

504 ["aws", "iam", "delete-role", *region_args, "--role-name", role_name],

505 ]

506

507

508# --------------------------------------------------------------------------

509# Pure parsers (unit-tested).

510# --------------------------------------------------------------------------

511

512

513def parse_cluster_network(stdout: str) -> tuple[str, str, list[str]]:

514 """Parse ``build_describe_cluster_network_command`` JSON → (vpc, sg, subnets)."""

515 data = json.loads(stdout or "{}")

516 vpc = data.get("vpc") or ""

517 sg = data.get("sg") or ""

518 subnets = data.get("subnets") or []

519 if not vpc or not sg:

520 raise RuntimeError(

521 "Cluster VPC or cluster security group not found in describe-cluster output; "

522 "cannot place an ephemeral bastion."

523 )

524 return vpc, sg, list(subnets)

525

526

527def _clean_scalar(stdout: str) -> str:

528 """Normalise an ``--output text`` scalar (strip; treat 'None' as empty)."""

529 value = (stdout or "").strip()

530 return "" if value in ("", "None") else value

531

532

533# --------------------------------------------------------------------------

534# Runtime wrappers (thin; shell out to the AWS CLI).

535# --------------------------------------------------------------------------

536

537

538def _run_aws(cmd: list[str], *, allow_exists: bool = False) -> str:

539 """Run an AWS CLI argv, returning stdout. Raise ``RuntimeError`` on failure.

540

541 ``allow_exists`` swallows idempotent "already exists" IAM errors so repeated

542 runs reuse the standing role/profile instead of failing.

543 """

544 try:

545 result = subprocess.run( # nosemgrep: dangerous-subprocess-use-audit - argv built by validated builders; list form, no shell=True

546 cmd, capture_output=True, text=True

547 )

548 except FileNotFoundError as exc:

549 raise RuntimeError(

550 "AWS CLI not found. Please install the AWS CLI and ensure it's in your PATH."

551 ) from exc

552 if result.returncode != 0:

553 stderr = result.stderr or ""

554 if allow_exists and (

555 "EntityAlreadyExists" in stderr

556 or "already exists" in stderr

557 # IAM reports an idempotent add-role-to-instance-profile call as

558 # this quota error when the profile already contains its one role.

559 or "Cannot exceed quota for InstanceSessionsPerInstanceProfile" in stderr

560 ):

561 return result.stdout or ""

562 raise RuntimeError(f"AWS CLI command failed ({' '.join(cmd[:3])}): {stderr.strip()}")

563 return result.stdout or ""

564

565

566def resolve_bastion_ami(region: str) -> str:

567 """Resolve the latest AL2023 x86_64 AMI id for ``region``."""

568 ami = _clean_scalar(_run_aws(build_get_ami_command(region)))

569 return _validate(ami, _AMI_RE, "resolved AMI id")

570

571

572def resolve_bastion_network(cluster: str, region: str) -> BastionNetwork:

573 """Discover private VPC placement for the bastion.

574

575 GCO gives EKS private control-plane subnets with NAT egress. We select one

576 of those subnets and refuse a public-subnet fallback, ensuring the bastion

577 never receives a public IP. A deployment without NAT can provide Systems

578 Manager interface VPC endpoints instead.

579 """

580 vpc, sg, subnets = parse_cluster_network(

581 _run_aws(build_describe_cluster_network_command(cluster, region))

582 )

583 _validate(vpc, _VPC_RE, "cluster vpc id")

584 _validate(sg, _SG_RE, "cluster security group id")

585

586 if not subnets:

587 raise RuntimeError(

588 f"No cluster subnets found in VPC {vpc}; cannot place a private bastion."

589 )

590 private_subnet = _clean_scalar(

591 _run_aws(build_describe_private_cluster_subnet_command(list(subnets), region))

592 )

593 if not private_subnet:

594 raise RuntimeError(

595 f"No private EKS subnet found in VPC {vpc}; refusing to launch a public bastion."

596 )

597 return BastionNetwork(

598 vpc,

599 _validate(private_subnet, _SUBNET_RE, "private cluster subnet id"),

600 sg,

601 )

602

603

604def ensure_bastion_iam(

605 project_name: str = DEFAULT_PROJECT_NAME,

606 region: str = "us-east-1",

607) -> None:

608 """Idempotently create the bastion role + profile in one partition."""

609 role_name = bastion_role_name(project_name)

610 profile_name = bastion_profile_name(project_name)

611 _run_aws(build_create_role_command(role_name, region), allow_exists=True)

612 _run_aws(build_attach_role_policy_command(role_name, region), allow_exists=True)

613 _run_aws(build_create_instance_profile_command(profile_name, region), allow_exists=True)

614 _run_aws(

615 build_add_role_to_profile_command(role_name, profile_name, region),

616 allow_exists=True,

617 )

618

619

620def launch_bastion(

621 *,

622 network: BastionNetwork,

623 ami_id: str,

624 region: str,

625 ttl_minutes: int,

626 project_name: str = DEFAULT_PROJECT_NAME,

627 instance_type: str = BASTION_INSTANCE_TYPE,

628) -> str:

629 """Run the instance, retrying briefly while the instance profile propagates."""

630 cmd = build_run_instances_command(

631 ami_id=ami_id,

632 instance_type=instance_type,

633 subnet_id=network.subnet_id,

634 security_group_id=network.security_group_id,

635 profile_name=bastion_profile_name(project_name),

636 region=region,

637 user_data=render_user_data(ttl_minutes),

638 ttl_minutes=ttl_minutes,

639 instance_name=bastion_instance_name(project_name),

640 project_name=project_name,

641 )

642 last_error: Exception | None = None

643 for attempt in range(_PROFILE_PROPAGATION_RETRIES): 643 ↛ 659line 643 didn't jump to line 659 because the loop on line 643 didn't complete

644 try:

645 instance_id = _clean_scalar(_run_aws(cmd))

646 return _validate(instance_id, _INSTANCE_RE, "launched instance id")

647 except RuntimeError as exc: # noqa: PERF203 — bounded retry loop

648 last_error = exc

649 if (

650 "Invalid IAM Instance Profile" not in str(exc)

651 and "instance profile" not in str(exc).lower()

652 ):

653 raise

654 time.sleep(_PROFILE_PROPAGATION_WAIT_SECONDS)

655 logger.info(

656 "Instance profile not yet visible to EC2 (attempt %d); retrying launch.",

657 attempt + 1,

658 )

659 raise RuntimeError(

660 f"run-instances failed after {_PROFILE_PROPAGATION_RETRIES} attempts "

661 f"waiting for instance-profile propagation: {last_error}"

662 )

663

664

665def wait_until_ssm_online(

666 instance_id: str,

667 region: str,

668 *,

669 timeout_seconds: float = 240.0,

670 poll_interval_seconds: float = 10.0,

671) -> None:

672 """Block until the instance registers with SSM (PingStatus=Online) or time out."""

673 deadline = time.monotonic() + timeout_seconds

674 cmd = build_describe_ssm_ping_command(instance_id, region)

675 while time.monotonic() < deadline:

676 status = _clean_scalar(_run_aws(cmd))

677 if status == "Online": 677 ↛ 679line 677 didn't jump to line 679 because the condition on line 677 was always true

678 return

679 time.sleep(poll_interval_seconds)

680 raise RuntimeError(

681 f"Instance {instance_id} did not come Online in SSM within "

682 f"{int(timeout_seconds)}s. Check that the private subnet can reach the SSM service "

683 "through NAT egress or Systems Manager interface VPC endpoints."

684 )

685

686

687def create_ephemeral_bastion(

688 cluster: str,

689 region: str,

690 *,

691 project_name: str = DEFAULT_PROJECT_NAME,

692 ttl_minutes: int = DEFAULT_TTL_MINUTES,

693 wait_online: bool = True,

694) -> str:

695 """Provision a self-terminating SSM bastion in the cluster VPC. Returns its id.

696

697 The IAM role / instance profile are named for ``project_name`` (default

698 ``gco``) so they match the deployment's other project-scoped resources.

699 """

700 _validate(cluster, _CLUSTER_RE, "cluster name")

701 _validate(region, _REGION_RE, "region")

702 _validate_project(project_name)

703 ttl = _validate_ttl(ttl_minutes)

704

705 ami_id = resolve_bastion_ami(region)

706 network = resolve_bastion_network(cluster, region)

707 ensure_bastion_iam(project_name, region)

708 instance_id = launch_bastion(

709 network=network, ami_id=ami_id, region=region, ttl_minutes=ttl, project_name=project_name

710 )

711 logger.info("Launched ephemeral bastion %s in %s (%s).", instance_id, region, network.subnet_id)

712 if wait_online:

713 try:

714 wait_until_ssm_online(instance_id, region)

715 except Exception:

716 # Atomic create: never leak the instance we just launched if it

717 # fails to register with SSM. The self-terminate user-data is only a

718 # last-resort backstop, not the normal cleanup path.

719 try:

720 destroy_ephemeral_bastion(instance_id, region, project_name=project_name)

721 except Exception: # pragma: no cover - best effort

722 logger.exception(

723 "Failed to clean up bastion %s after online-wait failure", instance_id

724 )

725 raise

726 return instance_id

727

728

729def destroy_ephemeral_bastion(

730 instance_id: str,

731 region: str,

732 *,

733 project_name: str = DEFAULT_PROJECT_NAME,

734 delete_iam: bool = True,

735) -> None:

736 """Terminate the bastion and (best-effort) delete its IAM role + profile.

737

738 Termination is the cost-critical step and is always attempted. IAM cleanup is

739 best-effort: a leftover role/instance-profile costs nothing and is greppable

740 by its ``gco:ephemeral`` tag, so a failure here is logged, not raised. The

741 role / profile deleted are the ones named for ``project_name``.

742 """

743 _validate(instance_id, _INSTANCE_RE, "instance id")

744 _validate(region, _REGION_RE, "region")

745 _validate_project(project_name)

746

747 _run_aws(build_terminate_instances_command(instance_id, region))

748 logger.info("Terminating ephemeral bastion %s.", instance_id)

749

750 if not delete_iam:

751 return

752 teardown = build_iam_teardown_commands(

753 bastion_role_name(project_name),

754 bastion_profile_name(project_name),

755 region,

756 )

757 for step in teardown:

758 try:

759 _run_aws(step, allow_exists=True)

760 except RuntimeError as exc: # best-effort — never mask the termination

761 logger.warning("IAM teardown step failed (%s): %s", " ".join(step[:3]), exc)

762

763

764@contextmanager

765def ephemeral_bastion(

766 cluster: str,

767 region: str,

768 *,

769 project_name: str = DEFAULT_PROJECT_NAME,

770 ttl_minutes: int = DEFAULT_TTL_MINUTES,

771) -> Iterator[str]:

772 """Context manager: create a bastion, yield its id, guarantee teardown."""

773 instance_id = create_ephemeral_bastion(

774 cluster, region, project_name=project_name, ttl_minutes=ttl_minutes

775 )

776 try:

777 yield instance_id

778 finally:

779 destroy_ephemeral_bastion(instance_id, region, project_name=project_name)