Coverage for cli/commands/jobs_cmd.py: 90.08%

1"""Job management commands.""" 

2 

3import logging 

4import sys 

5from collections.abc import Mapping 

6from typing import Any 

7 

8import click 

9 

10from ..config import GCOConfig 

11from ..jobs import get_job_manager, resolve_submission_identity 

12from ..output import format_job_table, get_output_formatter 

13 

14logger = logging.getLogger(__name__) 

15 

16pass_config = click.make_pass_decorator(GCOConfig, ensure=True) 

17 

18 

19def _resolve_result_namespace(result: Any, fallback: str) -> str: 

20 """Pick the submitted Job namespace without assuming mapping resources.""" 

21 _job_name, namespace = resolve_submission_identity(result, fallback_namespace=fallback) 

22 return namespace or fallback 

23 

24 

25def _resolve_result_job_name(result: Any) -> str | None: 

26 """Pick the generated/submitted Job name from a submission response.""" 

27 job_name, _namespace = resolve_submission_identity(result) 

28 return job_name 

29 

30 

31@click.group() 

32@pass_config 

33def jobs(config: Any) -> None: 

34 """Manage jobs across GCO clusters.""" 

35 pass 

36 

37 

38@jobs.command("submit") 

39@click.argument("manifest_path", type=click.Path(exists=True)) 

40@click.option( 

41 "--namespace", 

42 "-n", 

43 help="Fallback namespace for manifests that don't declare their own", 

44) 

45@click.option("--region", "-r", "target_region", help="Target specific region") 

46@click.option("--dry-run", is_flag=True, help="Validate without applying") 

47@click.option("--label", "-l", multiple=True, help="Add labels (key=value)") 

48@click.option("--wait", "-w", is_flag=True, help="Wait for job completion") 

49@click.option("--timeout", default=3600, help="Wait timeout in seconds") 

50@pass_config 

51def submit_job( 

52 config: Any, 

53 manifest_path: Any, 

54 namespace: Any, 

55 target_region: Any, 

56 dry_run: Any, 

57 label: Any, 

58 wait: Any, 

59 timeout: Any, 

60) -> None: 

61 """Submit a job to GCO. 

62 

63 MANIFEST_PATH can be a YAML file or directory containing YAML files. 

64 """ 

65 formatter = get_output_formatter(config) 

66 job_manager = get_job_manager(config) 

67 

68 # Parse labels 

69 labels = {} 

70 for lbl in label: 

71 if "=" in lbl: 71 ↛ 70line 71 didn't jump to line 70 because the condition on line 71 was always true

72 k, v = lbl.split("=", 1) 

73 labels[k] = v 

74 

75 try: 

76 result = job_manager.submit_job( 

77 manifests=manifest_path, 

78 namespace=namespace, 

79 target_region=target_region, 

80 dry_run=dry_run, 

81 labels=labels if labels else None, 

82 ) 

83 

84 if dry_run: 

85 formatter.print_success("Dry run successful - manifests are valid") 

86 else: 

87 formatter.print_success("Job submitted successfully") 

88 

89 # Surface any rename warnings from mapping-shaped API resources. 

90 # Direct kubectl responses contain strings in ``resources``. 

91 resources = result.get("resources", []) if isinstance(result, Mapping) else [] 

92 for resource in resources: 

93 if not isinstance(resource, Mapping): 93 ↛ 94line 93 didn't jump to line 94 because the condition on line 93 was never true

94 continue 

95 msg = str(resource.get("message", "")) 

96 if "renamed" in msg.lower() or "still running" in msg.lower(): 96 ↛ 97line 96 didn't jump to line 97 because the condition on line 96 was never true

97 formatter.print_warning(msg) 

98 

99 formatter.print(result) 

100 

101 # Wait for completion if requested 

102 if wait and not dry_run: 

103 job_name = _resolve_result_job_name(result) 

104 if job_name: 104 ↛ exitline 104 didn't return from function 'submit_job' because the condition on line 104 was always true

105 # The API response tells us exactly where the resource landed 

106 # (may differ from --namespace since the manifest's own value 

107 # takes precedence). Fall back to the CLI flag or the config 

108 # default only if the response didn't include a namespace. 

109 resolved_ns = _resolve_result_namespace( 

110 result, fallback=namespace or config.default_namespace 

111 ) 

112 formatter.print_info(f"Waiting for job {job_name} to complete...") 

113 final_job = job_manager.wait_for_job( 

114 job_name=job_name, 

115 namespace=resolved_ns, 

116 region=target_region, 

117 timeout_seconds=timeout, 

118 ) 

119 formatter.print_success(f"Job completed with status: {final_job.status}") 

120 

121 except Exception as e: 

122 formatter.print_error(f"Failed to submit job: {e}") 

123 sys.exit(1) 

124 

125 

126@jobs.command("submit-direct") 

127@click.argument("manifest_path", type=click.Path(exists=True)) 

128@click.option("--region", "-r", required=True, help="Target region for direct submission") 

129@click.option( 

130 "--namespace", 

131 "-n", 

132 help="Fallback namespace for manifests that don't declare their own", 

133) 

134@click.option("--dry-run", is_flag=True, help="Validate without applying") 

135@click.option("--label", "-l", multiple=True, help="Add labels (key=value)") 

136@click.option("--wait", "-w", is_flag=True, help="Wait for job completion") 

137@click.option("--timeout", default=3600, help="Wait timeout in seconds") 

138@pass_config 

139def submit_job_direct( 

140 config: Any, 

141 manifest_path: Any, 

142 region: Any, 

143 namespace: Any, 

144 dry_run: Any, 

145 label: Any, 

146 wait: Any, 

147 timeout: Any, 

148) -> None: 

149 """Submit a job directly to a regional cluster using kubectl. 

150 

151 This bypasses the API Gateway and submits directly to the EKS cluster. 

152 

153 REQUIREMENTS: 

154 - kubectl installed and in PATH 

155 - EKS access entry configured for your IAM principal 

156 - AWS credentials with eks:DescribeCluster permission 

157 

158 To configure EKS access, run: 

159 

160 aws eks create-access-entry --cluster-name gco-REGION --principal-arn YOUR_ARN 

161 

162 aws eks associate-access-policy --cluster-name gco-REGION \\ 

163 --principal-arn YOUR_ARN \\ 

164 --policy-arn arn:<partition>:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy \\ 

165 --access-scope type=cluster 

166 

167 Examples: 

168 gco jobs submit-direct job.yaml --region us-east-1 

169 gco jobs submit-direct job.yaml -r us-west-2 -n gco-jobs --wait 

170 """ 

171 formatter = get_output_formatter(config) 

172 job_manager = get_job_manager(config) 

173 

174 # Parse labels 

175 labels = {} 

176 for lbl in label: 

177 if "=" in lbl: 177 ↛ 176line 177 didn't jump to line 176 because the condition on line 177 was always true

178 k, v = lbl.split("=", 1) 

179 labels[k] = v 

180 

181 try: 

182 formatter.print_info(f"Submitting directly to cluster in {region} via kubectl...") 

183 

184 result = job_manager.submit_job_direct( 

185 manifests=manifest_path, 

186 region=region, 

187 namespace=namespace, 

188 dry_run=dry_run, 

189 labels=labels if labels else None, 

190 ) 

191 

192 if dry_run: 

193 formatter.print_success("Dry run successful - manifests are valid") 

194 else: 

195 formatter.print_success(f"Job submitted directly to {region}") 

196 

197 # Surface any warnings (e.g. job was renamed due to name collision) 

198 # without mutating or assuming the shape of the direct result. 

199 warnings = result.get("warnings", []) if isinstance(result, Mapping) else [] 

200 for warning in warnings: 200 ↛ 201line 200 didn't jump to line 201 because the loop on line 200 never started

201 formatter.print_warning(str(warning)) 

202 

203 formatter.print(result) 

204 

205 # Wait for completion if requested 

206 if wait and not dry_run: 

207 job_name = _resolve_result_job_name(result) 

208 if job_name: 208 ↛ exitline 208 didn't return from function 'submit_job_direct' because the condition on line 208 was always true

209 resolved_ns = _resolve_result_namespace( 

210 result, fallback=namespace or config.default_namespace 

211 ) 

212 formatter.print_info(f"Waiting for job {job_name} to complete...") 

213 final_job = job_manager.wait_for_job( 

214 job_name=job_name, 

215 namespace=resolved_ns, 

216 region=region, 

217 timeout_seconds=timeout, 

218 ) 

219 formatter.print_success(f"Job completed with status: {final_job.status}") 

220 

221 except Exception as e: 

222 formatter.print_error(f"Failed to submit job directly: {e}") 

223 sys.exit(1) 

224 

225 

226@jobs.command("submit-sqs") 

227@click.argument("manifest_path", type=click.Path(exists=True)) 

228@click.option("--region", "-r", help="Target region (auto-selects optimal if not specified)") 

229@click.option( 

230 "--namespace", 

231 "-n", 

232 help="Fallback namespace for manifests that don't declare their own", 

233) 

234@click.option("--label", "-l", multiple=True, help="Add labels (key=value)") 

235@click.option("--priority", "-p", default=0, help="Job priority (higher = more important)") 

236@click.option("--auto-region", is_flag=True, help="Auto-select optimal region based on capacity") 

237@pass_config 

238def submit_job_sqs( 

239 config: Any, 

240 manifest_path: Any, 

241 region: Any, 

242 namespace: Any, 

243 label: Any, 

244 priority: Any, 

245 auto_region: Any, 

246) -> None: 

247 """Submit a job to a regional SQS queue for processing. 

248 

249 This is the recommended way to submit jobs as it: 

250 - Decouples submission from processing 

251 - Enables KEDA-based autoscaling 

252 - Provides better fault tolerance 

253 

254 If --auto-region is specified, the CLI will analyze capacity across all 

255 regions and submit to the optimal one. 

256 

257 Examples: 

258 gco jobs submit-sqs job.yaml --region us-east-1 

259 gco jobs submit-sqs job.yaml --auto-region 

260 gco jobs submit-sqs job.yaml -r us-west-2 --priority 10 

261 """ 

262 formatter = get_output_formatter(config) 

263 job_manager = get_job_manager(config) 

264 

265 # Parse labels 

266 labels = {} 

267 for lbl in label: 

268 if "=" in lbl: 268 ↛ 267line 268 didn't jump to line 267 because the condition on line 268 was always true

269 k, v = lbl.split("=", 1) 

270 labels[k] = v 

271 

272 try: 

273 # Auto-select region if requested 

274 if auto_region and not region: 

275 formatter.print_info("Analyzing capacity across regions...") 

276 from ..capacity import get_capacity_checker 

277 

278 checker = get_capacity_checker(config) 

279 recommendation = checker.recommend_region_for_job() 

280 region = recommendation["region"] 

281 formatter.print_info(f"Selected region: {region} ({recommendation['reason']})") 

282 elif not region: 

283 region = config.default_region 

284 

285 formatter.print_info(f"Submitting job to SQS queue in {region}...") 

286 

287 result = job_manager.submit_job_sqs( 

288 manifests=manifest_path, 

289 region=region, 

290 namespace=namespace, 

291 labels=labels if labels else None, 

292 priority=priority, 

293 ) 

294 

295 formatter.print_success(f"Job queued successfully in {region}") 

296 formatter.print(result) 

297 

298 except Exception as e: 

299 formatter.print_error(f"Failed to submit job to SQS: {e}") 

300 sys.exit(1) 

301 

302 

303@jobs.command("queue-status") 

304@click.option("--region", "-r", help="Specific region to check") 

305@click.option("--all-regions", "-a", is_flag=True, help="Check all regions") 

306@pass_config 

307def queue_status(config: Any, region: Any, all_regions: Any) -> None: 

308 """Show job queue status across regions. 

309 

310 Displays the number of pending, in-flight, and failed messages 

311 in the job queues. 

312 

313 Examples: 

314 gco jobs queue-status --region us-east-1 

315 gco jobs queue-status --all-regions 

316 """ 

317 formatter = get_output_formatter(config) 

318 job_manager = get_job_manager(config) 

319 

320 try: 

321 if all_regions: 

322 from ..aws_client import get_aws_client 

323 

324 aws_client = get_aws_client(config) 

325 stacks = aws_client.discover_regional_stacks() 

326 

327 results = [] 

328 for stack_region in stacks: 

329 try: 

330 status = job_manager.get_queue_status(stack_region) 

331 results.append(status) 

332 except Exception as e: 

333 logger.debug("Failed to get queue status for %s: %s", stack_region, e) 

334 continue 

335 

336 if not results: 

337 formatter.print_warning("No queue status available") 

338 return 

339 

340 # Format as table 

341 print("\n REGION PENDING IN-FLIGHT DELAYED DLQ") 

342 print(" " + "-" * 55) 

343 for r in results: 

344 dlq = r.get("dlq_messages", 0) 

345 print( 

346 f" {r['region']:<15} {r['messages_available']:>7} " 

347 f"{r['messages_in_flight']:>9} {r['messages_delayed']:>7} {dlq:>3}" 

348 ) 

349 else: 

350 target_region = region or config.default_region 

351 status = job_manager.get_queue_status(target_region) 

352 formatter.print(status) 

353 

354 except Exception as e: 

355 formatter.print_error(f"Failed to get queue status: {e}") 

356 sys.exit(1) 

357 

358 

359@jobs.command("list") 

360@click.option("--namespace", "-n", help="Filter by namespace") 

361@click.option("--region", "-r", help="Target region (required unless --all-regions)") 

362@click.option("--status", "-s", type=click.Choice(["pending", "running", "succeeded", "failed"])) 

363@click.option("--all-regions", "-a", is_flag=True, help="Query all regions via global API") 

364@click.option("--limit", "-l", default=50, help="Maximum jobs to return") 

365@pass_config 

366def list_jobs( 

367 config: Any, namespace: Any, region: Any, status: Any, all_regions: Any, limit: Any 

368) -> None: 

369 """List jobs in GCO clusters. 

370 

371 You must specify either --region for a specific cluster or --all-regions 

372 to query all clusters via the global aggregation API. 

373 

374 Examples: 

375 gco jobs list --region us-east-1 

376 gco jobs list --all-regions 

377 gco jobs list -r us-west-2 -n gco-jobs --status running 

378 """ 

379 formatter = get_output_formatter(config) 

380 job_manager = get_job_manager(config) 

381 

382 # Require explicit region or --all-regions 

383 if not region and not all_regions: 

384 formatter.print_error("You must specify --region or --all-regions") 

385 formatter.print_info(" Use --region/-r to query a specific cluster") 

386 formatter.print_info(" Use --all-regions/-a to query all clusters") 

387 sys.exit(1) 

388 

389 try: 

390 if all_regions: 

391 # Use global aggregation API 

392 result = job_manager.list_jobs_global( 

393 namespace=namespace, 

394 status=status, 

395 limit=limit, 

396 ) 

397 

398 if config.output_format == "table": 398 ↛ 433line 398 didn't jump to line 433 because the condition on line 398 was always true

399 # Print summary 

400 print("\n Global Jobs Summary") 

401 print(" " + "-" * 50) 

402 print(f" Total jobs: {result.get('total', 0)}") 

403 print(f" Regions queried: {result.get('regions_queried', 0)}") 

404 print(f" Regions successful: {result.get('regions_successful', 0)}") 

405 

406 # Print region summaries 

407 if result.get("region_summaries"): 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true

408 print("\n REGION COUNT TOTAL") 

409 print(" " + "-" * 35) 

410 for r in result["region_summaries"]: 

411 print(f" {r['region']:<15} {r['count']:>5} {r['total']:>5}") 

412 

413 # Print jobs 

414 jobs_data = result.get("jobs", []) 

415 if jobs_data: 415 ↛ 416line 415 didn't jump to line 416 because the condition on line 415 was never true

416 print( 

417 "\n NAME NAMESPACE REGION STATUS" 

418 ) 

419 print(" " + "-" * 75) 

420 for job in jobs_data[:limit]: 

421 name = job.get("metadata", {}).get("name", "")[:30] 

422 ns = job.get("metadata", {}).get("namespace", "")[:14] 

423 job_region = job.get("_source_region", "")[:14] 

424 job_status = job.get("computed_status", "unknown")[:10] 

425 print(f" {name:<30} {ns:<15} {job_region:<15} {job_status}") 

426 

427 # Print errors if any 

428 if result.get("errors"): 428 ↛ 429line 428 didn't jump to line 429 because the condition on line 428 was never true

429 print("\n Errors:") 

430 for err in result["errors"]: 

431 formatter.print_warning(f" {err['region']}: {err['error']}") 

432 else: 

433 formatter.print(result) 

434 else: 

435 # Query specific region 

436 jobs_list = job_manager.list_jobs( 

437 region=region, namespace=namespace, status=status, all_regions=False 

438 ) 

439 

440 if config.output_format == "table": 

441 print(format_job_table(jobs_list)) 

442 else: 

443 formatter.print(jobs_list) 

444 

445 except Exception as e: 

446 formatter.print_error(f"Failed to list jobs: {e}") 

447 sys.exit(1) 

448 

449 

450@jobs.command("get") 

451@click.argument("job_name") 

452@click.option("--namespace", "-n", default="gco-jobs", help="Job namespace") 

453@click.option("--region", "-r", required=True, help="Job region (required)") 

454@pass_config 

455def get_job(config: Any, job_name: Any, namespace: Any, region: Any) -> None: 

456 """Get details of a specific job. 

457 

458 Examples: 

459 gco jobs get my-job --region us-east-1 

460 gco jobs get training-job -r us-west-2 -n ml-jobs 

461 """ 

462 formatter = get_output_formatter(config) 

463 job_manager = get_job_manager(config) 

464 

465 try: 

466 job = job_manager.get_job(job_name, namespace, region) 

467 if job: 

468 formatter.print(job) 

469 else: 

470 formatter.print_error(f"Job {job_name} not found") 

471 sys.exit(1) 

472 except Exception as e: 

473 formatter.print_error(f"Failed to get job: {e}") 

474 sys.exit(1) 

475 

476 

477@jobs.command("logs") 

478@click.argument("job_name") 

479@click.option("--namespace", "-n", default="gco-jobs", help="Job namespace") 

480@click.option("--region", "-r", required=True, help="Job region (required)") 

481@click.option("--tail", "-t", default=100, help="Number of lines to show") 

482@click.option( 

483 "--since", "-s", default=24, type=int, help="Hours to look back in CloudWatch (default: 24)" 

484) 

485@click.option("--container", "-c", help="Container name (for multi-container pods)") 

486@pass_config 

487def get_logs( 

488 config: Any, job_name: Any, namespace: Any, region: Any, tail: Any, since: Any, container: Any 

489) -> None: 

490 """Get logs from a job. 

491 

492 Fetches logs from the Kubernetes API if the pod is still running. 

493 If the pod is gone, falls back to CloudWatch Logs automatically. 

494 Use --since to control how far back CloudWatch searches. 

495 

496 Examples: 

497 gco jobs logs my-job --region us-east-1 

498 gco jobs logs training-job -r us-west-2 -n ml-jobs --tail 500 

499 gco jobs logs old-job -r us-east-1 --since 72 

500 gco jobs logs multi-container-job -r us-east-1 --container sidecar 

501 """ 

502 formatter = get_output_formatter(config) 

503 job_manager = get_job_manager(config) 

504 

505 try: 

506 logs = job_manager.get_job_logs( 

507 job_name, namespace, region, tail_lines=tail, since_hours=since 

508 ) 

509 print(logs) 

510 except Exception as e: 

511 formatter.print_error(f"Failed to get logs: {e}") 

512 sys.exit(1) 

513 

514 

515@jobs.command("delete") 

516@click.argument("job_name") 

517@click.option("--namespace", "-n", default="gco-jobs", help="Job namespace") 

518@click.option("--region", "-r", required=True, help="Job region (required)") 

519@click.option("--yes", "-y", is_flag=True, help="Skip confirmation") 

520@pass_config 

521def delete_job(config: Any, job_name: Any, namespace: Any, region: Any, yes: Any) -> None: 

522 """Delete a job. 

523 

524 Examples: 

525 gco jobs delete my-job --region us-east-1 

526 gco jobs delete old-job -r us-west-2 -n ml-jobs -y 

527 """ 

528 formatter = get_output_formatter(config) 

529 job_manager = get_job_manager(config) 

530 

531 if not yes: 531 ↛ 532line 531 didn't jump to line 532 because the condition on line 531 was never true

532 click.confirm(f"Delete job {job_name} in namespace {namespace} ({region})?", abort=True) 

533 

534 try: 

535 job_manager.delete_job(job_name, namespace, region) 

536 formatter.print_success(f"Job {job_name} deleted") 

537 except Exception as e: 

538 formatter.print_error(f"Failed to delete job: {e}") 

539 sys.exit(1) 

540 

541 

542@jobs.command("events") 

543@click.argument("job_name") 

544@click.option("--namespace", "-n", default="gco-jobs", help="Job namespace") 

545@click.option("--region", "-r", required=True, help="Job region (required)") 

546@pass_config 

547def get_job_events(config: Any, job_name: Any, namespace: Any, region: Any) -> None: 

548 """Get Kubernetes events for a job. 

549 

550 Shows events related to the job and its pods, useful for debugging 

551 scheduling issues, resource problems, or startup failures. 

552 

553 Examples: 

554 gco jobs events my-job --region us-east-1 

555 gco jobs events training-job -n ml-jobs -r us-west-2 

556 """ 

557 formatter = get_output_formatter(config) 

558 job_manager = get_job_manager(config) 

559 

560 try: 

561 result = job_manager.get_job_events(job_name, namespace, region) 

562 

563 if config.output_format == "table": 563 ↛ 579line 563 didn't jump to line 579 because the condition on line 563 was always true

564 events = result.get("events", []) 

565 if not events: 565 ↛ 566line 565 didn't jump to line 566 because the condition on line 565 was never true

566 formatter.print_info("No events found for this job") 

567 return 

568 

569 print(f"\n Events for {job_name} ({result.get('count', 0)} total)") 

570 print(" " + "-" * 70) 

571 for event in events: 

572 event_type = event.get("type") or "Normal" 

573 reason = (event.get("reason") or "")[:20] 

574 message = (event.get("message") or "")[:50] 

575 timestamp = (event.get("lastTimestamp") or event.get("firstTimestamp") or "")[:19] 

576 marker = "⚠" if event_type == "Warning" else "✓" 

577 print(f" {marker} [{timestamp}] {reason:<20} {message}") 

578 else: 

579 formatter.print(result) 

580 

581 except Exception as e: 

582 formatter.print_error(f"Failed to get job events: {e}") 

583 sys.exit(1) 

584 

585 

586@jobs.command("pods") 

587@click.argument("job_name") 

588@click.option("--namespace", "-n", default="gco-jobs", help="Job namespace") 

589@click.option("--region", "-r", required=True, help="Job region (required)") 

590@pass_config 

591def get_job_pods(config: Any, job_name: Any, namespace: Any, region: Any) -> None: 

592 """Get pod details for a job. 

593 

594 Shows all pods created by the job with their status, node placement, 

595 and container information. 

596 

597 Examples: 

598 gco jobs pods my-job -r us-east-1 

599 gco jobs pods training-job -n ml-jobs -r us-west-2 

600 """ 

601 formatter = get_output_formatter(config) 

602 job_manager = get_job_manager(config) 

603 

604 try: 

605 result = job_manager.get_job_pods(job_name, namespace, region) 

606 

607 if config.output_format == "table": 607 ↛ 629line 607 didn't jump to line 629 because the condition on line 607 was always true

608 pods = result.get("pods", []) 

609 if not pods: 609 ↛ 610line 609 didn't jump to line 610 because the condition on line 609 was never true

610 formatter.print_info("No pods found for this job") 

611 return 

612 

613 print(f"\n Pods for {job_name} ({result.get('count', 0)} total)") 

614 print(" " + "-" * 80) 

615 print( 

616 " NAME NODE STATUS RESTARTS" 

617 ) 

618 print(" " + "-" * 80) 

619 for pod in pods: 

620 name = (pod.get("metadata", {}).get("name") or "")[:40] 

621 node = (pod.get("spec", {}).get("nodeName") or "")[:22] 

622 phase = (pod.get("status", {}).get("phase") or "Unknown")[:10] 

623 restarts = sum( 

624 c.get("restartCount", 0) 

625 for c in (pod.get("status", {}).get("containerStatuses") or []) 

626 ) 

627 print(f" {name:<40} {node:<23} {phase:<10} {restarts}") 

628 else: 

629 formatter.print(result) 

630 

631 except Exception as e: 

632 formatter.print_error(f"Failed to get job pods: {e}") 

633 sys.exit(1) 

634 

635 

636@jobs.command("pod-logs") 

637@click.argument("job_name") 

638@click.argument("pod_name") 

639@click.option("--namespace", "-n", default="gco-jobs", help="Job namespace") 

640@click.option("--region", "-r", required=True, help="Job region (required)") 

641@click.option("--tail", "-t", default=100, help="Number of lines to show") 

642@click.option("--container", "-c", help="Container name (for multi-container pods)") 

643@pass_config 

644def get_pod_logs_cmd( 

645 config: Any, 

646 job_name: Any, 

647 pod_name: Any, 

648 namespace: Any, 

649 region: Any, 

650 tail: Any, 

651 container: Any, 

652) -> None: 

653 """Get logs from a specific pod of a job. 

654 

655 Use 'gco jobs pods' first to list available pods, then use this 

656 command to get logs from a specific pod. 

657 

658 Examples: 

659 gco jobs pod-logs my-job my-job-abc123 -r us-east-1 

660 gco jobs pod-logs training-job training-job-xyz789 -r us-west-2 --tail 500 

661 gco jobs pod-logs multi-job multi-job-pod1 -r us-east-1 --container sidecar 

662 """ 

663 formatter = get_output_formatter(config) 

664 job_manager = get_job_manager(config) 

665 

666 try: 

667 result = job_manager.get_pod_logs( 

668 job_name=job_name, 

669 pod_name=pod_name, 

670 namespace=namespace, 

671 region=region, 

672 tail_lines=tail, 

673 container=container, 

674 ) 

675 

676 # Print logs directly 

677 logs = result.get("logs", "") 

678 if logs: 

679 print(logs) 

680 else: 

681 formatter.print_info("No logs available") 

682 

683 except Exception as e: 

684 formatter.print_error(f"Failed to get pod logs: {e}") 

685 sys.exit(1) 

686 

687 

688@jobs.command("metrics") 

689@click.argument("job_name") 

690@click.option("--namespace", "-n", default="gco-jobs", help="Job namespace") 

691@click.option("--region", "-r", required=True, help="Job region (required)") 

692@pass_config 

693def get_job_metrics(config: Any, job_name: Any, namespace: Any, region: Any) -> None: 

694 """Get resource usage metrics for a job. 

695 

696 Shows CPU and memory usage for all pods in the job. Requires 

697 metrics-server to be installed in the cluster. 

698 

699 Examples: 

700 gco jobs metrics my-job --region us-east-1 

701 gco jobs metrics training-job -n ml-jobs -r us-west-2 

702 """ 

703 formatter = get_output_formatter(config) 

704 job_manager = get_job_manager(config) 

705 

706 try: 

707 result = job_manager.get_job_metrics(job_name, namespace, region) 

708 

709 if config.output_format == "table": 709 ↛ 728line 709 didn't jump to line 728 because the condition on line 709 was always true

710 summary = result.get("summary", {}) 

711 pods = result.get("pods", []) 

712 

713 print(f"\n Resource Metrics for {job_name}") 

714 print(" " + "-" * 50) 

715 print(f" Total CPU: {summary.get('total_cpu_millicores', 0)}m") 

716 print(f" Total Memory: {summary.get('total_memory_mib', 0):.1f} MiB") 

717 print(f" Pod Count: {summary.get('pod_count', 0)}") 

718 

719 if pods: 

720 print("\n POD CPU(m) MEMORY(MiB)") 

721 print(" " + "-" * 65) 

722 for pod in pods: 

723 pod_name = pod.get("pod_name", "")[:40] 

724 cpu = sum(c.get("cpu_millicores", 0) for c in pod.get("containers", [])) 

725 mem = sum(c.get("memory_mib", 0) for c in pod.get("containers", [])) 

726 print(f" {pod_name:<40} {cpu:>6} {mem:>10.1f}") 

727 else: 

728 formatter.print(result) 

729 

730 except Exception as e: 

731 formatter.print_error(f"Failed to get job metrics: {e}") 

732 sys.exit(1) 

733 

734 

735@jobs.command("retry") 

736@click.argument("job_name") 

737@click.option("--namespace", "-n", default="gco-jobs", help="Job namespace") 

738@click.option("--region", "-r", required=True, help="Job region (required)") 

739@click.option("--yes", "-y", is_flag=True, help="Skip confirmation") 

740@pass_config 

741def retry_job(config: Any, job_name: Any, namespace: Any, region: Any, yes: Any) -> None: 

742 """Retry a failed job. 

743 

744 Creates a new job from the failed job's spec with a new name. 

745 The original job is preserved for debugging. 

746 

747 Examples: 

748 gco jobs retry failed-job --region us-east-1 

749 gco jobs retry training-job -n ml-jobs -r us-west-2 -y 

750 """ 

751 formatter = get_output_formatter(config) 

752 job_manager = get_job_manager(config) 

753 

754 if not yes: 754 ↛ 755line 754 didn't jump to line 755 because the condition on line 754 was never true

755 click.confirm(f"Retry job {job_name} in namespace {namespace} ({region})?", abort=True) 

756 

757 try: 

758 result = job_manager.retry_job(job_name, namespace, region) 

759 

760 if result.get("success"): 

761 formatter.print_success(f"Job retry created: {result.get('new_job')}") 

762 else: 

763 formatter.print_error(f"Failed to retry job: {result.get('message')}")