diff --git a/pulumi/AWSMegatests/.gitignore b/pulumi/AWSMegatests/.gitignore index 3e3e8e26..313ae838 100644 --- a/pulumi/AWSMegatests/.gitignore +++ b/pulumi/AWSMegatests/.gitignore @@ -1,8 +1,8 @@ -*.pyc -venv/ - -# Auto-generated Seqera SDK -sdks/ +# OpenTofu +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars # Team data files - contain email addresses (private data) scripts/maintainers_data.json @@ -14,7 +14,7 @@ scripts/best_participants_result.json # Debug and test scripts - not needed in git scripts/debug_*.py -scripts/discover_*.py +scripts/discover_*.py scripts/test_*.py scripts/get_all_*.py scripts/fetch_all_*.py diff --git a/pulumi/AWSMegatests/.terraform.lock.hcl b/pulumi/AWSMegatests/.terraform.lock.hcl new file mode 100644 index 00000000..b49010ce --- /dev/null +++ b/pulumi/AWSMegatests/.terraform.lock.hcl @@ -0,0 +1,107 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/1password/onepassword" { + version = "2.2.1" + constraints = "~> 2.0" + hashes = [ + "h1:aH5pZUimlQdiGnDLHRC49W4xnkx52wfd8XRLt69+764=", + "zh:025709a6b5f1b3685d277f2c48f7cb8b53d14b3699c1123d7e9a2135c099c533", + "zh:037fc89d150063a8aacdcab08ba26038b489fe2468d509b842d298ea59096ca6", + "zh:233777182b25faf1658e8ce171b684460983bb41cff79fb243662f3f9dc5ca6c", + "zh:2fb5ca2fc8c37b1d1c54da646ed13bf40897941fe92eece784fba496f677b533", + "zh:4b25b5ce1f694ec265e65234fc85d6bdf3810297ffeaced54ad46a1ba28142de", + "zh:5509d1e4fb7b45c63124ec66fd1f9d6757daa8bf1f7bdd724d5adb2965b61436", + "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", + "zh:a23ba946c629ec912b2fcbf606a2eb8853626ec0e0bee749f2d39146a872c082", + "zh:a3d3024485426237d7b4a4350b12dda4d29d88f3942246a9370be35ec2a51e9e", + "zh:a6ef65544ab8fc26d468b38636407a3d2d902e35c51b648729bf97c31d1937f9", + "zh:afbe9480a0da0ad8dc514b277f1e4be36b8931f045021d05c21665ef1ac0b7c8", + "zh:b2e96e69fa9ff7e179dccdef5b785cd020eb46bb2b3d1d507d009d71be6b0c26", + "zh:ceefaede9e8a3104463523ba267e3e985b27a706f7628a9ddd37330c2ca59d4d", + "zh:ea77786bd6809ff4f8043b84a0212fec4de18b7d51bc420417ba10999ca99887", + "zh:f7d8160c3669c8ab76a2da14ea740d91a08ca23d1fb657669e52a840b2b113d9", + ] +} + +provider "registry.opentofu.org/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.0" + hashes = [ + "h1:BrNG7eFOdRrRRbHdvrTjMJ8X8Oh/tiegURiKf7J2db8=", + "zh:1a41f3ee26720fee7a9a0a361890632a1701b5dc1cf5355dc651ddbe115682ff", + "zh:30457f36690c19307921885cc5e72b9dbeba369445815903acd5c39ac0e41e7a", + "zh:42c22674d5f23f6309eaf3ac3a4f1f8b66b566c1efe1dcb0dd2fb30c17ce1f78", + "zh:4cc271c795ff8ce6479ec2d11a8ba65a0a9ed6331def6693f4b9dccb6e662838", + "zh:60932aa376bb8c87cd1971240063d9d38ba6a55502c867fdbb9f5361dc93d003", + "zh:864e42784bde77b18393ebfcc0104cea9123da5f4392e8a059789e296952eefa", + "zh:9750423138bb01ecaa5cec1a6691664f7783d301fb1628d3b64a231b6b564e0e", + "zh:e5d30c4dec271ef9d6fe09f48237ec6cfea1036848f835b4e47f274b48bda5a7", + "zh:e62bd314ae97b43d782e0841b13e68a3f8ec85cc762004f973ce5ce7b6cdbfd0", + "zh:ea851a3c072528a4445ac6236ba2ce58ffc99ec466019b0bd0e4adde63a248e4", + ] +} + +provider "registry.opentofu.org/hashicorp/null" { + version = "3.2.4" + constraints = "~> 3.0" + hashes = [ + "h1:i+WKhUHL2REY5EGmiHjfUljJB8UKZ9QdhdM5uTeUhC4=", + "zh:1769783386610bed8bb1e861a119fe25058be41895e3996d9216dd6bb8a7aee3", + "zh:32c62a9387ad0b861b5262b41c5e9ed6e940eda729c2a0e58100e6629af27ddb", + "zh:339bf8c2f9733fce068eb6d5612701144c752425cebeafab36563a16be460fb2", + "zh:36731f23343aee12a7e078067a98644c0126714c4fe9ac930eecb0f2361788c4", + "zh:3d106c7e32a929e2843f732625a582e562ff09120021e510a51a6f5d01175b8d", + "zh:74bcb3567708171ad83b234b92c9d63ab441ef882b770b0210c2b14fdbe3b1b6", + "zh:90b55bdbffa35df9204282251059e62c178b0ac7035958b93a647839643c0072", + "zh:ae24c0e5adc692b8f94cb23a000f91a316070fdc19418578dcf2134ff57cf447", + "zh:b5c10d4ad860c4c21273203d1de6d2f0286845edf1c64319fa2362df526b5f58", + "zh:e05bbd88e82e1d6234988c85db62fd66f11502645838fff594a2ec25352ecd80", + ] +} + +provider "registry.opentofu.org/integrations/github" { + version = "6.11.1" + constraints = "~> 6.0" + hashes = [ + "h1:nanzeesukYMHAFrSaq7rnWx7iRDHMpme5KzQI3m/ZZo=", + "zh:0a5262b033a30d8a77ebf844dc3afd7e726d5f53ac1c9d4072cf9157820d1f73", + "zh:437236181326f92d1a7c56985b2ac3223efd73f75c528323b90f4b7d1b781090", + "zh:49a12c14d1d3a143a124ba81f15fbf18714af90752c993698c76e84fa85da004", + "zh:61eaf17b559a26ca14deb597375a6678d054d739e8b81c586ef1d0391c307916", + "zh:7f3f1e2c36f4787ca9a5aeb5317b8c3f6cc652368d1f8f00fb80f404109d4db1", + "zh:85a232f2e96e5adafa2676f38a96b8cc074e96f715caf6ee1d169431174897d2", + "zh:979d005af2a9003d887413195948c899e9f5aba4a79cce1eed40f3ba50301af1", + "zh:b8c8cd3254504d2184d2b2233ad41b5fdfda91a36fc864926cbc5c7eee1bfea3", + "zh:d00959e62930fb75d2b97c1d66ab0143120541d5a1b3f26d3551f24cb0361f83", + "zh:d0b544eed171c7563387fe87f0af3d238bb3804798159b4d0453c97927237daf", + "zh:ecfa19b1219aa55b1ece98d8cff5b1494dc0387329c8ae0d8f762ec3871fb75d", + "zh:f2c99825f38c92ac599ad36b9d093ea0c0d790fd0c02e861789e14735a605f86", + "zh:f33b5abe14ad5fb9978da5dbd3bc6989f69766150d4b30ed283a2c281871eda3", + "zh:f6c2fe9dd958c554170dc0c35ca41b60fcc6253304cde0b9941c5c872b18ac54", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} + +provider "registry.terraform.io/seqeralabs/seqera" { + version = "0.30.5" + constraints = "~> 0.25" + hashes = [ + "h1:dq3IkBKWWDwN37/Ibqk824BRyEVj88PyERYkjsBZgr4=", + "zh:2548055fb2cbd9c658ea8cff5575e15994e82962216a7c6d56cd3a69baed0f48", + "zh:275c85b2e1e88f42d40f10d202e6b152d4940fa9ca454ab03fc8b2642163ade1", + "zh:33962053ab0bc29a5c7ff061dd2e61b9f28e2f31832682869689006ec760b82e", + "zh:5b515c8124d437b2bd7b72413cd82e159d33e9d50274bfc79720d272ed9611e0", + "zh:617d59923f76b61f0a39ed04ac0b86427f5c761e2cdaee1f41f54dafc9aa4639", + "zh:6f455ee7efc4f6ec849ca1a76e69a2033e45c8904bc92f8fe4c9c6237a146e9e", + "zh:76d1cdc5b7078086f097f77082daeb5fdfcaadf9aa24252a7ef37b3e3075e5ed", + "zh:7aa6870292b3d8951fbd258ccf9085ca1dee408db9159d922b857e06d9560e4c", + "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", + "zh:8ce99a4baaa06c2867da38b4c242c8859f07d13c268b5283d95c9e2e0b856c41", + "zh:97ed7d5e4ed0602bb369e4533570dd0735c540011bdf1269ad3b2809a0277dbd", + "zh:989b477286d2d83aacbd6d9d8e076350ec290a12115b17c5419315554a426331", + "zh:a0967e61c38fff77bffc9cb357c635b711447b00ef04f001c6cea1ed3ccaa7bb", + "zh:ea51792ce96d52b8fe110e69332ccb7bf44ddb115c70860f7a96f3696918a2ca", + "zh:f863d9f77912ff758ae63892fad39e07ead44676ef5e826b689e0bf80852b27a", + ] +} diff --git a/pulumi/AWSMegatests/CLAUDE.md b/pulumi/AWSMegatests/CLAUDE.md index 73644db3..177b59b8 100644 --- a/pulumi/AWSMegatests/CLAUDE.md +++ b/pulumi/AWSMegatests/CLAUDE.md @@ -1,375 +1,52 @@ -# AWS Megatests - Claude Code Context - -This file provides Claude Code with context for the AWS Megatests Pulumi infrastructure project. +# AWS Megatests - OpenTofu Infrastructure ## Project Overview -This is a **Pulumi Infrastructure as Code (IaC) project** that manages AWS infrastructure for nf-core megatests. The project integrates multiple technologies: - -- **Pulumi**: Python-based infrastructure as code -- **Seqera Terraform Provider**: Native Terraform provider integration for Seqera Platform -- **Pulumi ESC**: Environment management and secret storage -- **GitHub**: Automated variable and secret deployment for CI/CD workflows - -## Architecture - -### Code Organization (Updated Structure) - -The codebase has been restructured for better maintainability and organization: - -**Key Improvements:** -- **Modular Structure**: Code is organized into logical modules (`config`, `providers`, `infrastructure`, `integrations`, `utils`) -- **Separation of Concerns**: Each module has a specific responsibility -- **Type Safety**: Comprehensive type hints and custom exception classes -- **Centralized Constants**: All configuration values and constants in one place -- **Better Error Handling**: Custom exception hierarchies with clear error messages -- **Improved Testing**: More modular code structure makes testing easier - -**Module Responsibilities:** -- `config/`: Configuration management with typed classes and validation -- `providers/`: Individual provider configurations (AWS, GitHub, Seqera) -- `infrastructure/`: Core infrastructure components (S3, credentials, compute environments) -- `integrations/`: Third-party service integrations (GitHub) -- `utils/`: Shared utilities, constants, and logging functions - -### Core Components - -1. **S3 Bucket Management** - - Imports existing `nf-core-awsmegatests` bucket - - Manages bucket configuration through Pulumi - -2. **Seqera Compute Environments** - - Three AWS Batch environments: CPU, ARM, GPU - - Deployed via native Seqera Terraform provider - - All environments have fusion snapshots enabled - -3. **GitHub Integration** - - Automatically pushes compute environment IDs to GitHub organization variables - - Manages Seqera Platform access tokens for CI/CD - -4. **Pulumi ESC Integration** - - Secure credential management using Pulumi ESC environments - - AWS and GitHub credentials from ESC - -### File Structure - -``` -AWSMegatests/ -├── __main__.py # Main Pulumi program entry point -├── src/ # Organized source code modules -│ ├── __init__.py -│ ├── config/ # Configuration management -│ │ ├── __init__.py -│ │ └── settings.py # ESC configuration management (typed) -│ ├── providers/ # Provider configurations -│ │ ├── __init__.py -│ │ ├── aws.py # AWS provider configuration -│ │ ├── github.py # GitHub provider configuration -│ │ └── seqera.py # Seqera provider configuration -│ ├── infrastructure/ # Infrastructure components -│ │ ├── __init__.py -│ │ ├── s3.py # S3 bucket management -│ │ ├── credentials.py # AWS IAM credentials for TowerForge -│ │ └── compute_environments.py # Seqera compute environment deployment -│ ├── integrations/ # Third-party integrations -│ │ ├── __init__.py -│ │ └── github.py # GitHub variables and secrets management -│ └── utils/ # Utility functions and constants -│ ├── __init__.py -│ ├── constants.py # Centralized constants and configuration -│ └── logging.py # Logging utilities -├── pyproject.toml # Python project configuration -├── uv.lock # UV lock file -├── Pulumi.yaml # Pulumi project configuration -├── README.md # Project documentation -├── CLAUDE.md # This file - Claude context -├── sdks/seqera/ # Auto-generated Seqera SDK -└── seqerakit/ # Configuration files (read-only) - ├── *.yml # Seqerakit YAML configurations (for reference) - └── *.json # Compute environment JSON specs (used by Terraform provider) -``` +OpenTofu IaC project managing nf-core's AWS megatest infrastructure. Integrates AWS, Seqera Platform, GitHub, and 1Password. ## Common Commands -### Prerequisites - -```bash -# Install UV and dependencies -curl -LsSf https://astral.sh/uv/install.sh | sh -uv sync - -# Login to Pulumi -pulumi login - -# Ensure Pulumi ESC environment is properly configured -pulumi env ls -``` - -### Development Workflow - -```bash -# Preview changes -uv run pulumi preview - -# Deploy infrastructure -uv run pulumi up - -# View outputs -uv run pulumi stack output - -# Refresh state to match actual infrastructure -uv run pulumi refresh -``` - -### State Management - -```bash -# List stack resources -uv run pulumi stack --show-urns - -# Unprotect resources -uv run pulumi state unprotect - -# Remove resources from state (without deleting from cloud) -uv run pulumi state delete - -# Protect critical resources -uv run pulumi state protect -``` - -## Key Technologies - -### Pulumi Providers Used - -1. **AWS Provider** (`pulumi-aws`) - - Manages S3 bucket and IAM resources - - Uses ESC-provided AWS credentials - -2. **Seqera Terraform Provider** (`pulumi-seqera`) - - Native Terraform provider for Seqera Platform - - Manages compute environments directly - -3. **GitHub Provider** (`pulumi-github`) - - Manages organization variables and secrets - - Uses ESC-provided GitHub token - -### Environment Configuration - -The project uses **Pulumi ESC** for all configuration and secrets: - -- **AWS credentials**: Provided by ESC environment -- **Seqera Platform tokens**: Stored in ESC -- **GitHub tokens**: Managed through ESC -- **Workspace configuration**: Defined in ESC environment - -## Development Guidelines - -### When Working with This Project - -1. **Use standard Pulumi commands** - ESC handles credential loading automatically -2. **Never commit secrets** - all credentials managed through Pulumi ESC -3. **Test with preview** before deploying: `uv run pulumi preview` -4. **Check outputs** after deployment: `uv run pulumi stack output` - -### Common Issues and Solutions - -#### Infrastructure and Configuration - -**"No valid credential sources found"** -- Solution: Ensure Pulumi ESC environment is properly configured -- Check: `pulumi env ls` and `pulumi env open ` - -**"Provider configuration error"** -- Solution: Verify ESC environment contains required credentials -- Check: ESC environment has `aws`, `seqera`, and `github` provider configurations - -**Protected resource errors** -- Solution: Unprotect then delete from state: `pulumi state unprotect ` then `pulumi state delete ` -- Note: This removes from Pulumi state without affecting actual cloud resources - -#### API-Specific Issues and Troubleshooting - -**Seqera API 403 Forbidden Errors** -- **Cause**: TOWER_ACCESS_TOKEN lacks required permissions -- **Solution**: - 1. Verify token has `WORKSPACE_ADMIN` or `COMPUTE_ENV_ADMIN` scope - 2. Check token is valid and not expired - 3. Ensure workspace ID is correct -- **ESC Environment**: Add `TOWER_ACCESS_TOKEN` with proper permissions -- **Commands to verify**: - ```bash - curl -H "Authorization: Bearer $TOWER_ACCESS_TOKEN" \ - https://api.cloud.seqera.io/user-info - ``` - -**GitHub API 409 Already Exists (Variables)** -- **Status**: Usually harmless - indicates variables already exist -- **Cause**: GitHub organization variables from previous deployments -- **Behavior**: Pulumi will update values automatically with `delete_before_replace=True` -- **No action required**: This is expected behavior when re-deploying -- **External Variables**: Some variables like `AWS_S3_BUCKET` are managed outside Pulumi to avoid conflicts - -**GitHub API Permission Issues** -- **Cause**: GITHUB_TOKEN lacks organization-level permissions -- **Solution**: Ensure token has `admin:org` scope for organization variables -- **ESC Environment**: Add `GITHUB_TOKEN` with proper permissions - -**Missing Variable Values** -- **Cause**: Environment variables not properly set in ESC -- **Solution**: Add missing variables to ESC environment: - - `TOWER_WORKSPACE_ID`: Seqera workspace ID - - `TOWER_ACCESS_TOKEN`: Seqera API token with admin permissions - - `GITHUB_TOKEN`: GitHub token with org admin permissions - -#### Debugging Commands - -**Check ESC Environment Variables**: -```bash -# List environments -pulumi env ls - -# Open environment in editor -pulumi env open - -# Get specific environment values -pulumi env get -``` - -**Test API Connectivity**: ```bash -# Test Seqera API -curl -H "Authorization: Bearer $TOWER_ACCESS_TOKEN" \ - https://api.cloud.seqera.io/user-info - -# Test GitHub API -curl -H "Authorization: token $GITHUB_TOKEN" \ - https://api.github.com/orgs/nf-core -``` - -**Pulumi Diagnostics**: -```bash -# Detailed logging during deployment -uv run pulumi up --verbose 2 --continue-on-error --diff - -# Show current stack outputs -uv run pulumi stack output - -# Refresh state to match actual resources -uv run pulumi refresh -``` - -#### Error Recovery Strategies - -1. **Seqera 403 Errors**: Update token permissions in ESC environment -2. **GitHub 409 Errors**: Ignore - variables will be updated automatically -3. **Configuration Errors**: Verify all required environment variables in ESC -4. **Network Issues**: Check connectivity to api.cloud.seqera.io and api.github.com - -#### Expected Warnings vs Critical Errors - -**Expected/Harmless**: -- GitHub 409 "Already exists" for organization variables -- Seqera compute environment updates (existing resources) -- S3 bucket "already exists" messages (imported resource) - -**Critical Issues Requiring Action**: -- Seqera 403 "Forbidden" - fix token permissions -- Missing TOWER_ACCESS_TOKEN or GITHUB_TOKEN -- Invalid workspace ID -- Network connectivity failures - -The infrastructure includes comprehensive error handling with detailed diagnostic messages to help identify and resolve these common API issues. - -### Code Patterns - -**Seqera Terraform Provider Usage:** -```python -# Create Seqera provider -provider = seqera.Provider( - "seqera-provider", - bearer_auth=config["tower_access_token"], - server_url="https://api.cloud.seqera.io", -) - -# Create compute environment -compute_env = seqera.ComputeEnv( - "environment-name", - compute_env=compute_env_args, - workspace_id=workspace_id, - opts=pulumi.ResourceOptions(provider=provider) -) +tofu init # Initialize providers +tofu plan # Preview changes +tofu apply # Deploy +tofu fmt -check # Check formatting +tofu output # View outputs ``` -**GitHub Variable Deployment:** -```python -github_variable = github.ActionsOrganizationVariable( - "variable-name", - variable_name="VARIABLE_NAME", - value=variable_value, - visibility="all", - opts=pulumi.ResourceOptions(provider=github_provider), -) -``` - -## Seqera Integration - -The project uses the native Seqera Terraform provider for compute environment management: - -- **Configuration**: Reads existing seqerakit JSON files for compute environment specifications -- **Deployment**: Uses native Terraform provider resources instead of CLI commands -- **Environment IDs**: Direct access to compute environment IDs as resource outputs -- **Features**: All environments have fusion snapshots enabled - -The `seqerakit/` directory contains configuration files that are read by the Terraform provider integration but are not actively used for deployment. - -## ESC Environment Setup - -The project requires the following environment variables in Pulumi ESC: - -### Required Variables -```yaml -# ESC Environment Configuration Example -values: - # Seqera Platform Configuration - TOWER_ACCESS_TOKEN: "your-seqera-api-token-with-admin-permissions" - TOWER_WORKSPACE_ID: "59994744926013" # Add to avoid fallback warning - - # GitHub Integration - GITHUB_TOKEN: "your-github-token-with-org-admin-scope" - - # AWS credentials are provided by ESC OIDC integration -``` - -### Token Permission Requirements - -**TOWER_ACCESS_TOKEN**: -- Scope: `WORKSPACE_ADMIN` or `COMPUTE_ENV_ADMIN` -- Used for: Creating and managing compute environments -- Test command: `curl -H "Authorization: Bearer $TOWER_ACCESS_TOKEN" https://api.cloud.seqera.io/user-info` +## Architecture -**GITHUB_TOKEN**: -- Scope: `admin:org` (organization administration) -- Used for: Creating organization variables for CI/CD workflows -- Test command: `curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/orgs/nf-core` +### File Layout -## Outputs and Monitoring +- `versions.tf` / `providers.tf` / `variables.tf` — scaffold +- `onepassword.tf` — 1Password data sources for secrets +- `locals.tf` — Nextflow config merging + shared forge defaults +- `s3.tf` — S3 bucket (imported), lifecycle rules, CORS +- `iam.tf` — TowerForge IAM user + 3 policies + access keys +- `seqera_credentials.tf` — AWS creds uploaded to Seqera Platform +- `github_credential.tf` — GitHub fine-grained token in Seqera +- `compute_environments.tf` — 3 compute envs (CPU, GPU, ARM) +- `github.tf` — 5 GitHub org variables +- `participants.tf` — workspace participant sync via scripts +- `outputs.tf` — all stack outputs -The stack provides several outputs for monitoring and integration: +### Key Values -- `megatests_bucket`: S3 bucket information -- `compute_env_ids`: Seqera compute environment IDs -- `workspace_id`: Seqera workspace ID -- `terraform_resources`: Terraform provider resource information -- `github_resources`: GitHub integration status +- **Region**: eu-west-1 +- **S3 Bucket**: nf-core-awsmegatests +- **Workspace ID**: 59994744926013 +- **Org ID**: 252464779077610 +- **Seqera API**: https://api.cloud.seqera.io -## Security Considerations +### Secrets (from 1Password) -- **Pulumi ESC Integration**: All secrets managed through Pulumi ESC environments -- **Provider Security**: Native Terraform provider with proper authentication -- **GitHub Permissions**: Organization-level variables with appropriate access control -- **AWS IAM**: Compute environments use dedicated TowerForge service roles with least privilege +All secrets come from the `Dev` vault via `onepassword` provider: +- `Seqera Platform` — TOWER_ACCESS_TOKEN +- `AWS megatests` — IAM access keys (for reference, not used directly) +- `GitHub nf-core PA Token Pulumi` — GitHub token +- `GitHub nf-core Org Token` — Platform GitHub org token -## Related Documentation +### Kept As-Is -- **Main README**: `README.md` - User-facing project documentation -- **Seqerakit Configs**: `seqerakit/README.md` - Configuration reference (read-only) \ No newline at end of file +- `configs/` — Nextflow config files (read by TF via `file()`) +- `scripts/` — team management Python scripts (run via `null_resource`) diff --git a/pulumi/AWSMegatests/Pulumi.prod.yaml b/pulumi/AWSMegatests/Pulumi.prod.yaml deleted file mode 100644 index 8aabce54..00000000 --- a/pulumi/AWSMegatests/Pulumi.prod.yaml +++ /dev/null @@ -1,2 +0,0 @@ -environment: - - AWSMegatests/prod diff --git a/pulumi/AWSMegatests/Pulumi.yaml b/pulumi/AWSMegatests/Pulumi.yaml deleted file mode 100644 index 79ebad4a..00000000 --- a/pulumi/AWSMegatests/Pulumi.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: AWSMegatests -description: AWS Megatests Production Infrastructure -runtime: - name: python - options: - toolchain: uv - virtualenv: .venv -config: - # GitHub Provider Configuration - github:owner: - value: nf-core - # Note: AWS and other tokens are provided via ESC environment - # AWS credentials should come from ESC OIDC integration -packages: - seqera: - source: terraform-provider - version: 0.25.2 - parameters: - - registry.terraform.io/seqeralabs/seqera diff --git a/pulumi/AWSMegatests/README.md b/pulumi/AWSMegatests/README.md index 39089dae..50245d51 100644 --- a/pulumi/AWSMegatests/README.md +++ b/pulumi/AWSMegatests/README.md @@ -1,295 +1,85 @@ -# AWS Megatests - Pulumi Infrastructure +# AWS Megatests - OpenTofu Infrastructure -This repository manages AWS infrastructure for nf-core megatests using Pulumi Infrastructure as Code, integrating seqerakit compute environment deployment with automated GitHub secrets management. +Manages AWS infrastructure for nf-core megatests using OpenTofu, integrating Seqera Platform compute environments with automated GitHub variable management. ## Overview -The AWS Megatests project provides: - -- **S3 Bucket**: Imported and managed `nf-core-awsmegatests` bucket for workflow data -- **Compute Environments**: Three AWS Batch environments (CPU, ARM, GPU) deployed via seqerakit -- **GitHub Secrets**: Automated deployment of compute environment IDs and credentials to GitHub -- **Workspace Participants**: Automated management of nf-core team members in Seqera Platform workspace -- **1Password Integration**: Secure credential management using the Pulumi 1Password provider +- **S3 Bucket**: Imported `nf-core-awsmegatests` bucket with lifecycle rules +- **Compute Environments**: Three AWS Batch environments (CPU, ARM, GPU) via Seqera Terraform provider +- **GitHub Variables**: Automated deployment of compute environment IDs to GitHub org variables +- **Workspace Participants**: Automated nf-core team member management in Seqera Platform +- **1Password Integration**: Secrets from 1Password via the Terraform provider ## Quick Start -### Prerequisites - ```bash -# Install dependencies -uv sync -brew install direnv - -# Allow environment variables -direnv allow - -# Login to Pulumi -pulumi login -``` - -### Workspace Participants +# Install OpenTofu +brew install opentofu -The infrastructure automatically manages nf-core team members in the Seqera Platform workspace: - -- **Automatic team data generation**: Fetches GitHub teams and maps emails during deployment -- **Role precedence**: Core team members get OWNER role, maintainers get MAINTAIN role -- **Individual tracking**: Creates separate Pulumi Command for each of 35+ team members -- **Privacy protection**: Email data generated at runtime, never committed to git - -**Privacy Note**: Team member email addresses are generated locally during deployment and excluded from git to protect privacy. Only public GitHub emails and workspace admin-accessible Seqera participant data are used for email mapping. - -### Deployment - -```bash -# Preview infrastructure changes -direnv exec . uv run pulumi preview +# Initialize providers +tofu init -# Deploy infrastructure -direnv exec . uv run pulumi up +# Preview changes +tofu plan -# View current stack outputs -direnv exec . uv run pulumi stack output +# Deploy +tofu apply ``` ## Architecture -### Infrastructure Components +### Compute Environments -#### S3 Storage +| Environment | Instance Types | Max CPUs | Features | +|---|---|---|---| +| CPU | c6id, m6id, r6id | 1000 | Fusion v2, Wave, NVMe, Snapshots | +| GPU | g4dn, g5, c6id, m6id, r6id | 500 | GPU, Fusion v2, Wave, NVMe, Snapshots | +| ARM | m6gd, r6gd, c6gd | 500 | ARM64, Fusion v2, Wave, NVMe, Snapshots | -- **Bucket**: `nf-core-awsmegatests` (imported from existing AWS resources) -- **Region**: eu-west-1 -- **Purpose**: Primary work directory and data storage for workflows +### GitHub Organization Variables -#### Compute Environments +- `TOWER_COMPUTE_ENV_CPU` / `TOWER_COMPUTE_ENV_GPU` / `TOWER_COMPUTE_ENV_ARM` +- `TOWER_WORKSPACE_ID` +- `AWS_S3_BUCKET` -Three AWS Batch compute environments managed through seqerakit: +### Secrets Management -1. **CPU Environment** (`aws_ireland_fusionv2_nvme_cpu`) - - **Instance Types**: c6id, m6id, r6id (Intel x86_64) - - **Max CPUs**: 500 - - **Features**: Fusion v2, Wave, NVMe storage, snapshots enabled +All secrets come from 1Password via the `onepassword` provider: +- Seqera Platform token +- GitHub tokens +- AWS credentials (for IAM user access keys) -2. **ARM Environment** (`aws_ireland_fusionv2_nvme_cpu_ARM_snapshots`) - - **Instance Types**: m6gd, r6gd, c6gd (ARM Graviton) - - **Max CPUs**: 1000 - - **Features**: Fusion v2, Wave, NVMe storage, snapshots enabled - -3. **GPU Environment** (`aws_ireland_fusionv2_nvme_gpu_snapshots`) - - **Instance Types**: g4dn, g5 (GPU) + c6id, m6id, r6id (CPU) - - **Max CPUs**: 500 - - **Features**: GPU enabled, Fusion v2, Wave, NVMe storage, snapshots enabled - -#### GitHub Secrets - -Automatically deployed GitHub organization secrets: - -- `TOWER_ACCESS_TOKEN`: Seqera Platform access token -- `TOWER_WORKSPACE_ID`: Workspace ID for AWSmegatests -- `TOWER_COMPUTE_ENV_CPU`: CPU compute environment ID -- `TOWER_COMPUTE_ENV_ARM`: ARM compute environment ID -- `TOWER_COMPUTE_ENV_GPU`: GPU compute environment ID - -### Technology Stack - -- **Infrastructure**: Pulumi with Python -- **Cloud Provider**: AWS (eu-west-1) -- **Compute**: AWS Batch with SPOT instances -- **Storage**: S3 + NVMe local storage -- **Container Platform**: Docker with Wave optimization -- **Secrets Management**: 1Password with Pulumi provider -- **CI/CD**: GitHub Actions with automated secret deployment - -## Configuration - -### Environment Variables (.envrc) - -```bash -# Organization settings -export ORGANIZATION_NAME="nf-core" -export WORKSPACE_NAME="AWSmegatests" -export AWS_REGION="eu-west-1" +## File Structure -# AWS configuration -export AWS_WORK_DIR="s3://nf-core-awsmegatests" -export AWS_CREDENTIALS_NAME="tower-awstest" -export AWS_COMPUTE_ENV_ALLOWED_BUCKETS="s3://ngi-igenomes,s3://annotation-cache" - -# 1Password secret references -from_op TOWER_ACCESS_TOKEN="op://Dev/zwsrkl26xz3biqwcmw64qizxie/Tower key for megatests" -from_op AWS_ACCESS_KEY_ID="op://Dev/AWS megatests/username" -from_op AWS_SECRET_ACCESS_KEY="op://Dev/AWS megatests/password" -from_op GITHUB_TOKEN="op://Dev/GitHub nf-core PA Token Pulumi/token" -from_op OP_SERVICE_ACCOUNT_TOKEN="op://Employee/doroenisttgrfcmzihhunyizg4/credential" -``` - -### Pulumi Configuration - -```bash -# Set 1Password service account token -direnv exec . uv run pulumi config set --secret pulumi-onepassword:service_account_token "$OP_SERVICE_ACCOUNT_TOKEN" ``` - -## Seqerakit Integration - -The project uses seqerakit for compute environment deployment through Pulumi's command provider: - -```python -# Seqerakit deployment commands -deploy_cpu = command.local.Command("deploy-cpu-environment", - create="seqerakit seqerakit/aws_ireland_fusionv2_nvme_cpu_current.yml") - -deploy_arm = command.local.Command("deploy-arm-environment", - create="seqerakit seqerakit/aws_ireland_fusionv2_nvme_cpu_arm_current.yml") - -deploy_gpu = command.local.Command("deploy-gpu-environment", - create="seqerakit seqerakit/aws_ireland_fusionv2_nvme_gpu_current.yml") +AWSMegatests/ +├── versions.tf # Provider version constraints +├── providers.tf # Provider configurations +├── variables.tf # Input variables +├── onepassword.tf # 1Password data sources +├── locals.tf # Nextflow config merging + shared defaults +├── s3.tf # S3 bucket + lifecycle + CORS +├── iam.tf # IAM user, policies, access keys +├── seqera_credentials.tf # Seqera AWS credential +├── github_credential.tf # Seqera GitHub credential +├── compute_environments.tf # 3 Seqera compute environments +├── github.tf # GitHub org variables +├── participants.tf # Workspace participant management +├── outputs.tf # Stack outputs +├── configs/ # Nextflow config files +└── scripts/ # Team management scripts ``` -### Seqerakit Configurations - -Located in `seqerakit/` directory: - -- **YAML configs**: Reference JSON configurations with metadata -- **JSON configs**: Complete compute environment specifications -- **Features**: All environments have fusion snapshots enabled - -See [seqerakit/README.md](seqerakit/README.md) for detailed configuration information. - -## Outputs - -The Pulumi stack provides these outputs: +## Common Commands ```bash -# View all outputs -direnv exec . uv run pulumi stack output - -# Specific outputs -direnv exec . uv run pulumi stack output megatests_bucket # S3 bucket info -direnv exec . uv run pulumi stack output compute_env_ids # Environment IDs -direnv exec . uv run pulumi stack output workspace_id # Seqera workspace ID -direnv exec . uv run pulumi stack output github_secrets # Secret names +tofu init # Initialize providers +tofu plan # Preview changes +tofu apply # Deploy infrastructure +tofu output # View outputs +tofu fmt -check # Check formatting ``` -## Development Workflow - -### For Contributors - -**📝 Contributing to Infrastructure** - -Contributors **do not need** to run infrastructure locally. The workflow is: - -1. **Make your changes** to seqerakit configurations in `seqerakit/` directory -2. **Create a Pull Request** with your changes -3. **Core team will review** and run `pulumi preview` in Pulumi Cloud -4. **Infrastructure is deployed** automatically via Pulumi Cloud + 1Password integration - -**Requirements for Contributors:** - -- ✅ None! No 1Password, AWS, or Pulumi access needed -- ✅ Just edit the configuration files and make a PR -- ✅ Core team handles all infrastructure operations - -### For Core Team (Infrastructure Access) - -**🔧 Infrastructure Management** - -Core team members with 1Password and Pulumi access: - -1. **Setup credentials**: Ensure 1Password service account token is configured -2. **Preview changes**: Review PR changes in Pulumi Cloud preview -3. **Deploy**: Merge PR triggers automatic deployment via Pulumi Cloud -4. **Monitor**: Check Pulumi Console and AWS Console for deployment status - -**Local Development (Optional):** - -```bash -# If you need to run locally (requires 1Password access) -direnv allow -direnv exec . uv run pulumi preview -direnv exec . uv run pulumi up -``` - -### Debugging Features - -**🔍 Enhanced Debugging** - -The system includes detailed debugging output: - -- **Tower CLI Availability**: Checks if Tower CLI is installed and accessible -- **Authentication Status**: Verifies Tower CLI can authenticate with Seqera Platform -- **Environment Listing**: Shows available compute environments for troubleshooting -- **ID Extraction**: Detailed logging of compute environment ID retrieval process - -### Common Operations (Core Team) - -```bash -# View current state -uv run pulumi stack output - -# Refresh state to match actual infrastructure -uv run pulumi refresh - -# View infrastructure in Pulumi Console -uv run pulumi console - -# Debug failed deployments -uv run pulumi logs - -# Import existing AWS resources (if needed) -uv run pulumi import aws:s3/bucket:Bucket nf-core-awsmegatests nf-core-awsmegatests -``` - -### Troubleshooting - -**"No valid credential sources found"** - -- Run commands with `direnv exec .` to load AWS credentials -- Ensure `.envrc` is allowed: `direnv allow` - -**"static credentials are empty"** - -- Check 1Password service account token: `echo $OP_SERVICE_ACCOUNT_TOKEN` -- Verify 1Password items exist and are accessible - -**Protected resource errors** - -- Unprotect resources: `uv run pulumi state unprotect ` -- Remove from state: `uv run pulumi state delete ` - -## Security - -### Credential Management - -- **1Password Integration**: All secrets stored in 1Password vaults -- **Service Account**: Uses 1Password service account for automated access -- **GitHub Secrets**: Automatically deployed for CI/CD workflows -- **No Hardcoded Secrets**: All credentials loaded from secure sources - -### Access Control - -- **AWS IAM**: Compute environments use dedicated service roles -- **Seqera Platform**: Token-based authentication with workspace isolation -- **GitHub**: Organization-level secrets with appropriate permissions - -## Monitoring and Observability - -- **Pulumi Console**: Infrastructure state and history -- **AWS Console**: Compute environment and job monitoring -- **Seqera Platform**: Workflow execution and resource usage -- **GitHub Actions**: Deployment logs and status - -## Related Projects - -- **nf-core**: Main nf-core project repository -- **nf-core/test-datasets**: Test data and validation workflows -- **Seqera Platform**: Workflow execution and monitoring platform - -## Support - -For issues related to: +## Contributing -- **Infrastructure**: Check Pulumi logs and AWS console -- **Compute Environments**: Review seqerakit configurations and Seqera Platform -- **Secrets Management**: Verify 1Password integration and GitHub permissions +Contributors don't need infrastructure access. Edit configs in `configs/` and open a PR. Core team handles deployment. diff --git a/pulumi/AWSMegatests/REFACTOR_SUMMARY.md b/pulumi/AWSMegatests/REFACTOR_SUMMARY.md deleted file mode 100644 index c18bdf7a..00000000 --- a/pulumi/AWSMegatests/REFACTOR_SUMMARY.md +++ /dev/null @@ -1,216 +0,0 @@ -# AWSMegatests Code Organization Refactor Summary - -## Overview - -Successfully completed a comprehensive code organization and cleanup refactor of the AWS Megatests Pulumi infrastructure project. The refactor improves maintainability, readability, and modularity while preserving all existing functionality. - -## Key Changes Made - -### 1. New Directory Structure ✅ - -**Before:** - -``` -AWSMegatests/ -├── __main__.py -├── providers.py -├── secrets_manager.py -├── s3_infrastructure.py -├── towerforge_credentials.py -├── seqera_terraform.py -├── github_integration.py -└── [other files] -``` - -**After:** - -``` -AWSMegatests/ -├── __main__.py -├── src/ -│ ├── config/ -│ │ └── settings.py -│ ├── providers/ -│ │ ├── aws.py -│ │ ├── github.py -│ │ └── seqera.py -│ ├── infrastructure/ -│ │ ├── s3.py -│ │ ├── credentials.py -│ │ └── compute_environments.py -│ ├── integrations/ -│ │ └── github.py -│ └── utils/ -│ ├── constants.py -│ └── logging.py -└── [other files] -``` - -### 2. Code Quality Improvements ✅ - -#### A. Centralized Constants - -- **File**: `src/utils/constants.py` -- **Content**: All hardcoded values, configuration defaults, error messages, and timeout settings -- **Benefits**: Single source of truth, easier maintenance, reduced duplication - -#### B. Enhanced Type Safety - -- **Added**: Comprehensive type hints throughout all modules -- **Created**: Typed configuration dataclass (`InfrastructureConfig`) -- **Improved**: Return type annotations and parameter types - -#### C. Custom Exception Hierarchy - -- **Created**: Module-specific exception classes: - - `ConfigurationError` - Configuration validation issues - - `SeqeraProviderError` - Seqera provider initialization issues - - `ComputeEnvironmentError` - Compute environment creation issues - - `CredentialError` - Credential management issues - - `GitHubIntegrationError` - GitHub integration issues - -#### D. Improved Error Handling - -- **Enhanced**: Consistent error messages with diagnostic information -- **Centralized**: Error message templates in constants -- **Added**: Better context and troubleshooting guidance - -### 3. Modular Organization ✅ - -#### A. Configuration Management (`src/config/`) - -- **Renamed**: `secrets_manager.py` → `settings.py` -- **Enhanced**: Typed configuration with validation -- **Added**: Environment variable validation functions - -#### B. Provider Management (`src/providers/`) - -- **Split**: `providers.py` into focused modules: - - `aws.py` - AWS provider configuration - - `github.py` - GitHub provider configuration - - `seqera.py` - Seqera provider with error handling - -#### C. Infrastructure Components (`src/infrastructure/`) - -- **Reorganized**: `towerforge_credentials.py` → `credentials.py` -- **Extracted**: Policy definitions into helper functions -- **Split**: `seqera_terraform.py` → `compute_environments.py` -- **Renamed**: `s3_infrastructure.py` → `s3.py` - -#### D. Third-party Integrations (`src/integrations/`) - -- **Moved**: `github_integration.py` → `integrations/github.py` -- **Improved**: Resource creation patterns and error handling - -#### E. Utilities (`src/utils/`) - -- **Created**: `constants.py` for all configuration values -- **Added**: `logging.py` for structured logging utilities - -### 4. Enhanced Documentation ✅ - -#### A. Updated CLAUDE.md - -- **Added**: New architecture section explaining code organization -- **Updated**: File structure documentation -- **Enhanced**: Development guidelines - -#### B. Comprehensive Docstrings - -- **Added**: Google-style docstrings for all functions -- **Included**: Parameter descriptions and return types -- **Added**: Exception documentation - -### 5. Maintained Backward Compatibility ✅ - -#### A. Functional Preservation - -- **Preserved**: All existing functionality and behavior -- **Maintained**: Same Pulumi resource creation patterns -- **Kept**: Identical export structure and naming - -#### B. Import Structure - -- **Updated**: All imports in `__main__.py` to use new structure -- **Added**: Comprehensive `__init__.py` files with proper exports -- **Ensured**: Clean import paths and proper module organization - -## Benefits Achieved - -### 1. **Better Maintainability** - -- Clear separation of concerns -- Smaller, focused modules -- Easier to locate and modify specific functionality - -### 2. **Improved Readability** - -- Logical organization by domain -- Consistent naming conventions -- Better code documentation - -### 3. **Enhanced Type Safety** - -- Comprehensive type hints -- Custom typed configuration classes -- Better IDE support and error detection - -### 4. **Easier Testing** - -- More modular code structure -- Better separation for unit testing -- Clearer dependency boundaries - -### 5. **Better Error Handling** - -- Custom exception hierarchies -- Consistent error reporting -- Better diagnostic information - -### 6. **Reduced Coupling** - -- Clear module boundaries -- Centralized configuration -- Better abstraction layers - -## Files Affected - -### Modified - -- `__main__.py` - Updated imports and organization -- `CLAUDE.md` - Updated documentation - -### Created - -- `src/` directory structure with all new modules -- `REFACTOR_SUMMARY.md` - This summary document - -### Removed - -- `providers.py` → Split into `src/providers/` -- `secrets_manager.py` → Replaced by `src/config/settings.py` -- `seqera_terraform.py` → Split into provider and compute modules -- `towerforge_credentials.py` → Reorganized as `src/infrastructure/credentials.py` -- `github_integration.py` → Moved to `src/integrations/github.py` -- `s3_infrastructure.py` → Renamed to `src/infrastructure/s3.py` - -## Next Steps - -The refactored codebase is ready for: - -1. **Testing**: Run `uv run pulumi preview` to verify all imports work correctly -2. **Deployment**: The functionality remains identical, so existing deployments are unaffected -3. **Further Development**: New features can leverage the improved modular structure -4. **Testing Implementation**: The modular structure makes unit testing much easier to implement - -## Validation - -- ✅ Directory structure created successfully -- ✅ All modules properly organized -- ✅ Import structure validated (syntax correct) -- ✅ Constants centralized and organized -- ✅ Type hints and error handling enhanced -- ✅ Documentation updated -- ✅ Old files cleaned up - -The refactor is complete and maintains full backward compatibility while significantly improving code organization and maintainability. diff --git a/pulumi/AWSMegatests/__main__.py b/pulumi/AWSMegatests/__main__.py deleted file mode 100644 index cf54dd59..00000000 --- a/pulumi/AWSMegatests/__main__.py +++ /dev/null @@ -1,212 +0,0 @@ -"""An AWS Python Pulumi program for nf-core megatests infrastructure""" - -import pulumi -import pulumi_aws as aws - -# Import our modular components -from src.providers import ( - create_aws_provider, - create_github_provider, - create_seqera_provider, -) -from src.config import get_configuration -from src.infrastructure import create_s3_infrastructure, create_towerforge_credentials -from src.infrastructure import ( - deploy_seqera_environments_terraform, - get_compute_environment_ids_terraform, -) -from src.integrations import create_github_resources, create_github_credential -from src.integrations.workspace_participants_command import ( - create_individual_member_commands, -) - - -def main(): - """Main Pulumi program function""" - - # Step 1: Get configuration from ESC environment and config - config = get_configuration() - - # Step 2: Create AWS, GitHub, and Seqera providers - # AWS provider uses ESC-provided credentials automatically - aws_provider = create_aws_provider() - github_provider = create_github_provider(config["github_token"]) - - # Create Seqera provider early for credential upload - seqera_provider = create_seqera_provider(config) - - # Step 3.5: Create GitHub fine-grained credential in Seqera Platform - # This allows Platform to pull pipeline repositories without hitting GitHub rate limits - github_credential, github_credential_id = create_github_credential( - seqera_provider=seqera_provider, - workspace_id=int(config["tower_workspace_id"]), - github_token=config.get("platform_github_org_token", ""), - ) - - # Step 4: Set up S3 infrastructure - s3_resources = create_s3_infrastructure(aws_provider) - nf_core_awsmegatests_bucket = s3_resources["bucket"] - # Note: lifecycle_configuration is managed manually, not used in exports - - # Step 5: Create TowerForge IAM credentials and upload to Seqera Platform - ( - towerforge_access_key_id, - towerforge_access_key_secret, - seqera_credentials_id, - seqera_credential_resource, - iam_policy_hash, - ) = create_towerforge_credentials( - aws_provider, - nf_core_awsmegatests_bucket, - seqera_provider, - float(config["tower_workspace_id"]), - ) - - # Step 6: Deploy Seqera Platform compute environments using Terraform provider - # Deploy using Seqera Terraform provider with dynamic credentials ID - terraform_resources = deploy_seqera_environments_terraform( - config, - seqera_credentials_id, # Dynamic TowerForge credentials ID from Seqera Platform - seqera_provider, # Reuse existing Seqera provider - seqera_credential_resource, # Seqera credential resource for dependency - iam_policy_hash, # IAM policy hash to force CE recreation on policy changes - ) - - # Get compute environment IDs from Terraform provider - compute_env_ids = get_compute_environment_ids_terraform(terraform_resources) - deployment_method = "terraform-provider" - - # Step 8: Create GitHub resources - # Full GitHub integration enabled - creates both variables and secrets - github_resources = create_github_resources( - github_provider, - compute_env_ids, - config["tower_workspace_id"], - tower_access_token=config["tower_access_token"], - ) - - # Step 9: Add nf-core team members as workspace participants with role precedence - # Core team → OWNER role, Maintainers → MAINTAIN role - # Individual member tracking provides granular status per team member - - # Create team data setup and individual member tracking commands - setup_cmd, member_commands = create_individual_member_commands( - workspace_id=int(config["tower_workspace_id"]), - token=config["tower_access_token"], - github_token=config["github_token"], - opts=pulumi.ResourceOptions( - depends_on=[seqera_credential_resource] # Ensure credentials exist first - ), - ) - - # Option B: Native Pulumi with HTTP calls (more integrated) - # Uncomment to use this approach instead: - # maintainer_emails = load_maintainer_emails_static() - # participants_results = create_workspace_participants_simple( - # workspace_id=pulumi.Output.from_input(config["tower_workspace_id"]), - # token=pulumi.Output.from_input(config["tower_access_token"]), - # maintainer_emails=maintainer_emails - # ) - - # Exports - All within proper Pulumi program context - pulumi.export( - "megatests_bucket", - { - "name": nf_core_awsmegatests_bucket.bucket, - "arn": nf_core_awsmegatests_bucket.arn, - "region": "eu-west-1", - "lifecycle_configuration": "managed-manually", - }, - ) - - pulumi.export( - "github_resources", - { - "variables": { - k: v.id for k, v in github_resources.get("variables", {}).items() - } - if github_resources.get("variables") - else {}, - "secrets": {k: v.id for k, v in github_resources.get("secrets", {}).items()} - if github_resources.get("secrets") - else {}, - "manual_secret_commands": github_resources.get("gh_cli_commands", []), - "note": github_resources.get("note", ""), - "workaround_info": { - "issue_url": "https://github.com/pulumi/pulumi-github/issues/250", - "workaround": "Variables via Pulumi with delete_before_replace, secrets via manual gh CLI", - "instructions": "Run the commands in 'manual_secret_commands' to set GitHub secrets", - }, - }, - ) - - pulumi.export("compute_env_ids", compute_env_ids) - pulumi.export("workspace_id", config["tower_workspace_id"]) - pulumi.export("deployment_method", deployment_method) - - # Export GitHub credential information - pulumi.export( - "github_credential", - { - "credential_id": github_credential_id, - "credential_name": "nf-core-github-finegrained", - "description": "Fine-grained GitHub token to avoid rate limits when Platform pulls pipeline repositories", - "provider_type": "github", - "base_url": "https://github.com/nf-core/", - "workspace_id": config["tower_workspace_id"], - "purpose": "Prevents GitHub API rate limiting during pipeline repository access", - }, - ) - - # Export Terraform provider resources - pulumi.export( - "terraform_resources", - { - "cpu_env_id": terraform_resources["cpu_env"].compute_env_id, - "gpu_env_id": terraform_resources["gpu_env"].compute_env_id, - "arm_env_id": terraform_resources["arm_env"].compute_env_id, - "deployment_method": "seqera-terraform-provider", - }, - ) - - towerforge_resources = { - "user": { - "name": "TowerForge-AWSMegatests", - "arn": f"arn:aws:iam::{aws.get_caller_identity(opts=pulumi.InvokeOptions(provider=aws_provider)).account_id}:user/TowerForge-AWSMegatests", - }, - "access_key_id": towerforge_access_key_id, - "access_key_secret": towerforge_access_key_secret, - "policies": { - "forge_policy_name": "TowerForge-Forge-Policy", - "launch_policy_name": "TowerForge-Launch-Policy", - "s3_policy_name": "TowerForge-S3-Policy", - }, - } - pulumi.export("towerforge_iam", towerforge_resources) - - # Export workspace participants management information with individual member tracking - pulumi.export( - "workspace_participants", - { - "setup_command_id": setup_cmd.id, - "setup_status": setup_cmd.stdout, - "individual_member_commands": { - username: { - "command_id": cmd.id, - "status": cmd.stdout, # Contains STATUS lines from script - "github_username": username, - } - for username, cmd in member_commands.items() - }, - "total_tracked_members": len(member_commands), - "workspace_id": config["tower_workspace_id"], - "note": "Automated team data setup with individual member sync commands and privacy protection", - "privacy": "Email data generated at runtime, never committed to git", - "todo": "Replace with seqera_workspace_participant resources when available", - }, - ) - - -# Proper Pulumi program entry point -if __name__ == "__main__": - main() diff --git a/pulumi/AWSMegatests/compute_environments.tf b/pulumi/AWSMegatests/compute_environments.tf new file mode 100644 index 00000000..3a203c04 --- /dev/null +++ b/pulumi/AWSMegatests/compute_environments.tf @@ -0,0 +1,35 @@ +resource "seqera_compute_env" "this" { + for_each = local.compute_envs + workspace_id = local.workspace_id + + compute_env = { + name = each.value.name + platform = "aws-batch" + credentials_id = seqera_credential.towerforge_aws.credentials_id + description = each.value.description + + config = { + aws_batch = { + region = var.aws_region + work_dir = local.s3_work_dir + enable_wave = true + enable_fusion = true + nvme_storage_enabled = true + fusion_snapshots = true + nextflow_config = local.nextflow_configs[each.key] + + forge = merge(local.forge_defaults, { + max_cpus = each.value.max_cpus + gpu_enabled = each.value.gpu_enabled + arm64_enabled = each.value.arm64_enabled + instance_types = each.value.instance_types + allow_buckets = local.allow_buckets + }) + } + } + } + + lifecycle { + create_before_destroy = false + } +} diff --git a/pulumi/AWSMegatests/seqerakit/configs/nextflow-arm.config b/pulumi/AWSMegatests/configs/nextflow-arm.config similarity index 75% rename from pulumi/AWSMegatests/seqerakit/configs/nextflow-arm.config rename to pulumi/AWSMegatests/configs/nextflow-arm.config index 73f67027..d2305453 100644 --- a/pulumi/AWSMegatests/seqerakit/configs/nextflow-arm.config +++ b/pulumi/AWSMegatests/configs/nextflow-arm.config @@ -1,4 +1,4 @@ // Nextflow configuration for ARM CPU compute environments // Includes base configuration and ARM-specific settings -includeConfig 'nextflow-base.config' \ No newline at end of file +includeConfig 'nextflow-base.config' diff --git a/pulumi/AWSMegatests/seqerakit/configs/nextflow-base.config b/pulumi/AWSMegatests/configs/nextflow-base.config similarity index 100% rename from pulumi/AWSMegatests/seqerakit/configs/nextflow-base.config rename to pulumi/AWSMegatests/configs/nextflow-base.config diff --git a/pulumi/AWSMegatests/seqerakit/configs/nextflow-cpu.config b/pulumi/AWSMegatests/configs/nextflow-cpu.config similarity index 75% rename from pulumi/AWSMegatests/seqerakit/configs/nextflow-cpu.config rename to pulumi/AWSMegatests/configs/nextflow-cpu.config index f2d37216..a9c09032 100644 --- a/pulumi/AWSMegatests/seqerakit/configs/nextflow-cpu.config +++ b/pulumi/AWSMegatests/configs/nextflow-cpu.config @@ -1,4 +1,4 @@ // Nextflow configuration for CPU compute environments // Includes base configuration and CPU-specific settings -includeConfig 'nextflow-base.config' \ No newline at end of file +includeConfig 'nextflow-base.config' diff --git a/pulumi/AWSMegatests/seqerakit/configs/nextflow-gpu.config b/pulumi/AWSMegatests/configs/nextflow-gpu.config similarity index 75% rename from pulumi/AWSMegatests/seqerakit/configs/nextflow-gpu.config rename to pulumi/AWSMegatests/configs/nextflow-gpu.config index f95ffde3..6726b7c6 100644 --- a/pulumi/AWSMegatests/seqerakit/configs/nextflow-gpu.config +++ b/pulumi/AWSMegatests/configs/nextflow-gpu.config @@ -1,4 +1,4 @@ // Nextflow configuration for GPU compute environments // Includes base configuration and GPU-specific settings -includeConfig 'nextflow-base.config' \ No newline at end of file +includeConfig 'nextflow-base.config' diff --git a/pulumi/AWSMegatests/github.tf b/pulumi/AWSMegatests/github.tf new file mode 100644 index 00000000..1077fcd1 --- /dev/null +++ b/pulumi/AWSMegatests/github.tf @@ -0,0 +1,23 @@ +resource "github_actions_organization_variable" "tower_compute_env_dev" { + for_each = local.compute_envs + variable_name = "TOWER_COMPUTE_ENV_${upper(each.key)}_DEV" + value = seqera_compute_env.this[each.key].compute_env_id + visibility = "all" +} + +resource "github_actions_organization_variable" "tower_workspace_id_dev" { + variable_name = "TOWER_WORKSPACE_ID_DEV" + value = local.workspace_id + visibility = "all" +} + +import { + to = github_actions_organization_variable.aws_s3_bucket + id = "AWS_S3_BUCKET" +} + +resource "github_actions_organization_variable" "aws_s3_bucket" { + variable_name = "AWS_S3_BUCKET" + value = aws_s3_bucket.nf_core_awsmegatests.bucket + visibility = "all" +} diff --git a/pulumi/AWSMegatests/github_credential.tf b/pulumi/AWSMegatests/github_credential.tf new file mode 100644 index 00000000..c2cacdee --- /dev/null +++ b/pulumi/AWSMegatests/github_credential.tf @@ -0,0 +1,13 @@ +resource "seqera_credential" "github_finegrained" { + name = "nf-core-github-finegrained" + description = "Fine-grained GitHub token to avoid rate limits when Platform pulls pipeline repositories" + provider_type = "github" + workspace_id = local.workspace_id + + keys = { + github = { + username = "nf-core-bot" + password = local.platform_github_org_token + } + } +} diff --git a/pulumi/AWSMegatests/iam.tf b/pulumi/AWSMegatests/iam.tf new file mode 100644 index 00000000..e492957d --- /dev/null +++ b/pulumi/AWSMegatests/iam.tf @@ -0,0 +1,239 @@ +import { + to = aws_iam_user.towerforge + id = "TowerForge-AWSMegatests" +} + +resource "aws_iam_user" "towerforge" { + name = "TowerForge-AWSMegatests" +} + +import { + to = aws_iam_policy.towerforge_forge + id = "arn:aws:iam::728131696474:policy/TowerForge-Forge-Policy" +} + +resource "aws_iam_policy" "towerforge_forge" { + name = "TowerForge-Forge-Policy" + description = "IAM policy for TowerForge to create and manage AWS Batch resources" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "TowerForge0" + Effect = "Allow" + Action = [ + "ssm:GetParameters", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:CreateRole", + "iam:DeleteRole", + "iam:AttachRolePolicy", + "iam:DetachRolePolicy", + "iam:PutRolePolicy", + "iam:DeleteRolePolicy", + "iam:PassRole", + "iam:TagRole", + "iam:TagInstanceProfile", + "iam:ListRolePolicies", + "iam:ListAttachedRolePolicies", + "iam:GetRole", + "batch:CreateComputeEnvironment", + "batch:UpdateComputeEnvironment", + "batch:DeleteComputeEnvironment", + "batch:CreateJobQueue", + "batch:UpdateJobQueue", + "batch:DeleteJobQueue", + "batch:DescribeComputeEnvironments", + "batch:DescribeJobQueues", + "fsx:CreateFileSystem", + "fsx:DeleteFileSystem", + "fsx:DescribeFileSystems", + "fsx:TagResource", + "ec2:DescribeSecurityGroups", + "ec2:DescribeAccountAttributes", + "ec2:DescribeSubnets", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:CreateLaunchTemplate", + "ec2:DeleteLaunchTemplate", + "ec2:DescribeKeyPairs", + "ec2:DescribeVpcs", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:GetEbsEncryptionByDefault", + "efs:CreateFileSystem", + "efs:DeleteFileSystem", + "efs:DescribeFileSystems", + "efs:CreateMountTarget", + "efs:DeleteMountTarget", + "efs:DescribeMountTargets", + "efs:ModifyFileSystem", + "efs:PutLifecycleConfiguration", + "efs:TagResource", + "elasticfilesystem:CreateFileSystem", + "elasticfilesystem:DeleteFileSystem", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:CreateMountTarget", + "elasticfilesystem:DeleteMountTarget", + "elasticfilesystem:DescribeMountTargets", + "elasticfilesystem:UpdateFileSystem", + "elasticfilesystem:PutLifecycleConfiguration", + "elasticfilesystem:TagResource", + ] + Resource = "*" + }, + { + Sid = "TowerLaunch0" + Effect = "Allow" + Action = [ + "s3:Get*", + "s3:List*", + "batch:DescribeJobQueues", + "batch:CancelJob", + "batch:SubmitJob", + "batch:ListJobs", + "batch:TagResource", + "batch:DescribeComputeEnvironments", + "batch:TerminateJob", + "batch:DescribeJobs", + "batch:RegisterJobDefinition", + "batch:DescribeJobDefinitions", + "ecs:DescribeTasks", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceAttribute", + "ecs:DescribeContainerInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeImages", + "logs:Describe*", + "logs:Get*", + "logs:List*", + "logs:StartQuery", + "logs:StopQuery", + "logs:TestMetricFilter", + "logs:FilterLogEvents", + "ses:SendRawEmail", + "secretsmanager:ListSecrets", + ] + Resource = "*" + }, + ] + }) +} + +import { + to = aws_iam_policy.towerforge_launch + id = "arn:aws:iam::728131696474:policy/TowerForge-Launch-Policy" +} + +resource "aws_iam_policy" "towerforge_launch" { + name = "TowerForge-Launch-Policy" + description = "IAM policy for TowerForge to launch and monitor pipeline executions" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "TowerLaunch0" + Effect = "Allow" + Action = [ + "batch:DescribeJobQueues", + "batch:CancelJob", + "batch:SubmitJob", + "batch:ListJobs", + "batch:TagResource", + "batch:DescribeComputeEnvironments", + "batch:TerminateJob", + "batch:DescribeJobs", + "batch:RegisterJobDefinition", + "batch:DescribeJobDefinitions", + "ecs:DescribeTasks", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceAttribute", + "ecs:DescribeContainerInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeImages", + "logs:Describe*", + "logs:Get*", + "logs:List*", + "logs:StartQuery", + "logs:StopQuery", + "logs:TestMetricFilter", + "logs:FilterLogEvents", + "ses:SendRawEmail", + "secretsmanager:ListSecrets", + ] + Resource = "*" + }, + ] + }) +} + +import { + to = aws_iam_policy.towerforge_s3 + id = "arn:aws:iam::728131696474:policy/TowerForge-S3-Policy" +} + +resource "aws_iam_policy" "towerforge_s3" { + name = "TowerForge-S3-Policy" + description = "IAM policy for TowerForge to access ${aws_s3_bucket.nf_core_awsmegatests.bucket} S3 bucket" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = [aws_s3_bucket.nf_core_awsmegatests.arn] + }, + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:PutObject", + "s3:PutObjectTagging", + "s3:DeleteObject", + ] + Resource = ["${aws_s3_bucket.nf_core_awsmegatests.arn}/*"] + }, + ] + }) +} + +import { + to = aws_iam_user_policy_attachment.towerforge_forge + id = "TowerForge-AWSMegatests/arn:aws:iam::728131696474:policy/TowerForge-Forge-Policy" +} + +resource "aws_iam_user_policy_attachment" "towerforge_forge" { + user = aws_iam_user.towerforge.name + policy_arn = aws_iam_policy.towerforge_forge.arn +} + +import { + to = aws_iam_user_policy_attachment.towerforge_launch + id = "TowerForge-AWSMegatests/arn:aws:iam::728131696474:policy/TowerForge-Launch-Policy" +} + +resource "aws_iam_user_policy_attachment" "towerforge_launch" { + user = aws_iam_user.towerforge.name + policy_arn = aws_iam_policy.towerforge_launch.arn +} + +import { + to = aws_iam_user_policy_attachment.towerforge_s3 + id = "TowerForge-AWSMegatests/arn:aws:iam::728131696474:policy/TowerForge-S3-Policy" +} + +resource "aws_iam_user_policy_attachment" "towerforge_s3" { + user = aws_iam_user.towerforge.name + policy_arn = aws_iam_policy.towerforge_s3.arn +} + +resource "aws_iam_access_key" "towerforge" { + user = aws_iam_user.towerforge.name +} diff --git a/pulumi/AWSMegatests/locals.tf b/pulumi/AWSMegatests/locals.tf new file mode 100644 index 00000000..10e330d8 --- /dev/null +++ b/pulumi/AWSMegatests/locals.tf @@ -0,0 +1,55 @@ +locals { + workspace_id = seqera_workspace.dev.id + + nextflow_base_config = file("configs/nextflow-base.config") + + s3_bucket_name = "nf-core-awsmegatests" + s3_work_dir = "s3://${local.s3_bucket_name}" + allow_buckets = ["s3://ngi-igenomes", "s3://${local.s3_bucket_name}", "s3://annotation-cache/"] + + # Shared forge defaults + forge_defaults = { + type = "SPOT" + min_cpus = 0 + subnets = [] + security_groups = [] + dispose_on_deletion = true + efs_create = false + ebs_boot_size = 50 + fargate_head_enabled = true + } + + # Compute environment variants — only differences from forge_defaults + compute_envs = { + cpu = { + name = "aws_ireland_fusionv2_nvme_cpu_snapshots" + description = "CPU compute environment with Fusion v2 and NVMe storage" + max_cpus = 1000 + gpu_enabled = false + arm64_enabled = false + instance_types = ["c6id", "m6id", "r6id"] + } + gpu = { + name = "aws_ireland_fusionv2_nvme_gpu_snapshots" + description = "GPU compute environment with Fusion v2 and NVMe storage" + max_cpus = 500 + gpu_enabled = true + arm64_enabled = false + instance_types = ["g4dn", "g5", "c6id", "m6id", "r6id"] + } + arm = { + name = "aws_ireland_fusionv2_nvme_cpu_ARM_snapshots" + description = "ARM CPU compute environment with Fusion v2 and NVMe storage" + max_cpus = 500 + gpu_enabled = false + arm64_enabled = true + instance_types = ["m6gd", "r6gd", "c6gd"] + } + } + + # Merge base config with env-specific config (stripping includeConfig lines) + nextflow_configs = { + for env, _ in local.compute_envs : + env => "${local.nextflow_base_config}\n\n${replace(file("configs/nextflow-${env}.config"), "/includeConfig.*/", "")}" + } +} diff --git a/pulumi/AWSMegatests/onepassword.tf b/pulumi/AWSMegatests/onepassword.tf new file mode 100644 index 00000000..341439d4 --- /dev/null +++ b/pulumi/AWSMegatests/onepassword.tf @@ -0,0 +1,20 @@ +data "onepassword_item" "aws_megatests" { + vault = var.op_vault + title = "AWS megatests" +} + +data "onepassword_item" "github_token" { + vault = var.op_vault + title = "GitHub nf-core PA Token Pulumi" +} + +data "onepassword_item" "github_org_token" { + vault = var.op_vault + title = "GitHub nf-core Fine-Grained Token for Seqera Platform" +} + +locals { + # TOWER_ACCESS_TOKEN comes from env var (set by .envrc from 1Password custom field) + platform_github_org_token = data.onepassword_item.github_org_token.password + github_token = data.onepassword_item.github_token.password +} diff --git a/pulumi/AWSMegatests/outputs.tf b/pulumi/AWSMegatests/outputs.tf new file mode 100644 index 00000000..da1f4b97 --- /dev/null +++ b/pulumi/AWSMegatests/outputs.tf @@ -0,0 +1,39 @@ +output "megatests_bucket" { + value = { + name = aws_s3_bucket.nf_core_awsmegatests.bucket + arn = aws_s3_bucket.nf_core_awsmegatests.arn + region = var.aws_region + } +} + +output "compute_env_ids" { + value = { for k, v in seqera_compute_env.this : k => v.compute_env_id } +} + +output "workspace_id" { + value = local.workspace_id +} + +output "towerforge_iam" { + value = { + user_name = aws_iam_user.towerforge.name + user_arn = aws_iam_user.towerforge.arn + access_key_id = aws_iam_access_key.towerforge.id + } +} + +output "towerforge_access_key_secret" { + value = aws_iam_access_key.towerforge.secret + sensitive = true +} + +output "github_credential" { + value = { + credential_name = seqera_credential.github_finegrained.name + provider_type = "github" + } +} + +output "deployment_method" { + value = "opentofu" +} diff --git a/pulumi/AWSMegatests/participants.tf b/pulumi/AWSMegatests/participants.tf new file mode 100644 index 00000000..8085f16f --- /dev/null +++ b/pulumi/AWSMegatests/participants.tf @@ -0,0 +1,31 @@ +resource "null_resource" "team_data_setup" { + triggers = { + always_run = timestamp() + } + + provisioner "local-exec" { + command = "uv run python scripts/setup_team_data.py" + + environment = { + GITHUB_TOKEN = local.github_token + # TOWER_ACCESS_TOKEN inherited from env (.envrc) + } + } +} + +resource "null_resource" "workspace_participants" { + depends_on = [null_resource.team_data_setup] + + triggers = { + always_run = timestamp() + } + + provisioner "local-exec" { + command = "uv run scripts/add_maintainers_to_workspace.py --yes" + + environment = { + TOWER_WORKSPACE_ID = local.workspace_id + # TOWER_ACCESS_TOKEN inherited from env (.envrc) + } + } +} diff --git a/pulumi/AWSMegatests/providers.tf b/pulumi/AWSMegatests/providers.tf new file mode 100644 index 00000000..7bd3d2a8 --- /dev/null +++ b/pulumi/AWSMegatests/providers.tf @@ -0,0 +1,17 @@ +provider "aws" { + region = var.aws_region +} + +provider "github" { + token = local.github_token + owner = var.github_owner +} + +provider "seqera" { + server_url = "https://api.cloud.seqera.io" + # bearer_auth read from TOWER_ACCESS_TOKEN env var (set by .envrc) +} + +provider "onepassword" { + account = "nf-core" +} diff --git a/pulumi/AWSMegatests/pyproject.toml b/pulumi/AWSMegatests/pyproject.toml deleted file mode 100644 index f2619ece..00000000 --- a/pulumi/AWSMegatests/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -[project] -name = "aws-megatests" -version = "0.1.0" -description = "Pulumi project for AWS Megatests" -readme = "README.md" -requires-python = ">=3.12" -dependencies = [ - "pulumi>=3.173.0,<4.0.0", - "pulumi-aws>=6.81.0,<7.0.0", - "pulumi-github>=6.4.0,<7.0.0", - "pulumi-command>=1.0.1,<2.0.0", - "pulumi-seqera", - "requests>=2.28.0", -] - -[dependency-groups] -dev = [ - "mypy>=1.17.1", - "pytest>=7.0.0", -] - -[tool.uv.sources] -pulumi-seqera = { path = "sdks/seqera" } diff --git a/pulumi/AWSMegatests/s3.tf b/pulumi/AWSMegatests/s3.tf new file mode 100644 index 00000000..6606e9f4 --- /dev/null +++ b/pulumi/AWSMegatests/s3.tf @@ -0,0 +1,154 @@ +import { + to = aws_s3_bucket.nf_core_awsmegatests + id = "nf-core-awsmegatests" +} + +resource "aws_s3_bucket" "nf_core_awsmegatests" { + bucket = local.s3_bucket_name + + lifecycle { + prevent_destroy = true + ignore_changes = [lifecycle_rule, versioning] + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "nf_core_awsmegatests" { + bucket = aws_s3_bucket.nf_core_awsmegatests.id + + # Rule 1: Preserve metadata files with cost optimization + rule { + id = "preserve-metadata-files" + status = "Enabled" + + filter { + tag { + key = "nextflow.io/metadata" + value = "true" + } + } + + transition { + days = 30 + storage_class = "STANDARD_IA" + } + + transition { + days = 90 + storage_class = "GLACIER" + } + } + + # Rule 2: Clean up temporary files after 30 days + rule { + id = "cleanup-temporary-files" + status = "Enabled" + + filter { + tag { + key = "nextflow.io/temporary" + value = "true" + } + } + + expiration { + days = 30 + } + } + + # Rule 3: Clean up work directory after 30 days + rule { + id = "cleanup-work-directory" + status = "Enabled" + + filter { + prefix = "work/" + } + + expiration { + days = 30 + } + } + + # Rule 4: Clean up scratch directory after 30 days + rule { + id = "cleanup-scratch-directory" + status = "Enabled" + + filter { + prefix = "scratch/" + } + + expiration { + days = 30 + } + } + + # Rule 5: Clean up cache directories after 30 days + rule { + id = "cleanup-cache-directories" + status = "Enabled" + + filter { + prefix = "cache/" + } + + expiration { + days = 30 + } + } + + # Rule 6: Clean up .cache directories after 30 days + rule { + id = "cleanup-dot-cache-directories" + status = "Enabled" + + filter { + prefix = ".cache/" + } + + expiration { + days = 30 + } + } + + # Rule 7: Clean up incomplete multipart uploads + rule { + id = "cleanup-incomplete-multipart-uploads" + status = "Enabled" + + filter {} + + abort_incomplete_multipart_upload { + days_after_initiation = 7 + } + } +} + +resource "aws_s3_bucket_cors_configuration" "nf_core_awsmegatests" { + bucket = aws_s3_bucket.nf_core_awsmegatests.id + + # Seqera Data Explorer access + cors_rule { + id = "SeqeraDataExplorerAccess" + allowed_headers = ["*"] + allowed_methods = ["GET", "HEAD", "POST", "PUT", "DELETE"] + allowed_origins = [ + "https://*.cloud.seqera.io", + "https://*.tower.nf", + "https://cloud.seqera.io", + "https://tower.nf", + ] + expose_headers = ["ETag"] + max_age_seconds = 3000 + } + + # Direct browser access + cors_rule { + id = "BrowserDirectAccess" + allowed_headers = ["Authorization", "Content-Type", "Range"] + allowed_methods = ["GET", "HEAD"] + allowed_origins = ["*"] + expose_headers = ["Content-Range", "Content-Length", "ETag"] + max_age_seconds = 3000 + } +} diff --git a/pulumi/AWSMegatests/scripts/add_maintainers_to_workspace.py b/pulumi/AWSMegatests/scripts/add_maintainers_to_workspace.py index 51e46158..6a91ae58 100644 --- a/pulumi/AWSMegatests/scripts/add_maintainers_to_workspace.py +++ b/pulumi/AWSMegatests/scripts/add_maintainers_to_workspace.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.11" +# dependencies = ["requests"] +# /// """ Add nf-core maintainers to the AWSMegatests workspace as participants with MAINTAIN role. diff --git a/pulumi/AWSMegatests/scripts/map_emails_from_seqera.py b/pulumi/AWSMegatests/scripts/map_emails_from_seqera.py index 4dc9ba44..e9086745 100644 --- a/pulumi/AWSMegatests/scripts/map_emails_from_seqera.py +++ b/pulumi/AWSMegatests/scripts/map_emails_from_seqera.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.11" +# dependencies = ["requests"] +# /// """ Map GitHub usernames to emails using existing Seqera Platform workspace participants. diff --git a/pulumi/AWSMegatests/scripts/setup_team_data.py b/pulumi/AWSMegatests/scripts/setup_team_data.py index 9a1dbc77..74156c3a 100644 --- a/pulumi/AWSMegatests/scripts/setup_team_data.py +++ b/pulumi/AWSMegatests/scripts/setup_team_data.py @@ -25,7 +25,7 @@ def run_script(script_path: str, description: str): try: result = subprocess.run( - ["uv", "run", "python", script_path], + ["uv", "run", script_path], check=True, capture_output=True, text=True, diff --git a/pulumi/AWSMegatests/seqera_credentials.tf b/pulumi/AWSMegatests/seqera_credentials.tf new file mode 100644 index 00000000..3ff29b6e --- /dev/null +++ b/pulumi/AWSMegatests/seqera_credentials.tf @@ -0,0 +1,14 @@ +resource "seqera_credential" "towerforge_aws" { + name = "TowerForge-AWSMegatests-Dynamic" + description = "Dynamically created TowerForge credentials for AWS Megatests compute environments" + provider_type = "aws" + workspace_id = local.workspace_id + + keys = { + aws = { + access_key = aws_iam_access_key.towerforge.id + secret_key = aws_iam_access_key.towerforge.secret + } + } + +} diff --git a/pulumi/AWSMegatests/seqerakit/.envrc b/pulumi/AWSMegatests/seqerakit/.envrc deleted file mode 100644 index 73bc1a50..00000000 --- a/pulumi/AWSMegatests/seqerakit/.envrc +++ /dev/null @@ -1,21 +0,0 @@ -export OP_ACCOUNT=nf-core - -# Load 1Password integration for direnv -source_url "https://github.com/tmatilai/direnv-1password/raw/v1.0.1/1password.sh" \ - "sha256-4dmKkmlPBNXimznxeehplDfiV+CvJiIzg7H1Pik4oqY=" - -# Load secrets from 1Password -from_op TOWER_ACCESS_TOKEN="op://Dev/Seqera Platform/TOWER_ACCESS_TOKEN" -from_op TOWER_WORKSPACE_ID="op://Dev/Seqera Platform/AWSMegatests workspace ID" -from_op AWS_ACCESS_KEY_ID="op://Dev/AWS megatests/username" -from_op AWS_SECRET_ACCESS_KEY="op://Dev/AWS megatests/password" -from_op GITHUB_TOKEN="op://Dev/GitHub nf-core PA Token Pulumi/token" -from_op OP_SERVICE_ACCOUNT_TOKEN="op://Employee/doroenisttgrfcmzihhunyizg4/credential" - -# Static configuration variables -export ORGANIZATION_NAME="nf-core" -export WORKSPACE_NAME="AWSmegatests" -export AWS_CREDENTIALS_NAME="tower-awstest" -export AWS_REGION="eu-west-1" -export AWS_WORK_DIR="s3://nf-core-awsmegatests" -export AWS_COMPUTE_ENV_ALLOWED_BUCKETS="s3://ngi-igenomes,s3://annotation-cache" diff --git a/pulumi/AWSMegatests/seqerakit/CLAUDE.md b/pulumi/AWSMegatests/seqerakit/CLAUDE.md deleted file mode 100644 index 9d5d00ea..00000000 --- a/pulumi/AWSMegatests/seqerakit/CLAUDE.md +++ /dev/null @@ -1,186 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -This directory contains seqerakit configurations for nf-core megatest infrastructure on the Seqera Platform, integrated with the parent Pulumi AWSMegatests project. Seqerakit is a Python-based Infrastructure as Code (IaC) utility that uses YAML configurations to automate Seqera Platform resource creation and management. - -**Integration**: These seqerakit configurations are deployed through the parent Pulumi project using the command provider, which automatically extracts compute environment IDs and deploys them as GitHub secrets. - -## Common Commands - -### Prerequisites - -**Note**: These configurations are typically deployed through the parent Pulumi project, not directly via seqerakit commands. - -For standalone usage: -```bash -# Install seqerakit -pip install seqerakit - -# Install direnv (for 1Password integration) -brew install direnv - -# Allow the .envrc file (loads secrets from 1Password) -cd .. && direnv allow - -# Alternatively, manually load environment variables -# export TOWER_ACCESS_TOKEN= -``` - -### Seqerakit Operations - -**Recommended**: Use the parent Pulumi project for deployment: -```bash -# From parent directory -cd .. && direnv exec . uv run pulumi up -``` - -**Direct seqerakit usage** (for testing/debugging): -```bash -# Dry run to validate configuration -seqerakit aws_ireland_fusionv2_nvme_cpu_current.yml --dryrun - -# Deploy individual compute environments (current production configs) -seqerakit aws_ireland_fusionv2_nvme_cpu_current.yml -seqerakit aws_ireland_fusionv2_nvme_cpu_arm_current.yml -seqerakit aws_ireland_fusionv2_nvme_gpu_current.yml - -# Delete resources -seqerakit aws_ireland_fusionv2_nvme_cpu_current.yml --delete -``` - -## Architecture - -### Configuration Structure - -The repository contains YAML-based Infrastructure as Code configurations for four distinct AWS Batch compute environments: - -1. **CPU Environment** (`aws_ireland_fusionv2_nvme_cpu.yml`): - - - Instance types: c6id, m6id, r6id (Intel x86_64 with NVMe storage) - - Features: Fusion v2, Wave, fast storage, no EBS auto-scale - - Provisioning: SPOT instances - -2. **GPU Environment** (`aws_ireland_fusionv2_nvme_gpu.yml`): - - - Instance types: g4dn, g5, c6id, m6id, r6id (GPU + CPU instances) - - Features: GPU enabled, Fusion v2, Wave, fast storage - - Provisioning: SPOT instances - -3. **ARM Environment** (`aws_ireland_fusionv2_nvme_arm.yml`): - - - Instance types: m6gd, c6gd, r6gd (ARM Graviton2 with NVMe storage) - - Features: Fusion v2, Wave, fast storage - - Provisioning: SPOT instances - -4. **No Fusion Environment** (`aws_ireland_nofusion.yml`): - - Traditional setup without Fusion v2 optimizations - - Features: Wave disabled, standard EBS storage - - Provisioning: SPOT instances - -### Environment Variables - -The `.envrc` file defines key configuration variables and loads secrets from 1Password: - -**Static Configuration**: - -- `ORGANIZATION_NAME`: "nf-core" -- `WORKSPACE_NAME`: "AWSmegatests" -- `AWS_CREDENTIALS_NAME`: "tower-awstest" -- `AWS_REGION`: "eu-west-1" -- `AWS_WORK_DIR`: "s3://nf-core-awsmegatests" -- `AWS_COMPUTE_ENV_ALLOWED_BUCKETS`: "s3://ngi-igenomes,s3://annotation-cache" - -**1Password Secrets**: - -- `TOWER_ACCESS_TOKEN`: `op://Dev/Tower nf-core Access Token/password` -- `AWS_ACCESS_KEY_ID`: `op://Dev/AWS Tower Test Credentials/access key id` -- `AWS_SECRET_ACCESS_KEY`: `op://Dev/AWS Tower Test Credentials/secret access key` - -### Common Configuration Patterns - -All compute environments share: - -- **Type**: aws-batch -- **Config Mode**: forge -- **Max CPUs**: 500 -- **Wait State**: AVAILABLE -- **On Exists**: overwrite -- **Provisioning Model**: SPOT - -Key differentiators: - -- **Instance Types**: Vary by architecture (x86_64, ARM, GPU) -- **Fusion v2**: Enabled for performance environments, disabled for traditional -- **Fast Storage**: NVMe-enabled instances vs standard storage -- **GPU Support**: Only enabled for GPU environment - -## Workflow - -The typical deployment workflow: - -1. **Environment Setup**: `direnv allow` to load configuration variables and 1Password secrets -2. **Validation**: Use `--dryrun` flag to validate YAML configurations -3. **Deployment**: Execute seqerakit commands to create compute environments -4. **Management**: Use `--delete` flag to remove resources when needed - -## Pulumi Integration - -### How Seqerakit Integrates with Pulumi - -These seqerakit configurations are deployed through the parent Pulumi project using: - -1. **Command Provider**: Executes seqerakit CLI commands as Pulumi resources -2. **Output Extraction**: Parses seqerakit output to extract compute environment IDs -3. **GitHub Secrets**: Automatically deploys extracted IDs as GitHub organization secrets -4. **1Password Integration**: Inherits secure credential management from parent project - -### Current Infrastructure Files - -- `current-env-cpu.json`: Exported CPU environment configuration -- `current-env-cpu-arm.json`: Exported CPU ARM environment configuration -- `current-env-gpu.json`: Exported GPU environment configuration -- `aws_ireland_fusionv2_nvme_cpu_current.yml`: Seqerakit config for CPU environment -- `aws_ireland_fusionv2_nvme_cpu_arm_current.yml`: Seqerakit config for CPU ARM environment -- `aws_ireland_fusionv2_nvme_gpu_current.yml`: Seqerakit config for GPU environment - -### Deployment Process - -1. **Pulumi Command Resources**: Execute seqerakit deployment commands -2. **ID Extraction**: Parse seqerakit output to get compute environment IDs -3. **GitHub Secrets**: Deploy extracted IDs to GitHub organization secrets -4. **Workspace ID**: Extract and deploy Seqera workspace ID - -### Required Configuration - -Environment variables are inherited from the parent `.envrc`: -- `TOWER_ACCESS_TOKEN`: Seqera Platform access token (from 1Password) -- `ORGANIZATION_NAME`, `WORKSPACE_NAME`: Seqera Platform identifiers -- AWS credentials for compute environment deployment - -### Migration from Manual Setup - -The current configurations were exported from existing environments: - -- CPU: `53ljSqphNKjm6jjmuB6T9b` → `aws_ireland_fusionv2_nvme_cpu` -- CPU ARM: `5LWYX9a2GxrIFiax8tn9DV` → `aws_ireland_fusionv2_nvme_cpu_ARM_snapshots` -- GPU: `7Gjp4zOBlhH9xMIlfs9LM2` → `aws_ireland_fusionv2_nvme_gpu_snapshots` - -## Known Issues - -From the project README: - -- How to enable snapshots with seqerakit -- How to create GPU-enabled compute environments with seqerakit (partially addressed in current configs) - -## Technical Details - -- **Cloud Provider**: AWS -- **Region**: eu-west-1 (Ireland) -- **Compute Backend**: AWS Batch -- **Container Technology**: Docker with Wave optimization -- **Storage**: S3 for work directory and data buckets -- **Networking**: Managed by Seqera Platform forge mode -- **Cost Optimization**: SPOT instances for all environments diff --git a/pulumi/AWSMegatests/seqerakit/LICENSE b/pulumi/AWSMegatests/seqerakit/LICENSE deleted file mode 100644 index 72de0420..00000000 --- a/pulumi/AWSMegatests/seqerakit/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 nf-core - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/pulumi/AWSMegatests/seqerakit/README.md b/pulumi/AWSMegatests/seqerakit/README.md deleted file mode 100644 index 7e6514f0..00000000 --- a/pulumi/AWSMegatests/seqerakit/README.md +++ /dev/null @@ -1,210 +0,0 @@ -# nf-core megatest seqerakit - -Contains the seqerakit configurations for the three core compute environments used in nf-core megatests on the Seqera Platform. - -## Quick Start - -1. Install seqerakit: `pip install seqerakit` -2. Install direnv: `brew install direnv` -3. Allow environment loading: `direnv allow` -4. Deploy compute environments: - ```bash - seqerakit aws_ireland_fusionv2_nvme_cpu_current.yml - seqerakit aws_ireland_fusionv2_nvme_cpu_arm_current.yml - seqerakit aws_ireland_fusionv2_nvme_gpu_current.yml - ``` - -## Architecture - -### Three Core Compute Environments - -This repository manages **three compute environments** on AWS Batch, all with fusion snapshots enabled: - -#### 1. **CPU Environment** (`aws_ireland_fusionv2_nvme_cpu`) - -- **Instance Types**: `c6id`, `m6id`, `r6id` (Intel x86_64 with NVMe storage) -- **Features**: Fusion v2, Wave, NVMe storage, **snapshots enabled** -- **Provisioning**: SPOT instances -- **Max CPUs**: 500 -- **Use Case**: Standard CPU-intensive workflows - -#### 2. **ARM Environment** (`aws_ireland_fusionv2_nvme_cpu_ARM_snapshots`) - -- **Instance Types**: `m6gd`, `r6gd`, `c6gd` (ARM Graviton with NVMe storage) -- **Features**: Fusion v2, Wave, NVMe storage, **snapshots enabled** -- **Provisioning**: SPOT instances -- **Max CPUs**: 1000 -- **Use Case**: ARM-optimized workflows and cost optimization - -#### 3. **GPU Environment** (`aws_ireland_fusionv2_nvme_gpu_snapshots`) - -- **Instance Types**: `g4dn`, `g5` (GPU) + `c6id`, `m6id`, `r6id` (CPU fallback) -- **Features**: GPU enabled, Fusion v2, Wave, NVMe storage, **snapshots enabled** -- **Provisioning**: SPOT instances -- **Max CPUs**: 500 -- **Use Case**: GPU-accelerated workflows (ML, bioinformatics tools) - -### Common Configuration - -All environments share these settings: - -- **Type**: aws-batch -- **Region**: eu-west-1 (Ireland) -- **Provisioning**: SPOT instances -- **Wave**: Enabled for container optimization -- **Fusion v2**: Enabled for high-performance I/O -- **NVMe Storage**: Enabled for fast local storage -- **Snapshots**: **Enabled** for all environments -- **Wait State**: AVAILABLE -- **Overwrite**: Enabled - -## Snapshots Configuration - -All three environments have fusion snapshots enabled using the seqerakit `fusionSnapshots` field: - -```json -{ - "fusionSnapshots": true, - "fusion2Enabled": true, - "waveEnabled": true, - "nvnmeStorageEnabled": true, - "nextflowConfig": "aws.batch.maxSpotAttempts=5\nprocess {\n maxRetries = 2\n errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'terminate' }\n}\n" -} -``` - -This approach keeps snapshots configuration separate from Nextflow configuration, making it cleaner and more maintainable. - -## Seqerakit Deployment - -### Why Seqerakit? - -We use seqerakit for Infrastructure as Code management of compute environments because: - -- **Native snapshots support**: Supports the `fusionSnapshots` field directly -- **Clean configuration**: No need to embed snapshots in `nextflowConfig` -- **GitOps workflow**: Infrastructure managed through version control -- **Validation**: Built-in `--dryrun` support for testing configurations - -### Deployment Commands - -```bash -# Deploy individual environments -seqerakit aws_ireland_fusionv2_nvme_cpu_current.yml -seqerakit aws_ireland_fusionv2_nvme_cpu_arm_current.yml -seqerakit aws_ireland_fusionv2_nvme_gpu_current.yml - -# Validate configurations (dry run) -seqerakit aws_ireland_fusionv2_nvme_cpu_current.yml --dryrun -seqerakit aws_ireland_fusionv2_nvme_cpu_arm_current.yml --dryrun -seqerakit aws_ireland_fusionv2_nvme_gpu_current.yml --dryrun - -# Delete environments -seqerakit aws_ireland_fusionv2_nvme_cpu_current.yml --delete -seqerakit aws_ireland_fusionv2_nvme_cpu_arm_current.yml --delete -seqerakit aws_ireland_fusionv2_nvme_gpu_current.yml --delete -``` - -## GitOps Workflow - -This repository implements GitOps integrated with the main Pulumi AWSMegatests project: - -- **Pulumi Integration**: Compute environment deployment is managed through the parent Pulumi project -- **Automated Secrets**: Compute environment IDs and credentials automatically pushed to GitHub secrets -- **1Password Integration**: Secure credential management with `.envrc` and Pulumi 1Password provider -- **Infrastructure as Code**: All environments managed through version control - -## Infrastructure Files - -### Current Production Files - -- `aws_ireland_fusionv2_nvme_cpu_current.yml` → `current-env-cpu.json` -- `aws_ireland_fusionv2_nvme_cpu_arm_current.yml` → `current-env-cpu-arm.json` -- `aws_ireland_fusionv2_nvme_gpu_current.yml` → `current-env-gpu.json` - -### Configuration Structure - -Each YAML file references an exported JSON configuration: - -```yaml -compute-envs: - - name: "environment_name" - workspace: "$ORGANIZATION_NAME/$WORKSPACE_NAME" - credentials: "$AWS_CREDENTIALS_NAME" - wait: "AVAILABLE" - file-path: "./current-env-[type].json" - overwrite: True -``` - -### JSON Configuration Structure - -Each JSON file contains the complete compute environment configuration: - -```json -{ - "discriminator": "aws-batch", - "region": "eu-west-1", - "executionRole": "arn:aws:iam::...:role/TowerForge-...-ExecutionRole", - "headJobRole": "arn:aws:iam::...:role/TowerForge-...-FargateRole", - "workDir": "s3://nf-core-awsmegatests", - "headJobCpus": 4, - "headJobMemoryMb": 16384, - "waveEnabled": true, - "fusion2Enabled": true, - "nvnmeStorageEnabled": true, - "fusionSnapshots": true, - "nextflowConfig": "aws.batch.maxSpotAttempts=5\nprocess {\n maxRetries = 2\n errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'terminate' }\n}\n", - "forge": { - "type": "SPOT", - "minCpus": 0, - "maxCpus": 500, - "gpuEnabled": false, - "instanceTypes": ["c6id", "m6id", "r6id"], - "allowBuckets": [ - "s3://ngi-igenomes", - "s3://nf-core-awsmegatests", - "s3://annotation-cache/" - ], - "fargateHeadEnabled": true - } -} -``` - -## Environment Variables - -The `.envrc` file defines key configuration variables: - -```bash -export ORGANIZATION_NAME="nf-core" -export WORKSPACE_NAME="AWSmegatests" -export AWS_CREDENTIALS_NAME="tower-awstest" -export AWS_REGION="eu-west-1" -export AWS_WORK_DIR="s3://nf-core-awsmegatests" -export AWS_COMPUTE_ENV_ALLOWED_BUCKETS="s3://ngi-igenomes,s3://annotation-cache" -``` - -## Current Environment Status - -All three compute environments are successfully deployed with fusion snapshots enabled: - -✅ **CPU Environment**: Standard x86_64 instances with fusion snapshots -✅ **ARM Environment**: ARM Graviton instances with fusion snapshots -✅ **GPU Environment**: GPU + CPU instances with fusion snapshots - -## Environment IDs - -For reference, the current environment IDs are: - -- CPU: `53ljSqphNKjm6jjmuB6T9b` → `aws_ireland_fusionv2_nvme_cpu` -- ARM: `7eC1zALvNGIaFXbybVohP1` → `aws_ireland_fusionv2_nvme_cpu_ARM_snapshots` -- GPU: `2SRyFNKtLVAJCxMhcZRMfx` → `aws_ireland_fusionv2_nvme_gpu_snapshots` - -## Technical Details - -- **Cloud Provider**: AWS -- **Region**: eu-west-1 (Ireland) -- **Compute Backend**: AWS Batch -- **Container Technology**: Docker with Wave optimization -- **Storage**: S3 for work directory, NVMe for fast local storage -- **Networking**: Managed by Seqera Platform forge mode -- **Cost Optimization**: SPOT instances for all environments -- **Snapshots**: Enabled for optimized container layer caching using seqerakit's native `fusionSnapshots` field diff --git a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_arm_current.yml b/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_arm_current.yml deleted file mode 100644 index 2288cc70..00000000 --- a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_arm_current.yml +++ /dev/null @@ -1,7 +0,0 @@ -compute-envs: - - name: "aws_ireland_fusionv2_nvme_cpu_ARM_snapshots" - workspace: "$ORGANIZATION_NAME/$WORKSPACE_NAME" - credentials: "$AWS_CREDENTIALS_NAME" - wait: "AVAILABLE" - file-path: "./current-env-cpu-arm.json" - overwrite: True diff --git a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_current.yml b/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_current.yml deleted file mode 100644 index 8da31562..00000000 --- a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_current.yml +++ /dev/null @@ -1,7 +0,0 @@ -compute-envs: - - name: "aws_ireland_fusionv2_nvme_cpu_snapshots" - workspace: "$ORGANIZATION_NAME/$WORKSPACE_NAME" - credentials: "$AWS_CREDENTIALS_NAME" - wait: "AVAILABLE" - file-path: "./current-env-cpu.json" - overwrite: True diff --git a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_gpu_current.yml b/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_gpu_current.yml deleted file mode 100644 index e129b20e..00000000 --- a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_gpu_current.yml +++ /dev/null @@ -1,7 +0,0 @@ -compute-envs: - - name: "aws_ireland_fusionv2_nvme_gpu_snapshots" - workspace: "$ORGANIZATION_NAME/$WORKSPACE_NAME" - credentials: "$AWS_CREDENTIALS_NAME" - wait: "AVAILABLE" - file-path: "./current-env-gpu.json" - overwrite: True diff --git a/pulumi/AWSMegatests/seqerakit/current-env-cpu-arm.json b/pulumi/AWSMegatests/seqerakit/current-env-cpu-arm.json deleted file mode 100644 index 1134d92f..00000000 --- a/pulumi/AWSMegatests/seqerakit/current-env-cpu-arm.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "discriminator": "aws-batch", - "region": "eu-west-1", - "workDir": "s3://nf-core-awsmegatests", - "environment": [], - "waveEnabled": true, - "fusion2Enabled": true, - "nvnmeStorageEnabled": true, - "fusionSnapshots": true, - "forge": { - "type": "SPOT", - "minCpus": 0, - "maxCpus": 500, - "gpuEnabled": false, - "instanceTypes": ["m6gd", "r6gd", "c6gd"], - "subnets": [], - "securityGroups": [], - "disposeOnDeletion": true, - "allowBuckets": [ - "s3://ngi-igenomes", - "s3://nf-core-awsmegatests", - "s3://annotation-cache/" - ], - "efsCreate": false, - "ebsBootSize": 50, - "fargateHeadEnabled": true, - "arm64Enabled": true - }, - "labels": [] -} diff --git a/pulumi/AWSMegatests/seqerakit/current-env-cpu.json b/pulumi/AWSMegatests/seqerakit/current-env-cpu.json deleted file mode 100644 index a0390ed3..00000000 --- a/pulumi/AWSMegatests/seqerakit/current-env-cpu.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "discriminator": "aws-batch", - "region": "eu-west-1", - "workDir": "s3://nf-core-awsmegatests", - "environment": [], - "waveEnabled": true, - "fusion2Enabled": true, - "nvnmeStorageEnabled": true, - "fusionSnapshots": true, - "forge": { - "type": "SPOT", - "minCpus": 0, - "maxCpus": 1000, - "gpuEnabled": false, - "instanceTypes": ["c6id", "m6id", "r6id"], - "subnets": [], - "securityGroups": [], - "disposeOnDeletion": true, - "allowBuckets": [ - "s3://ngi-igenomes", - "s3://nf-core-awsmegatests", - "s3://annotation-cache/" - ], - "efsCreate": false, - "ebsBootSize": 50, - "fargateHeadEnabled": true, - "arm64Enabled": false - }, - "labels": [] -} diff --git a/pulumi/AWSMegatests/seqerakit/current-env-gpu.json b/pulumi/AWSMegatests/seqerakit/current-env-gpu.json deleted file mode 100644 index a4840f04..00000000 --- a/pulumi/AWSMegatests/seqerakit/current-env-gpu.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "discriminator": "aws-batch", - "region": "eu-west-1", - "workDir": "s3://nf-core-awsmegatests", - "environment": [], - "waveEnabled": true, - "fusion2Enabled": true, - "nvnmeStorageEnabled": true, - "fusionSnapshots": true, - "forge": { - "type": "SPOT", - "minCpus": 0, - "maxCpus": 500, - "gpuEnabled": true, - "instanceTypes": ["g4dn", "g5", "c6id", "m6id", "r6id"], - "subnets": [], - "securityGroups": [], - "disposeOnDeletion": true, - "allowBuckets": [ - "s3://ngi-igenomes", - "s3://nf-core-awsmegatests", - "s3://annotation-cache/" - ], - "efsCreate": false, - "dragenEnabled": false, - "ebsBootSize": 50, - "fargateHeadEnabled": true - }, - "labels": [] -} diff --git a/pulumi/AWSMegatests/src/__init__.py b/pulumi/AWSMegatests/src/__init__.py deleted file mode 100644 index b5e4da14..00000000 --- a/pulumi/AWSMegatests/src/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""AWS Megatests Infrastructure Package - -This package provides modular infrastructure components for the nf-core AWS Megatests -Pulumi project, including provider configurations, infrastructure resources, -and third-party integrations. -""" - -__version__ = "1.0.0" diff --git a/pulumi/AWSMegatests/src/config/__init__.py b/pulumi/AWSMegatests/src/config/__init__.py deleted file mode 100644 index 0d879069..00000000 --- a/pulumi/AWSMegatests/src/config/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Configuration management for AWS Megatests infrastructure.""" - -from .settings import get_configuration, ConfigurationError - -__all__ = ["get_configuration", "ConfigurationError"] diff --git a/pulumi/AWSMegatests/src/config/settings.py b/pulumi/AWSMegatests/src/config/settings.py deleted file mode 100644 index 335c5534..00000000 --- a/pulumi/AWSMegatests/src/config/settings.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Configuration management for AWS Megatests infrastructure using Pulumi ESC.""" - -import os -from typing import Dict, Any, Optional -from dataclasses import dataclass - -from ..utils.constants import DEFAULT_ENV_VARS - - -class ConfigurationError(Exception): - """Exception raised when configuration validation fails.""" - - pass - - -@dataclass -class InfrastructureConfig: - """Typed configuration for AWS Megatests infrastructure. - - Attributes: - tower_access_token: Seqera Platform access token - tower_workspace_id: Seqera Platform workspace ID - github_token: GitHub personal access token (classic) - platform_github_org_token: GitHub fine-grained token to avoid rate limits when pulling pipelines - """ - - tower_access_token: Optional[str] - tower_workspace_id: str - github_token: Optional[str] - platform_github_org_token: Optional[str] - - def validate(self) -> None: - """Validate configuration values. - - Raises: - ConfigurationError: If required configuration is missing or invalid - """ - missing_vars = [] - - if not self.tower_access_token: - missing_vars.append("TOWER_ACCESS_TOKEN") - - if not self.github_token: - missing_vars.append("GITHUB_TOKEN") - - # Validate workspace ID is numeric - if ( - not self.tower_workspace_id - or not self.tower_workspace_id.replace(".", "").isdigit() - ): - missing_vars.append("TOWER_WORKSPACE_ID (must be numeric)") - - if missing_vars: - raise ConfigurationError( - f"Missing or invalid required environment variables: {', '.join(missing_vars)}. " - "Please ensure these are set in your ESC environment." - ) - - -def _get_env_var_with_fallback( - var_name: str, fallback: Optional[str] = None -) -> Optional[str]: - """Get environment variable with optional fallback. - - Args: - var_name: Name of the environment variable - fallback: Optional fallback value if variable is not set - - Returns: - Optional[str]: Environment variable value or fallback - """ - value = os.environ.get(var_name) - if not value and fallback: - print( - f"Warning: {var_name} not found in ESC environment, using fallback: {fallback}" - ) - return fallback - return value - - -def get_configuration() -> Dict[str, Any]: - """Get configuration values from ESC environment variables. - - All configuration comes from ESC environment variables which are automatically - set when the ESC environment is imported. - - Returns: - Dict[str, Any]: Configuration dictionary compatible with existing code - - Raises: - ConfigurationError: If required configuration is missing or invalid - """ - # Get workspace ID from environment or fall back to default - workspace_id = _get_env_var_with_fallback( - "TOWER_WORKSPACE_ID", DEFAULT_ENV_VARS.get("TOWER_WORKSPACE_ID") - ) - - # Create typed configuration object - config = InfrastructureConfig( - tower_access_token=os.environ.get("TOWER_ACCESS_TOKEN"), - tower_workspace_id=workspace_id or "", - github_token=os.environ.get("GITHUB_TOKEN"), - platform_github_org_token=os.environ.get("PLATFORM_GITHUB_ORG_TOKEN"), - ) - - # Validate configuration - config.validate() - - # Return dictionary format for backward compatibility - return { - "tower_access_token": config.tower_access_token, - "tower_workspace_id": config.tower_workspace_id, - "github_token": config.github_token, - "platform_github_org_token": config.platform_github_org_token, - # AWS credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN) - # are automatically handled by ESC and picked up by the AWS provider - } diff --git a/pulumi/AWSMegatests/src/infrastructure/__init__.py b/pulumi/AWSMegatests/src/infrastructure/__init__.py deleted file mode 100644 index f342fd1e..00000000 --- a/pulumi/AWSMegatests/src/infrastructure/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Infrastructure components for AWS Megatests.""" - -from .s3 import create_s3_infrastructure -from .credentials import create_towerforge_credentials, get_towerforge_resources -from .compute_environments import ( - deploy_seqera_environments_terraform, - get_compute_environment_ids_terraform, -) - -__all__ = [ - "create_s3_infrastructure", - "create_towerforge_credentials", - "get_towerforge_resources", - "deploy_seqera_environments_terraform", - "get_compute_environment_ids_terraform", -] diff --git a/pulumi/AWSMegatests/src/infrastructure/compute_environments.py b/pulumi/AWSMegatests/src/infrastructure/compute_environments.py deleted file mode 100644 index ff254467..00000000 --- a/pulumi/AWSMegatests/src/infrastructure/compute_environments.py +++ /dev/null @@ -1,387 +0,0 @@ -"""Seqera Platform compute environment deployment using Seqera Terraform provider.""" - -import json -import os -from typing import Dict, Any, Optional - -import pulumi -import pulumi_seqera as seqera - -from ..utils.constants import ( - COMPUTE_ENV_NAMES, - COMPUTE_ENV_DESCRIPTIONS, - CONFIG_FILES, - NEXTFLOW_CONFIG_FILES, - DEFAULT_COMPUTE_ENV_CONFIG, - DEFAULT_FORGE_CONFIG, - TIMEOUTS, - ERROR_MESSAGES, -) - - -class ComputeEnvironmentError(Exception): - """Exception raised when compute environment operations fail.""" - - pass - - -class ConfigurationError(Exception): - """Exception raised when configuration loading fails.""" - - pass - - -def load_nextflow_config(env_type: str) -> str: - """Load and merge Nextflow configuration from base and environment-specific files. - - Args: - env_type: Environment type (cpu, gpu, arm) - - Returns: - str: Merged Nextflow configuration content - - Raises: - ConfigurationError: If file loading fails - """ - config_file = NEXTFLOW_CONFIG_FILES.get(env_type) - if not config_file: - raise ConfigurationError( - f"No Nextflow config file defined for environment type: {env_type}" - ) - - if not os.path.exists(config_file): - raise FileNotFoundError(f"Nextflow config file not found: {config_file}") - - # Load base configuration - base_config_file = os.path.join( - os.path.dirname(config_file), "nextflow-base.config" - ) - base_config = "" - if os.path.exists(base_config_file): - try: - with open(base_config_file, "r") as f: - base_config = f.read().strip() - except Exception as e: - raise ConfigurationError( - f"Failed to read base Nextflow config file {base_config_file}: {e}" - ) - - # Load environment-specific configuration - try: - with open(config_file, "r") as f: - env_config = f.read().strip() - except Exception as e: - raise ConfigurationError( - f"Failed to read Nextflow config file {config_file}: {e}" - ) - - # Remove includeConfig line from environment config since we're injecting base config - env_config_lines = env_config.split("\n") - env_config_filtered = [ - line - for line in env_config_lines - if not line.strip().startswith("includeConfig") - ] - env_config_clean = "\n".join(env_config_filtered) - - # Merge base config with environment-specific config - if base_config: - merged_config = f"{base_config}\n\n{env_config_clean}" - else: - merged_config = env_config_clean - - return merged_config.strip() - - -def load_config_file(filename: str) -> Dict[str, Any]: - """Load configuration file with comprehensive error handling. - - Args: - filename: Path to the JSON configuration file - - Returns: - Dict[str, Any]: Loaded configuration data - - Raises: - ConfigurationError: If file loading or parsing fails - """ - if not os.path.exists(filename): - raise FileNotFoundError( - ERROR_MESSAGES["config_file_not_found"].format(filename) - ) - - with open(filename, "r") as f: - config_data = json.load(f) - - return config_data - - -def create_forge_config( - config_args: Dict[str, Any], -) -> seqera.ComputeEnvComputeEnvConfigAwsBatchForgeArgs: - """Create forge configuration for AWS Batch compute environment. - - Args: - config_args: Configuration arguments from JSON file - - Returns: - seqera.ComputeEnvComputeEnvConfigAwsBatchForgeArgs: Forge configuration - """ - forge_data = config_args.get("forge", {}) - - return seqera.ComputeEnvComputeEnvConfigAwsBatchForgeArgs( - type=forge_data.get("type", DEFAULT_FORGE_CONFIG["type"]), - min_cpus=forge_data.get("minCpus", DEFAULT_FORGE_CONFIG["minCpus"]), - max_cpus=forge_data.get("maxCpus", DEFAULT_FORGE_CONFIG["maxCpus"]), - gpu_enabled=forge_data.get("gpuEnabled", DEFAULT_FORGE_CONFIG["gpuEnabled"]), - instance_types=forge_data.get( - "instanceTypes", DEFAULT_FORGE_CONFIG["instanceTypes"] - ), - subnets=forge_data.get("subnets", DEFAULT_FORGE_CONFIG["subnets"]), - security_groups=forge_data.get( - "securityGroups", DEFAULT_FORGE_CONFIG["securityGroups"] - ), - dispose_on_deletion=forge_data.get( - "disposeOnDeletion", DEFAULT_FORGE_CONFIG["disposeOnDeletion"] - ), - allow_buckets=forge_data.get( - "allowBuckets", DEFAULT_FORGE_CONFIG["allowBuckets"] - ), - efs_create=forge_data.get("efsCreate", DEFAULT_FORGE_CONFIG["efsCreate"]), - ebs_boot_size=forge_data.get( - "ebsBootSize", DEFAULT_FORGE_CONFIG["ebsBootSize"] - ), - fargate_head_enabled=forge_data.get( - "fargateHeadEnabled", DEFAULT_FORGE_CONFIG["fargateHeadEnabled"] - ), - arm64_enabled=forge_data.get( - "arm64Enabled", DEFAULT_FORGE_CONFIG["arm64Enabled"] - ), - ) - - -def create_compute_environment( - provider: seqera.Provider, - name: str, - credentials_id: str, - workspace_id: float, - config_args: Dict[str, Any], - env_type: str, - description: Optional[str] = None, - depends_on: Optional[list] = None, - iam_policy_version: Optional[str] = None, -) -> seqera.ComputeEnv: - """Create a Seqera compute environment using Terraform provider with error handling. - - Args: - provider: Configured Seqera provider instance - name: Name for the compute environment - credentials_id: Seqera credentials ID - workspace_id: Seqera workspace ID - config_args: Configuration arguments from JSON file - env_type: Environment type (cpu, gpu, arm) for loading external nextflow config - description: Optional description for the compute environment - depends_on: Optional list of resources this compute environment depends on - iam_policy_version: Optional IAM policy version hash to trigger recreation on policy changes - - Returns: - seqera.ComputeEnv: Created compute environment resource - - Raises: - ComputeEnvironmentError: If compute environment creation fails - ValueError: If required parameters are missing - ConfigurationError: If nextflow config loading fails - """ - pulumi.log.info(f"Creating compute environment: {name}") - - # Validate input parameters - if not name or not credentials_id: - raise ValueError(ERROR_MESSAGES["missing_compute_env_params"].format(name)) - - if not config_args: - raise ValueError(ERROR_MESSAGES["missing_config_args"].format(name)) - - # Create the forge configuration - forge_config = create_forge_config(config_args) - - # Load Nextflow configuration from external file - nextflow_config = load_nextflow_config(env_type) - - # Create AWS Batch configuration - aws_batch_config = seqera.ComputeEnvComputeEnvConfigAwsBatchArgs( - region=config_args.get("region", DEFAULT_COMPUTE_ENV_CONFIG["region"]), - work_dir=config_args.get("workDir", DEFAULT_COMPUTE_ENV_CONFIG["workDir"]), - forge=forge_config, - wave_enabled=config_args.get( - "waveEnabled", DEFAULT_COMPUTE_ENV_CONFIG["waveEnabled"] - ), - fusion2_enabled=config_args.get( - "fusion2Enabled", DEFAULT_COMPUTE_ENV_CONFIG["fusion2Enabled"] - ), - nvnme_storage_enabled=config_args.get( - "nvnmeStorageEnabled", DEFAULT_COMPUTE_ENV_CONFIG["nvnmeStorageEnabled"] - ), - fusion_snapshots=config_args.get( - "fusionSnapshots", DEFAULT_COMPUTE_ENV_CONFIG["fusionSnapshots"] - ), - nextflow_config=nextflow_config, # Use external config file - ) - - # Create the compute environment configuration - compute_env_config = seqera.ComputeEnvComputeEnvConfigArgs( - aws_batch=aws_batch_config - ) - - # Create the compute environment args - compute_env_args = seqera.ComputeEnvComputeEnvArgs( - name=name, - platform="aws-batch", - credentials_id=credentials_id, - config=compute_env_config, - description=description, - ) - - # Add IAM policy version to compute environment description to trigger recreation on policy changes - if iam_policy_version: - # Append policy version hash to description to force recreation when IAM policies change - policy_suffix = f" (IAM Policy Version: {iam_policy_version[:8]})" - if description: - compute_env_args = seqera.ComputeEnvComputeEnvArgs( - name=name, - platform="aws-batch", - credentials_id=credentials_id, - config=compute_env_config, - description=f"{description}{policy_suffix}", - ) - else: - compute_env_args = seqera.ComputeEnvComputeEnvArgs( - name=name, - platform="aws-batch", - credentials_id=credentials_id, - config=compute_env_config, - description=f"Compute environment{policy_suffix}", - ) - - # Create the compute environment resource - resource_options = pulumi.ResourceOptions( - provider=provider, - # Force delete before replace to avoid name conflicts - delete_before_replace=True, - # Add custom timeout for compute environment creation - custom_timeouts=pulumi.CustomTimeouts( - create=TIMEOUTS["compute_env_create"], - update=TIMEOUTS["compute_env_update"], - delete=TIMEOUTS["compute_env_delete"], - ), - ) - - # Add dependencies if specified - if depends_on: - resource_options.depends_on = depends_on - - compute_env = seqera.ComputeEnv( - name, - compute_env=compute_env_args, - workspace_id=workspace_id, - opts=resource_options, - ) - - return compute_env - - -def deploy_seqera_environments_terraform( - config: Dict[str, Any], - towerforge_credentials_id: str, - seqera_provider: Optional[seqera.Provider] = None, - seqera_credential_resource: Optional[seqera.Credential] = None, - iam_policy_hash: Optional[str] = None, -) -> Dict[str, Any]: - """Deploy Seqera Platform compute environments using Terraform provider. - - Args: - config: Configuration dictionary - towerforge_credentials_id: Dynamic TowerForge credentials ID - seqera_provider: Optional existing Seqera provider instance - seqera_credential_resource: Optional Seqera credential resource for dependency - iam_policy_hash: Optional IAM policy hash to force recreation on policy changes - - Returns: - Dict[str, Any]: Dictionary containing created compute environments and provider - - Raises: - ConfigurationError: If configuration loading fails - ComputeEnvironmentError: If compute environment creation fails - ValueError: If workspace ID is invalid - """ - pulumi.log.info( - "Starting Seqera compute environment deployment using Terraform provider" - ) - - # Use provided seqera provider or create a new one - if seqera_provider is not None: - provider = seqera_provider - pulumi.log.info("Using existing Seqera provider") - else: - # Import here to avoid circular imports - from ..providers.seqera import create_seqera_provider - - provider = create_seqera_provider(config) - - # Load all configuration files - cpu_config = load_config_file(CONFIG_FILES["cpu"]) - gpu_config = load_config_file(CONFIG_FILES["gpu"]) - arm_config = load_config_file(CONFIG_FILES["arm"]) - - # Validate workspace ID - workspace_id = float(config["tower_workspace_id"]) - - # Create all three compute environments - environments = {} - - # Set up dependencies - compute environments depend on Seqera credential resource - depends_on_resources = [] - if seqera_credential_resource: - depends_on_resources.append(seqera_credential_resource) - - for env_type, config_data in [ - ("cpu", cpu_config), - ("gpu", gpu_config), - ("arm", arm_config), - ]: - env_name = COMPUTE_ENV_NAMES[env_type] - description = COMPUTE_ENV_DESCRIPTIONS[env_type] - - environments[f"{env_type}_env"] = create_compute_environment( - provider=provider, - name=env_name, - credentials_id=towerforge_credentials_id, - workspace_id=workspace_id, - config_args=config_data, - env_type=env_type, - description=description, - depends_on=depends_on_resources if depends_on_resources else None, - iam_policy_version=iam_policy_hash, - ) - - return { - **environments, - "provider": provider, - } - - -def get_compute_environment_ids_terraform( - terraform_resources: Dict[str, Any], -) -> Dict[str, Any]: - """Extract compute environment IDs from Terraform provider resources. - - Args: - terraform_resources: Dictionary containing terraform resources - - Returns: - Dict[str, Any]: Dictionary mapping environment types to their IDs - """ - return { - "cpu": terraform_resources["cpu_env"].compute_env_id, - "gpu": terraform_resources["gpu_env"].compute_env_id, - "arm": terraform_resources["arm_env"].compute_env_id, - } diff --git a/pulumi/AWSMegatests/src/infrastructure/credentials.py b/pulumi/AWSMegatests/src/infrastructure/credentials.py deleted file mode 100644 index b766df61..00000000 --- a/pulumi/AWSMegatests/src/infrastructure/credentials.py +++ /dev/null @@ -1,496 +0,0 @@ -"""TowerForge IAM credentials management module. - -This module creates and manages IAM resources for TowerForge AWS operations, -including policies for Forge operations, Launch operations, and S3 bucket access. -It also uploads the credentials to Seqera Platform for use by compute environments. -""" - -import json -import hashlib -from typing import Optional, Tuple, Dict, Any - -import pulumi -import pulumi_aws as aws -import pulumi_seqera as seqera - -from ..utils.constants import ( - TOWERFORGE_USER_NAME, - TOWERFORGE_POLICY_NAMES, - TOWERFORGE_CREDENTIAL_NAME, - TOWERFORGE_CREDENTIAL_DESCRIPTION, - TIMEOUTS, -) - - -class CredentialError(Exception): - """Exception raised when credential operations fail.""" - - pass - - -def _create_forge_policy_document() -> Dict[str, Any]: - """Create TowerForge Forge Policy document with comprehensive permissions.""" - return { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "TowerForge0", - "Effect": "Allow", - "Action": [ - "ssm:GetParameters", - "iam:CreateInstanceProfile", - "iam:DeleteInstanceProfile", - "iam:AddRoleToInstanceProfile", - "iam:RemoveRoleFromInstanceProfile", - "iam:CreateRole", - "iam:DeleteRole", - "iam:AttachRolePolicy", - "iam:DetachRolePolicy", - "iam:PutRolePolicy", - "iam:DeleteRolePolicy", - "iam:PassRole", - "iam:TagRole", - "iam:TagInstanceProfile", - "iam:ListRolePolicies", - "iam:ListAttachedRolePolicies", - "iam:GetRole", - "batch:CreateComputeEnvironment", - "batch:UpdateComputeEnvironment", - "batch:DeleteComputeEnvironment", - "batch:CreateJobQueue", - "batch:UpdateJobQueue", - "batch:DeleteJobQueue", - "batch:DescribeComputeEnvironments", - "batch:DescribeJobQueues", - "fsx:CreateFileSystem", - "fsx:DeleteFileSystem", - "fsx:DescribeFileSystems", - "fsx:TagResource", - "ec2:DescribeSecurityGroups", - "ec2:DescribeAccountAttributes", - "ec2:DescribeSubnets", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeLaunchTemplateVersions", - "ec2:CreateLaunchTemplate", - "ec2:DeleteLaunchTemplate", - "ec2:DescribeKeyPairs", - "ec2:DescribeVpcs", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstanceTypeOfferings", - "ec2:GetEbsEncryptionByDefault", - "efs:CreateFileSystem", - "efs:DeleteFileSystem", - "efs:DescribeFileSystems", - "efs:CreateMountTarget", - "efs:DeleteMountTarget", - "efs:DescribeMountTargets", - "efs:ModifyFileSystem", - "efs:PutLifecycleConfiguration", - "efs:TagResource", - "elasticfilesystem:CreateFileSystem", - "elasticfilesystem:DeleteFileSystem", - "elasticfilesystem:DescribeFileSystems", - "elasticfilesystem:CreateMountTarget", - "elasticfilesystem:DeleteMountTarget", - "elasticfilesystem:DescribeMountTargets", - "elasticfilesystem:UpdateFileSystem", - "elasticfilesystem:PutLifecycleConfiguration", - "elasticfilesystem:TagResource", - ], - "Resource": "*", - }, - { - "Sid": "TowerLaunch0", - "Effect": "Allow", - "Action": [ - "s3:Get*", - "s3:List*", - "batch:DescribeJobQueues", - "batch:CancelJob", - "batch:SubmitJob", - "batch:ListJobs", - "batch:TagResource", - "batch:DescribeComputeEnvironments", - "batch:TerminateJob", - "batch:DescribeJobs", - "batch:RegisterJobDefinition", - "batch:DescribeJobDefinitions", - "ecs:DescribeTasks", - "ec2:DescribeInstances", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstanceAttribute", - "ecs:DescribeContainerInstances", - "ec2:DescribeInstanceStatus", - "ec2:DescribeImages", - "logs:Describe*", - "logs:Get*", - "logs:List*", - "logs:StartQuery", - "logs:StopQuery", - "logs:TestMetricFilter", - "logs:FilterLogEvents", - "ses:SendRawEmail", - "secretsmanager:ListSecrets", - ], - "Resource": "*", - }, - ], - } - - -def _create_launch_policy_document() -> Dict[str, Any]: - """Create TowerForge Launch Policy document with limited permissions.""" - return { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "TowerLaunch0", - "Effect": "Allow", - "Action": [ - "batch:DescribeJobQueues", - "batch:CancelJob", - "batch:SubmitJob", - "batch:ListJobs", - "batch:TagResource", - "batch:DescribeComputeEnvironments", - "batch:TerminateJob", - "batch:DescribeJobs", - "batch:RegisterJobDefinition", - "batch:DescribeJobDefinitions", - "ecs:DescribeTasks", - "ec2:DescribeInstances", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstanceAttribute", - "ecs:DescribeContainerInstances", - "ec2:DescribeInstanceStatus", - "ec2:DescribeImages", - "logs:Describe*", - "logs:Get*", - "logs:List*", - "logs:StartQuery", - "logs:StopQuery", - "logs:TestMetricFilter", - "logs:FilterLogEvents", - "ses:SendRawEmail", - "secretsmanager:ListSecrets", - ], - "Resource": "*", - } - ], - } - - -def _create_s3_policy_document(bucket_arn: str) -> Dict[str, Any]: - """Create S3 bucket access policy document. - - Args: - bucket_arn: ARN of the S3 bucket to grant access to - - Returns: - Dict[str, Any]: S3 policy document - """ - return { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:ListBucket"], - "Resource": [bucket_arn], - }, - { - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:PutObjectTagging", - "s3:DeleteObject", - ], - "Resource": [f"{bucket_arn}/*"], - "Effect": "Allow", - }, - ], - } - - -def create_seqera_credentials( - seqera_provider: seqera.Provider, - workspace_id: float, - access_key_id: pulumi.Output[str], - access_key_secret: pulumi.Output[str], -) -> seqera.Credential: - """Upload TowerForge AWS credentials to Seqera Platform. - - Args: - seqera_provider: Configured Seqera provider instance - workspace_id: Seqera Platform workspace ID - access_key_id: AWS access key ID from TowerForge IAM user - access_key_secret: AWS secret access key from TowerForge IAM user - - Returns: - seqera.Credential: Seqera credential resource with credentials_id - - Raises: - CredentialError: If credential upload fails - """ - pulumi.log.info("Uploading TowerForge credentials to Seqera Platform") - - # Create AWS credentials configuration for Seqera Platform - aws_keys = seqera.CredentialKeysArgs( - aws=seqera.CredentialKeysAwsArgs( - access_key=access_key_id, - secret_key=access_key_secret, - # Note: assume_role_arn is optional and not needed for direct IAM user credentials - ) - ) - - # Upload credentials to Seqera Platform - seqera_credential = seqera.Credential( - "towerforge-aws-credential", - name=TOWERFORGE_CREDENTIAL_NAME, - description=TOWERFORGE_CREDENTIAL_DESCRIPTION, - provider_type="aws", - workspace_id=workspace_id, - keys=aws_keys, - opts=pulumi.ResourceOptions( - provider=seqera_provider, - # Ensure credentials are uploaded after IAM access key is created - custom_timeouts=pulumi.CustomTimeouts( - create=TIMEOUTS["seqera_credential_create"], - update=TIMEOUTS["seqera_credential_update"], - delete=TIMEOUTS["seqera_credential_delete"], - ), - ), - ) - - return seqera_credential - - -def _create_iam_policies( - aws_provider: aws.Provider, s3_bucket -) -> Tuple[aws.iam.Policy, aws.iam.Policy, aws.iam.Policy]: - """Create IAM policies for TowerForge operations. - - Args: - aws_provider: Configured AWS provider instance - s3_bucket: S3 bucket resource for policy attachment - - Returns: - Tuple of (forge_policy, launch_policy, s3_policy) - """ - # TowerForge Forge Policy - Comprehensive permissions for resource creation - forge_policy = aws.iam.Policy( - "towerforge-forge-policy", - name=TOWERFORGE_POLICY_NAMES["forge"], - description="IAM policy for TowerForge to create and manage AWS Batch resources", - policy=json.dumps(_create_forge_policy_document()), - opts=pulumi.ResourceOptions(provider=aws_provider), - ) - - # TowerForge Launch Policy - Limited permissions for pipeline execution - launch_policy = aws.iam.Policy( - "towerforge-launch-policy", - name=TOWERFORGE_POLICY_NAMES["launch"], - description="IAM policy for TowerForge to launch and monitor pipeline executions", - policy=json.dumps(_create_launch_policy_document()), - opts=pulumi.ResourceOptions(provider=aws_provider), - ) - - # TowerForge S3 Bucket Access Policy - Access to specified S3 bucket - s3_policy = aws.iam.Policy( - "towerforge-s3-policy", - name=TOWERFORGE_POLICY_NAMES["s3"], - description=s3_bucket.bucket.apply( - lambda bucket_name: f"IAM policy for TowerForge to access {bucket_name} S3 bucket" - ), - policy=s3_bucket.arn.apply( - lambda arn: json.dumps(_create_s3_policy_document(arn)) - ), - opts=pulumi.ResourceOptions(provider=aws_provider, depends_on=[s3_bucket]), - ) - - return forge_policy, launch_policy, s3_policy - - -def _generate_policy_hash( - forge_policy: aws.iam.Policy, - launch_policy: aws.iam.Policy, - s3_policy: aws.iam.Policy, -) -> str: - """Generate a hash of IAM policies to detect changes. - - Args: - forge_policy: TowerForge Forge policy - launch_policy: TowerForge Launch policy - s3_policy: TowerForge S3 policy - - Returns: - str: SHA256 hash of the combined policy documents - """ - # Create a deterministic hash of all policy documents - forge_doc = _create_forge_policy_document() - launch_doc = _create_launch_policy_document() - - # Combine all policy documents for hashing - combined_policies = json.dumps( - { - "forge": forge_doc, - "launch": launch_doc, - # Note: S3 policy is bucket-specific, so we'll use a placeholder for consistent hashing - "s3": {"bucket_dependent": True}, - }, - sort_keys=True, - ) - - return hashlib.sha256(combined_policies.encode()).hexdigest() - - -def create_towerforge_credentials( - aws_provider: aws.Provider, - s3_bucket, - seqera_provider: seqera.Provider, - workspace_id: float, -) -> Tuple[ - pulumi.Output[str], pulumi.Output[str], pulumi.Output[str], seqera.Credential, str -]: - """Create TowerForge IAM resources and upload to Seqera Platform. - - Creates IAM policies, user, and access keys for TowerForge operations, - then uploads the credentials to Seqera Platform for use by compute environments. - Based on https://github.com/seqeralabs/nf-tower-aws - - Args: - aws_provider: Configured AWS provider instance - s3_bucket: S3 bucket resource for policy attachment - seqera_provider: Configured Seqera provider instance - workspace_id: Seqera Platform workspace ID - - Returns: - Tuple: (access_key_id, access_key_secret, seqera_credentials_id, seqera_credential_resource, iam_policy_hash) - """ - # Create IAM policies - forge_policy, launch_policy, s3_policy = _create_iam_policies( - aws_provider, s3_bucket - ) - - # Generate policy version hash for compute environment recreation on policy changes - iam_policy_hash = _generate_policy_hash(forge_policy, launch_policy, s3_policy) - - # Create TowerForge IAM User - towerforge_user = aws.iam.User( - "towerforge-user", - name=TOWERFORGE_USER_NAME, - opts=pulumi.ResourceOptions(provider=aws_provider), - ) - - # Attach policies to the TowerForge user - forge_attachment = aws.iam.UserPolicyAttachment( - "towerforge-forge-policy-attachment", - user=towerforge_user.name, - policy_arn=forge_policy.arn, - opts=pulumi.ResourceOptions( - provider=aws_provider, depends_on=[towerforge_user, forge_policy] - ), - ) - - launch_attachment = aws.iam.UserPolicyAttachment( - "towerforge-launch-policy-attachment", - user=towerforge_user.name, - policy_arn=launch_policy.arn, - opts=pulumi.ResourceOptions( - provider=aws_provider, - depends_on=[towerforge_user, launch_policy], - ), - ) - - s3_attachment = aws.iam.UserPolicyAttachment( - "towerforge-s3-policy-attachment", - user=towerforge_user.name, - policy_arn=s3_policy.arn, - opts=pulumi.ResourceOptions( - provider=aws_provider, depends_on=[towerforge_user, s3_policy] - ), - ) - - # Create access keys for the TowerForge user - towerforge_access_key = aws.iam.AccessKey( - "towerforge-access-key", - user=towerforge_user.name, - opts=pulumi.ResourceOptions( - provider=aws_provider, - depends_on=[forge_attachment, launch_attachment, s3_attachment], - additional_secret_outputs=["secret"], - ), - ) - - # Upload the credentials to Seqera Platform - seqera_credential = create_seqera_credentials( - seqera_provider=seqera_provider, - workspace_id=workspace_id, - access_key_id=towerforge_access_key.id, - access_key_secret=towerforge_access_key.secret, - ) - - # Return the access key credentials, Seqera credentials ID, credential resource, and policy hash - return ( - towerforge_access_key.id, - towerforge_access_key.secret, - seqera_credential.credentials_id, - seqera_credential, - iam_policy_hash, - ) - - -def get_towerforge_resources( - aws_provider: aws.Provider, - s3_bucket, - seqera_provider: Optional[seqera.Provider] = None, - workspace_id: Optional[float] = None, -) -> Dict[str, Any]: - """Create TowerForge resources and return resource information for exports. - - This function creates the TowerForge IAM resources and returns a dictionary - containing resource information for Pulumi exports. - - Args: - aws_provider: Configured AWS provider instance - s3_bucket: S3 bucket resource for policy attachment - seqera_provider: Optional Seqera provider for credential upload - workspace_id: Optional workspace ID for Seqera Platform - - Returns: - Dict[str, Any]: Resource information for Pulumi exports - - Raises: - ValueError: If required parameters are missing - """ - # Create the credentials (this will create all the resources) - if seqera_provider and workspace_id: - ( - access_key_id, - access_key_secret, - seqera_credentials_id, - seqera_credential, - iam_policy_hash, - ) = create_towerforge_credentials( - aws_provider, s3_bucket, seqera_provider, workspace_id - ) - else: - # Fallback for backward compatibility - this will raise an error since signature changed - raise ValueError( - "get_towerforge_resources now requires seqera_provider and workspace_id parameters. " - "Please update your code to use the new signature or call create_towerforge_credentials directly." - ) - - return { - "user": { - "name": TOWERFORGE_USER_NAME, - "arn": f"arn:aws:iam::{{aws_account_id}}:user/{TOWERFORGE_USER_NAME}", # Will be populated by Pulumi - }, - "access_key_id": access_key_id, - "access_key_secret": access_key_secret, - "seqera_credentials_id": seqera_credentials_id, - "policies": { - "forge_policy_name": TOWERFORGE_POLICY_NAMES["forge"], - "launch_policy_name": TOWERFORGE_POLICY_NAMES["launch"], - "s3_policy_name": TOWERFORGE_POLICY_NAMES["s3"], - }, - } diff --git a/pulumi/AWSMegatests/src/infrastructure/s3.py b/pulumi/AWSMegatests/src/infrastructure/s3.py deleted file mode 100644 index 70afda88..00000000 --- a/pulumi/AWSMegatests/src/infrastructure/s3.py +++ /dev/null @@ -1,189 +0,0 @@ -"""S3 infrastructure management for AWS Megatests.""" - -from typing import Dict, Any - -import pulumi -from pulumi_aws import s3 - -from ..utils.constants import S3_BUCKET_NAME - - -def create_s3_infrastructure(aws_provider) -> Dict[str, Any]: - """Create S3 bucket and lifecycle configuration. - - Args: - aws_provider: Configured AWS provider instance - - Returns: - Dict[str, Any]: Dictionary containing bucket and lifecycle configuration - """ - # Import existing AWS resources used by nf-core megatests - # S3 bucket for Nextflow work directory (already exists) - nf_core_awsmegatests_bucket = s3.Bucket( - "nf-core-awsmegatests", - bucket=S3_BUCKET_NAME, - opts=pulumi.ResourceOptions( - import_=S3_BUCKET_NAME, # Import existing bucket - protect=True, # Protect from accidental deletion - provider=aws_provider, # Use configured AWS provider - ignore_changes=[ - "lifecycle_rules", - "versioning", - ], # Don't modify existing configurations - managed manually due to permission constraints - ), - ) - - # S3 bucket lifecycle configuration - # Create lifecycle rules for automated cost optimization and cleanup - bucket_lifecycle_configuration = create_s3_lifecycle_configuration( - aws_provider, nf_core_awsmegatests_bucket - ) - - # S3 bucket CORS configuration for Seqera Data Explorer compatibility - bucket_cors_configuration = create_s3_cors_configuration( - aws_provider, nf_core_awsmegatests_bucket - ) - - return { - "bucket": nf_core_awsmegatests_bucket, - "lifecycle_configuration": bucket_lifecycle_configuration, - "cors_configuration": bucket_cors_configuration, - } - - -def create_s3_lifecycle_configuration(aws_provider, bucket): - """Create S3 lifecycle configuration with proper rules for Nextflow workflows. - - Args: - aws_provider: Configured AWS provider instance - bucket: S3 bucket resource - - Returns: - S3 bucket lifecycle configuration resource - """ - # S3 bucket lifecycle configuration for cost optimization and cleanup - # Rules designed specifically for Nextflow workflow patterns - lifecycle_configuration = s3.BucketLifecycleConfigurationV2( - "nf-core-awsmegatests-lifecycle", - bucket=bucket.id, - rules=[ - # Rule 1: Preserve metadata files with cost optimization - s3.BucketLifecycleConfigurationV2RuleArgs( - id="preserve-metadata-files", - status="Enabled", - filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( - tag=s3.BucketLifecycleConfigurationV2RuleFilterTagArgs( - key="nextflow.io/metadata", value="true" - ) - ), - transitions=[ - s3.BucketLifecycleConfigurationV2RuleTransitionArgs( - days=30, storage_class="STANDARD_IA" - ), - s3.BucketLifecycleConfigurationV2RuleTransitionArgs( - days=90, storage_class="GLACIER" - ), - ], - ), - # Rule 2: Clean up temporary files after 30 days - s3.BucketLifecycleConfigurationV2RuleArgs( - id="cleanup-temporary-files", - status="Enabled", - filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( - tag=s3.BucketLifecycleConfigurationV2RuleFilterTagArgs( - key="nextflow.io/temporary", value="true" - ) - ), - expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), - ), - # Rule 3: Clean up work directory after 30 days - s3.BucketLifecycleConfigurationV2RuleArgs( - id="cleanup-work-directory", - status="Enabled", - filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs(prefix="work/"), - expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), - ), - # Rule 4: Clean up scratch directory after 30 days - s3.BucketLifecycleConfigurationV2RuleArgs( - id="cleanup-scratch-directory", - status="Enabled", - filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( - prefix="scratch/" - ), - expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), - ), - # Rule 5: Clean up cache directories after 30 days - s3.BucketLifecycleConfigurationV2RuleArgs( - id="cleanup-cache-directories", - status="Enabled", - filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs(prefix="cache/"), - expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), - ), - # Rule 6: Clean up .cache directories after 30 days - s3.BucketLifecycleConfigurationV2RuleArgs( - id="cleanup-dot-cache-directories", - status="Enabled", - filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( - prefix=".cache/" - ), - expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), - ), - # Rule 7: Clean up incomplete multipart uploads - s3.BucketLifecycleConfigurationV2RuleArgs( - id="cleanup-incomplete-multipart-uploads", - status="Enabled", - abort_incomplete_multipart_upload=s3.BucketLifecycleConfigurationV2RuleAbortIncompleteMultipartUploadArgs( - days_after_initiation=7 - ), - ), - ], - opts=pulumi.ResourceOptions(provider=aws_provider, depends_on=[bucket]), - ) - - return lifecycle_configuration - - -def create_s3_cors_configuration(aws_provider, bucket): - """Create S3 CORS configuration for Seqera Data Explorer compatibility. - - Args: - aws_provider: Configured AWS provider instance - bucket: S3 bucket resource - - Returns: - S3 bucket CORS configuration resource - """ - # S3 CORS configuration for Seqera Data Explorer compatibility - # Based on official Seqera documentation: - # https://docs.seqera.io/platform-cloud/data/data-explorer#amazon-s3-cors-configuration - cors_configuration = s3.BucketCorsConfigurationV2( - "nf-core-awsmegatests-cors", - bucket=bucket.id, - cors_rules=[ - s3.BucketCorsConfigurationV2CorsRuleArgs( - id="SeqeraDataExplorerAccess", - allowed_headers=["*"], - allowed_methods=["GET", "HEAD", "POST", "PUT", "DELETE"], - allowed_origins=[ - "https://*.cloud.seqera.io", - "https://*.tower.nf", - "https://cloud.seqera.io", - "https://tower.nf", - ], - expose_headers=["ETag"], - max_age_seconds=3000, - ), - # Additional rule for direct browser access - s3.BucketCorsConfigurationV2CorsRuleArgs( - id="BrowserDirectAccess", - allowed_headers=["Authorization", "Content-Type", "Range"], - allowed_methods=["GET", "HEAD"], - allowed_origins=["*"], - expose_headers=["Content-Range", "Content-Length", "ETag"], - max_age_seconds=3000, - ), - ], - opts=pulumi.ResourceOptions(provider=aws_provider, depends_on=[bucket]), - ) - - return cors_configuration diff --git a/pulumi/AWSMegatests/src/integrations/__init__.py b/pulumi/AWSMegatests/src/integrations/__init__.py deleted file mode 100644 index ba069edd..00000000 --- a/pulumi/AWSMegatests/src/integrations/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Third-party integrations for AWS Megatests.""" - -from .github import create_github_resources -from .github_credentials import create_github_credential, get_github_credential_config - -__all__ = [ - "create_github_resources", - "create_github_credential", - "get_github_credential_config", -] diff --git a/pulumi/AWSMegatests/src/integrations/github.py b/pulumi/AWSMegatests/src/integrations/github.py deleted file mode 100644 index 7a43500d..00000000 --- a/pulumi/AWSMegatests/src/integrations/github.py +++ /dev/null @@ -1,169 +0,0 @@ -"""GitHub integration for AWS Megatests - secrets and variables management.""" - -from typing import Dict, Any, List, Optional, Union - -import pulumi -import pulumi_github as github - -from ..utils.constants import GITHUB_VARIABLE_NAMES, S3_BUCKET_NAME - - -class GitHubIntegrationError(Exception): - """Exception raised when GitHub integration operations fail.""" - - pass - - -def _create_organization_variable( - provider: github.Provider, - resource_name: str, - variable_name: str, - value: Union[str, pulumi.Output[str]], -) -> github.ActionsOrganizationVariable: - """Create a GitHub organization variable with consistent configuration. - - Args: - provider: GitHub provider instance - resource_name: Pulumi resource name - variable_name: GitHub variable name - value: Variable value - - Returns: - github.ActionsOrganizationVariable: Created variable resource - """ - return github.ActionsOrganizationVariable( - resource_name, - visibility="all", - variable_name=variable_name, - value=value, - opts=pulumi.ResourceOptions( - provider=provider, - delete_before_replace=True, # Workaround for GitHub provider issue #250 - ignore_changes=[ - "visibility" - ], # Ignore changes to visibility if variable exists - ), - ) - - -def _create_gh_commands( - workspace_id_val: str, cpu_env_id_val: str, tower_token_val: Optional[str] = None -) -> List[str]: - """Generate manual gh CLI commands for secrets management. - - Args: - workspace_id_val: Workspace ID value - cpu_env_id_val: CPU environment ID value - tower_token_val: Optional tower access token placeholder - - Returns: - List[str]: List of gh CLI commands - """ - commands = [] - - # Legacy workspace ID secret - commands.append( - f'gh secret set TOWER_WORKSPACE_ID --org nf-core --body "{workspace_id_val}" --visibility all' - ) - - # Legacy compute env secret (CPU) - commands.append( - f'gh secret set TOWER_COMPUTE_ENV --org nf-core --body "{cpu_env_id_val}" --visibility all' - ) - - # Tower access token (if provided) - if tower_token_val: - commands.append( - "OP_ACCOUNT=nf-core gh secret set TOWER_ACCESS_TOKEN --org nf-core " - "--body \"$(op read 'op://Dev/Seqera Platform/TOWER_ACCESS_TOKEN')\" --visibility all" - ) - - return commands - - -def create_github_resources( - github_provider: github.Provider, - compute_env_ids: Dict[str, Any], - tower_workspace_id: Union[str, pulumi.Output[str]], - tower_access_token: Optional[str] = None, -) -> Dict[str, Any]: - """Create GitHub organization variables and provide manual secret commands. - - Args: - github_provider: GitHub provider instance - compute_env_ids: Dictionary containing compute environment IDs - tower_workspace_id: Seqera Platform workspace ID - tower_access_token: Tower access token for manual secret commands (optional) - - Returns: - Dict[str, Any]: Dictionary containing created variables and manual commands - - Raises: - GitHubIntegrationError: If GitHub resource creation fails - """ - # Create org-level GitHub variables for compute environment IDs (non-sensitive) - # Using delete_before_replace to work around pulumi/pulumi-github#250 - variables = {} - - # Compute environment variables - for env_type in ["cpu", "gpu", "arm"]: - var_name = GITHUB_VARIABLE_NAMES[env_type] - resource_name = f"tower-compute-env-{env_type}" - - variables[env_type] = _create_organization_variable( - github_provider, - resource_name, - var_name, - compute_env_ids[env_type], - ) - - # Workspace ID variable - variables["workspace_id"] = _create_organization_variable( - github_provider, - "tower-workspace-id", - GITHUB_VARIABLE_NAMES["workspace_id"], - tower_workspace_id, - ) - - # Legacy S3 bucket variable - variables["legacy_s3_bucket"] = _create_organization_variable( - github_provider, - "legacy-aws-s3-bucket", - GITHUB_VARIABLE_NAMES["s3_bucket"], - S3_BUCKET_NAME, - ) - - # GitHub Secrets Management - Manual Commands Only - # NOTE: Due to pulumi/pulumi-github#250, secrets must be managed manually - # https://github.com/nf-core/ops/issues/162 - Legacy compatibility needed - - # Generate manual gh CLI commands for secrets management - if all(isinstance(compute_env_ids[k], str) for k in compute_env_ids) and isinstance( - tower_workspace_id, str - ): - # All static values - gh_cli_commands: Union[List[str], pulumi.Output[List[str]]] = ( - _create_gh_commands( - tower_workspace_id, - compute_env_ids["cpu"], - "" if tower_access_token else None, - ) - ) - else: - # Dynamic values - create commands that will be resolved at runtime - gh_cli_commands = pulumi.Output.all( - workspace_id=tower_workspace_id, cpu_env_id=compute_env_ids["cpu"] - ).apply( - lambda args: _create_gh_commands( - args["workspace_id"], - args["cpu_env_id"], - "" if tower_access_token else None, - ) - ) - - return { - "variables": variables, - "secrets": {}, # No Pulumi-managed secrets due to provider issue - "gh_cli_commands": gh_cli_commands, - "note": "Secrets must be managed manually due to pulumi/pulumi-github#250", - } diff --git a/pulumi/AWSMegatests/src/integrations/github_credentials.py b/pulumi/AWSMegatests/src/integrations/github_credentials.py deleted file mode 100644 index 707bc8c7..00000000 --- a/pulumi/AWSMegatests/src/integrations/github_credentials.py +++ /dev/null @@ -1,96 +0,0 @@ -"""GitHub credentials integration for Seqera Platform.""" - -import pulumi -import pulumi_seqera as seqera -from typing import Dict, Tuple - - -class GitHubCredentialError(Exception): - """Exception raised when GitHub credential creation fails.""" - - pass - - -def create_github_credential( - seqera_provider: seqera.Provider, - workspace_id: int, - github_token: str, - github_username: str = "nf-core-bot", - credential_name: str = "nf-core-github-finegrained", -) -> Tuple[seqera.Credential, str]: - """Create a GitHub fine-grained credential in Seqera Platform. - - This credential allows Seqera Platform to pull pipeline repositories from GitHub - without hitting GitHub rate limits. The fine-grained token provides secure, - scoped access to nf-core repositories with minimal required permissions. - - Args: - seqera_provider: Configured Seqera provider instance - workspace_id: Seqera workspace ID - github_token: Fine-grained GitHub personal access token for repository access - github_username: GitHub username (default: nf-core-bot) - credential_name: Name for the credential in Seqera - - Returns: - Tuple of (credential_resource, credential_id) - - Raises: - GitHubCredentialError: If credential creation fails - ValueError: If required parameters are missing - """ - # Validate required parameters - if not github_token: - raise ValueError("GitHub token is required") - if not workspace_id: - raise ValueError("Workspace ID is required") - - pulumi.log.info( - f"Creating GitHub credential '{credential_name}' in workspace {workspace_id}" - ) - - try: - # Create GitHub credential using Seqera Terraform provider - github_credential = seqera.Credential( - f"github-credential-{credential_name}", - name=credential_name, - description="Fine-grained GitHub token to avoid rate limits when Platform pulls pipeline repositories", - provider_type="github", - base_url="https://github.com/nf-core/", # Scope to nf-core organization - keys=seqera.CredentialKeysArgs( - github=seqera.CredentialKeysGithubArgs( - username=github_username, - password=github_token, # GitHub tokens go in the password field - ) - ), - workspace_id=workspace_id, - opts=pulumi.ResourceOptions( - provider=seqera_provider, - protect=True, # Protect credential from accidental deletion - ), - ) - - # Return both the resource and the credential ID for reference - return github_credential, github_credential.id - - except Exception as e: - pulumi.log.error(f"Failed to create GitHub credential: {str(e)}") - raise GitHubCredentialError( - f"GitHub credential creation failed: {str(e)}" - ) from e - - -def get_github_credential_config() -> Dict[str, str]: - """Get configuration for GitHub credential creation. - - Returns: - Dict containing configuration values from ESC environment - """ - import os - - return { - "github_finegrained_token": os.environ.get("PLATFORM_GITHUB_ORG_TOKEN", ""), - "github_username": os.environ.get("GITHUB_USERNAME", "nf-core-bot"), - "credential_name": os.environ.get( - "GITHUB_CREDENTIAL_NAME", "nf-core-github-finegrained" - ), - } diff --git a/pulumi/AWSMegatests/src/integrations/workspace_participants_command.py b/pulumi/AWSMegatests/src/integrations/workspace_participants_command.py deleted file mode 100644 index 7e99190c..00000000 --- a/pulumi/AWSMegatests/src/integrations/workspace_participants_command.py +++ /dev/null @@ -1,268 +0,0 @@ -"""Seqera Platform workspace participant management using Pulumi Command provider.""" - -import json -import pulumi -import pulumi_command as command -from typing import Dict, List, Optional -from ..utils.logging import log_info - - -def create_team_data_setup_command( - workspace_id: int, - seqera_token: str, - github_token: str, - opts: Optional[pulumi.ResourceOptions] = None, -) -> command.local.Command: - """ - Create a Pulumi Command that generates team data with proper credentials. - - This runs the team data setup scripts automatically during Pulumi deployment. - """ - setup_cmd = command.local.Command( - "team-data-setup", - create=""" -# Generate team member data with proper credentials -echo "=== Setting up team member data with privacy protection ===" - -# Run setup script with environment credentials -uv run python scripts/setup_team_data.py - -echo "✓ Team data setup completed" -echo "Files generated locally (not committed to git):" -echo " - scripts/maintainers_data.json" -echo " - scripts/core_team_data.json" -echo " - scripts/unified_team_data.json" - """, - environment={ - "GITHUB_TOKEN": github_token, - "TOWER_ACCESS_TOKEN": seqera_token, - }, - opts=opts, - ) - - return setup_cmd - - -def create_individual_member_commands( - workspace_id: int, - token: str, - github_token: str, - org_id: int = 252464779077610, # nf-core - opts: Optional[pulumi.ResourceOptions] = None, -) -> tuple[command.local.Command, Dict[str, command.local.Command]]: - """ - Create individual Pulumi Command resources for each GitHub team member. - - This provides granular tracking of each maintainer's workspace participant status. - """ - # First, ensure team data is set up with proper credentials - setup_cmd = create_team_data_setup_command(workspace_id, token, github_token, opts) - - # Generate team data at runtime to avoid committing private emails - log_info("Team data will be generated automatically during deployment...") - - # Load team data (will be available after setup command runs) - try: - with open("scripts/unified_team_data.json", "r") as f: - data = json.load(f) - team_members = data.get("seqera_participants", []) - log_info(f"Loaded {len(team_members)} team members from runtime data") - except FileNotFoundError: - # For preview purposes, show expected team count - log_info("Team data will be generated during deployment (35 expected members)") - team_members = [] - except Exception as e: - log_info(f"Team data will be generated at runtime: {e}") - team_members = [] - - member_commands = {} - - log_info(f"Creating individual tracking for {len(team_members)} team members") - - for member in team_members: - email = member["name"] - github_username = member["github_username"] - role = member["role"] # OWNER for core team, MAINTAIN for maintainers - - # Create safe resource name - safe_name = github_username.replace("-", "_").replace(".", "_") - - # Note: Role precedence handled in bash script (core team checked first) - - # Create individual command for this member - member_cmd = command.local.Command( - f"team_sync_{safe_name}", - create=f''' -#!/bin/bash -# Sync GitHub team member '{github_username}' to Seqera workspace with {role} role -echo "=== Syncing {github_username} ({email}) as {role} ===" - -# Verify user is still in appropriate GitHub teams -echo "Checking GitHub team membership..." -found_in_team=false - -# Check core team first (higher precedence) -if gh api orgs/nf-core/teams/core/members --jq '.[].login' | grep -q "^{github_username}$"; then - echo "✓ {github_username} confirmed in nf-core/core team (OWNER role)" - current_role="OWNER" - found_in_team=true -elif gh api orgs/nf-core/teams/maintainers/members --jq '.[].login' | grep -q "^{github_username}$"; then - echo "✓ {github_username} confirmed in nf-core/maintainers team (MAINTAIN role)" - current_role="MAINTAIN" - found_in_team=true -fi - -if [ "$found_in_team" = false ]; then - echo "⚠️ {github_username} not found in any relevant team, skipping" - exit 0 -fi - -# Ensure we're using the correct role (core team precedence) -target_role="{role}" -if [ "$current_role" != "$target_role" ]; then - echo "🔄 Role precedence: Using $current_role (detected) instead of $target_role" - target_role="$current_role" -fi - -# Check current email (in case it changed) -echo "Fetching current email..." -current_email=$(gh api /users/{github_username} --jq '.email // empty') - -# Handle members without public emails -if [[ "{email}" == github:* ]]; then - # Member has no public email, try to get current one - if [ -z "$current_email" ]; then - echo "⚠️ {github_username} has no public email - cannot add to Seqera Platform" - echo "STATUS:NO_EMAIL:{github_username}:$target_role" - exit 0 - else - echo "✓ {github_username} now has public email: $current_email" - fi -else - # Member had cached email, check if it changed - cached_email="{email}" - if [ -z "$current_email" ]; then - echo "⚠️ {github_username} email no longer public, using cached: $cached_email" - current_email="$cached_email" - elif [ "$current_email" != "$cached_email" ]; then - echo "🔄 {github_username} email changed: $cached_email → $current_email" - else - echo "✓ Current email confirmed: $current_email" - fi -fi - -# Add to Seqera workspace -echo "Adding to Seqera workspace {workspace_id}..." -response=$(curl -s -w "%{{http_code}}" -X PUT \\ - "https://api.cloud.seqera.io/orgs/{org_id}/workspaces/{workspace_id}/participants/add" \\ - -H "Authorization: Bearer {token}" \\ - -H "Content-Type: application/json" \\ - -d '{{"userNameOrEmail": "'$current_email'"}}') - -http_code="${{response: -3}}" -response_body="${{response%???}}" - -case $http_code in - 200|201|204) - echo "✓ Successfully added {github_username} with $target_role role" - echo "STATUS:ADDED:$current_email:$target_role" - ;; - 409) - echo "~ {github_username} already exists in workspace" - echo "STATUS:EXISTS:$current_email:$target_role" - ;; - 404) - echo "✗ User not found in Seqera Platform: $current_email" - echo "STATUS:USER_NOT_FOUND:$current_email:N/A" - ;; - *) - echo "✗ Failed to add {github_username}: HTTP $http_code" - echo "Response: $response_body" - echo "STATUS:FAILED:$current_email:ERROR" - exit 1 - ;; -esac - -echo "Completed sync for {github_username}" - ''', - environment={ - "GITHUB_TOKEN": github_token, - }, - opts=pulumi.ResourceOptions( - depends_on=[setup_cmd], - parent=opts.parent if opts else None, - ), - ) - - member_commands[github_username] = member_cmd - - return setup_cmd, member_commands - - -def create_workspace_participants_via_command( - workspace_id: int, - token: str, - participants_data: List[Dict[str, str]], - opts: Optional[pulumi.ResourceOptions] = None, -) -> command.local.Command: - """ - Create workspace participants using Pulumi Command provider to run the Python script. - - Args: - workspace_id: Seqera workspace ID - token: Seqera API access token - participants_data: List of participant dictionaries - opts: Pulumi resource options - - Returns: - Command resource that manages workspace participants - """ - # Create the command that will run our Python script - participant_count = len(participants_data) - log_info(f"Creating command to add {participant_count} workspace participants") - - # The command runs within the Pulumi execution context - add_participants_cmd = command.local.Command( - "add-workspace-participants", - create="uv run python scripts/add_maintainers_to_workspace.py --yes", - environment={ - "TOWER_ACCESS_TOKEN": token, - "TOWER_WORKSPACE_ID": str(workspace_id), - }, - opts=pulumi.ResourceOptions(**opts.__dict__ if opts else {}), - ) - - return add_participants_cmd - - -def create_workspace_participants_verification( - workspace_id: int, - token: str, - depends_on: List[pulumi.Resource], - opts: Optional[pulumi.ResourceOptions] = None, -) -> command.local.Command: - """ - Create a verification command that checks workspace participants were added correctly. - - Args: - workspace_id: Seqera workspace ID - token: Seqera API access token - depends_on: Resources this command depends on - opts: Pulumi resource options - - Returns: - Command resource that verifies workspace participants - """ - verification_cmd = command.local.Command( - "verify-workspace-participants", - create="uv run python scripts/inspect_participants.py", - environment={ - "TOWER_ACCESS_TOKEN": token, - "TOWER_WORKSPACE_ID": str(workspace_id), - }, - opts=pulumi.ResourceOptions( - depends_on=depends_on, **(opts.__dict__ if opts else {}) - ), - ) - - return verification_cmd diff --git a/pulumi/AWSMegatests/src/integrations/workspace_participants_simple.py b/pulumi/AWSMegatests/src/integrations/workspace_participants_simple.py deleted file mode 100644 index 5cd50d1a..00000000 --- a/pulumi/AWSMegatests/src/integrations/workspace_participants_simple.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Simple workspace participant management using Pulumi's apply() pattern.""" - -import json -import pulumi -import requests -from typing import Dict, List, Any -from ..utils.logging import log_info - - -def add_workspace_participant_simple( - email: str, - role: str, - workspace_id: pulumi.Output[str], - token: pulumi.Output[str], - org_id: int = 252464779077610, -) -> pulumi.Output[Dict[str, Any]]: - """ - Add a workspace participant using Pulumi's apply() pattern. - - This is simpler than dynamic resources but still integrates with Pulumi. - """ - - def _add_participant(args): - """Internal function that does the actual API call.""" - workspace_id_val, token_val = args - - headers = { - "Authorization": f"Bearer {token_val}", - "Content-Type": "application/json", - } - - url = f"https://api.cloud.seqera.io/orgs/{org_id}/workspaces/{workspace_id_val}/participants/add" - payload = {"userNameOrEmail": email} - - try: - response = requests.put(url, headers=headers, json=payload, timeout=30) - - if response.status_code in [200, 201, 204]: - return { - "email": email, - "role": role, - "workspace_id": workspace_id_val, - "status": "added", - "participant_id": f"{org_id}:{workspace_id_val}:{email}", - } - elif response.status_code == 409: - return { - "email": email, - "role": role, - "workspace_id": workspace_id_val, - "status": "already_exists", - "participant_id": f"{org_id}:{workspace_id_val}:{email}", - } - else: - return { - "email": email, - "role": role, - "workspace_id": workspace_id_val, - "status": "failed", - "error": f"HTTP {response.status_code}", - } - - except Exception as e: - return { - "email": email, - "role": role, - "workspace_id": workspace_id_val, - "status": "error", - "error": str(e), - } - - # Use Pulumi's apply to handle the async nature of Outputs - return pulumi.Output.all(workspace_id, token).apply(_add_participant) - - -def create_workspace_participants_simple( - workspace_id: pulumi.Output[str], - token: pulumi.Output[str], - maintainer_emails: List[str], - role: str = "MAINTAIN", -) -> pulumi.Output[List[Dict[str, Any]]]: - """ - Create multiple workspace participants using the simple approach. - - Args: - workspace_id: Seqera workspace ID as Pulumi Output - token: Seqera API token as Pulumi Output - maintainer_emails: List of email addresses to add - role: Role to assign (default: MAINTAIN) - - Returns: - Pulumi Output containing list of participant creation results - """ - - participant_outputs = [] - - for email in maintainer_emails: - participant_result = add_workspace_participant_simple( - email, role, workspace_id, token - ) - participant_outputs.append(participant_result) - - # Combine all outputs into a single list - return pulumi.Output.all(*participant_outputs) - - -def load_maintainer_emails_static() -> List[str]: - """Load maintainer emails from the JSON file (static version for Pulumi).""" - try: - with open("scripts/maintainers_data.json", "r") as f: - data = json.load(f) - - participants = data.get("seqera_participants", []) - emails = [p["name"] for p in participants] - - log_info(f"Loaded {len(emails)} maintainer emails for workspace participants") - return emails - - except FileNotFoundError: - log_info("Maintainers data file not found, skipping workspace participants") - return [] - except Exception as e: - log_info(f"Error loading maintainers data: {e}") - return [] diff --git a/pulumi/AWSMegatests/src/providers/__init__.py b/pulumi/AWSMegatests/src/providers/__init__.py deleted file mode 100644 index 39eb2523..00000000 --- a/pulumi/AWSMegatests/src/providers/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Provider configurations for AWS Megatests infrastructure.""" - -from .aws import create_aws_provider -from .github import create_github_provider -from .seqera import create_seqera_provider - -__all__ = ["create_aws_provider", "create_github_provider", "create_seqera_provider"] diff --git a/pulumi/AWSMegatests/src/providers/aws.py b/pulumi/AWSMegatests/src/providers/aws.py deleted file mode 100644 index 9c0febdb..00000000 --- a/pulumi/AWSMegatests/src/providers/aws.py +++ /dev/null @@ -1,19 +0,0 @@ -"""AWS provider configuration for AWS Megatests infrastructure.""" - -import pulumi_aws as aws -from ..utils.constants import AWS_REGION - - -def create_aws_provider() -> aws.Provider: - """Create AWS provider using ESC OIDC authentication. - - The ESC environment should automatically provide AWS credentials - when the environment is imported in Pulumi.prod.yaml. - - Returns: - aws.Provider: Configured AWS provider instance - """ - return aws.Provider( - "aws-provider", - region=AWS_REGION, - ) diff --git a/pulumi/AWSMegatests/src/providers/github.py b/pulumi/AWSMegatests/src/providers/github.py deleted file mode 100644 index 03ac9db9..00000000 --- a/pulumi/AWSMegatests/src/providers/github.py +++ /dev/null @@ -1,16 +0,0 @@ -"""GitHub provider configuration for AWS Megatests infrastructure.""" - -import pulumi_github as github -from ..utils.constants import GITHUB_ORG - - -def create_github_provider(github_token: str) -> github.Provider: - """Create GitHub provider with token authentication. - - Args: - github_token: GitHub personal access token with org admin permissions - - Returns: - github.Provider: Configured GitHub provider instance - """ - return github.Provider("github-provider", token=github_token, owner=GITHUB_ORG) diff --git a/pulumi/AWSMegatests/src/providers/seqera.py b/pulumi/AWSMegatests/src/providers/seqera.py deleted file mode 100644 index 465f728c..00000000 --- a/pulumi/AWSMegatests/src/providers/seqera.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Seqera provider configuration for AWS Megatests infrastructure.""" - -import pulumi -import pulumi_seqera as seqera -from typing import Dict, Any -from ..utils.constants import SEQERA_API_URL, ERROR_MESSAGES - - -class SeqeraProviderError(Exception): - """Exception raised when Seqera provider initialization fails.""" - - pass - - -def create_seqera_provider(config: Dict[str, Any]) -> seqera.Provider: - """Create and configure the Seqera provider with error handling. - - Args: - config: Configuration dictionary containing tower_access_token - - Returns: - seqera.Provider: Configured Seqera provider instance - - Raises: - SeqeraProviderError: If provider creation fails - ValueError: If required configuration is missing - """ - # Validate required configuration - if not config.get("tower_access_token"): - raise ValueError(ERROR_MESSAGES["missing_tower_token"]) - - pulumi.log.info("Creating Seqera provider with Cloud API endpoint") - - return seqera.Provider( - "seqera-provider", - bearer_auth=config["tower_access_token"], - server_url=SEQERA_API_URL, - ) diff --git a/pulumi/AWSMegatests/src/utils/__init__.py b/pulumi/AWSMegatests/src/utils/__init__.py deleted file mode 100644 index 5687405f..00000000 --- a/pulumi/AWSMegatests/src/utils/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Utility functions and constants for AWS Megatests.""" - -from .constants import ( - AWS_REGION, - S3_BUCKET_NAME, - SEQERA_API_URL, - COMPUTE_ENV_NAMES, - TOWERFORGE_POLICY_NAMES, -) -from .logging import ( - log_info, - log_error, - log_warning, - log_step, - log_resource_creation, - log_resource_success, -) - -__all__ = [ - "AWS_REGION", - "S3_BUCKET_NAME", - "SEQERA_API_URL", - "COMPUTE_ENV_NAMES", - "TOWERFORGE_POLICY_NAMES", - "log_info", - "log_error", - "log_warning", - "log_step", - "log_resource_creation", - "log_resource_success", -] diff --git a/pulumi/AWSMegatests/src/utils/constants.py b/pulumi/AWSMegatests/src/utils/constants.py deleted file mode 100644 index baf1eefa..00000000 --- a/pulumi/AWSMegatests/src/utils/constants.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Constants and configuration values for AWS Megatests infrastructure.""" - -# AWS Configuration -AWS_REGION = "eu-west-1" -S3_BUCKET_NAME = "nf-core-awsmegatests" -S3_WORK_DIR = f"s3://{S3_BUCKET_NAME}" - -# Seqera Configuration -SEQERA_API_URL = "https://api.cloud.seqera.io" - -# Compute Environment Names -COMPUTE_ENV_NAMES = { - "cpu": "aws_ireland_fusionv2_nvme_cpu_snapshots", - "gpu": "aws_ireland_fusionv2_nvme_gpu_snapshots", - "arm": "aws_ireland_fusionv2_nvme_cpu_ARM_snapshots", -} - -# Compute Environment Descriptions -COMPUTE_ENV_DESCRIPTIONS = { - "cpu": "CPU compute environment with Fusion v2 and NVMe storage", - "gpu": "GPU compute environment with Fusion v2 and NVMe storage", - "arm": "ARM CPU compute environment with Fusion v2 and NVMe storage", -} - -# Configuration File Paths -CONFIG_FILES = { - "cpu": "seqerakit/current-env-cpu.json", - "gpu": "seqerakit/current-env-gpu.json", - "arm": "seqerakit/current-env-cpu-arm.json", -} - -# Nextflow configuration files for compute environments -NEXTFLOW_CONFIG_FILES = { - "cpu": "seqerakit/configs/nextflow-cpu.config", - "gpu": "seqerakit/configs/nextflow-gpu.config", - "arm": "seqerakit/configs/nextflow-arm.config", -} - -# TowerForge Configuration -TOWERFORGE_USER_NAME = "TowerForge-AWSMegatests" -TOWERFORGE_POLICY_NAMES = { - "forge": "TowerForge-Forge-Policy", - "launch": "TowerForge-Launch-Policy", - "s3": "TowerForge-S3-Policy", -} - -TOWERFORGE_CREDENTIAL_NAME = "TowerForge-AWSMegatests-Dynamic" -TOWERFORGE_CREDENTIAL_DESCRIPTION = ( - "Dynamically created TowerForge credentials for AWS Megatests compute environments" -) - -# GitHub Configuration -GITHUB_ORG = "nf-core" -GITHUB_VARIABLE_NAMES = { - "cpu": "TOWER_COMPUTE_ENV_CPU", - "gpu": "TOWER_COMPUTE_ENV_GPU", - "arm": "TOWER_COMPUTE_ENV_ARM", - "workspace_id": "TOWER_WORKSPACE_ID", - "s3_bucket": "AWS_S3_BUCKET", -} - -# Timeout Configuration (in minutes) -TIMEOUTS = { - "seqera_credential_create": "5m", - "seqera_credential_update": "5m", - "seqera_credential_delete": "2m", - "compute_env_create": "10m", - "compute_env_update": "10m", - "compute_env_delete": "5m", -} - -# Default Compute Environment Settings -DEFAULT_COMPUTE_ENV_CONFIG = { - "region": AWS_REGION, - "workDir": S3_WORK_DIR, - "waveEnabled": True, - "fusion2Enabled": True, - "nvnmeStorageEnabled": True, - "fusionSnapshots": True, - "nextflowConfig": "", -} - -DEFAULT_FORGE_CONFIG = { - "type": "SPOT", - "minCpus": 0, - "maxCpus": 1000, - "gpuEnabled": False, - "instanceTypes": [], - "subnets": [], - "securityGroups": [], - "disposeOnDeletion": True, - "allowBuckets": [], - "efsCreate": False, - "ebsBootSize": 50, - "fargateHeadEnabled": True, - "arm64Enabled": False, -} - -# Error Messages -ERROR_MESSAGES = { - "missing_tower_token": ( - "TOWER_ACCESS_TOKEN is required for Seqera provider. " - "Please ensure it's set in your ESC environment with proper permissions: " - "WORKSPACE_ADMIN or COMPUTE_ENV_ADMIN scope." - ), - "seqera_provider_init_failed": ( - "Seqera provider initialization failed. " - "This usually indicates token permissions issues. " - "Ensure your TOWER_ACCESS_TOKEN has WORKSPACE_ADMIN or COMPUTE_ENV_ADMIN permissions." - ), - "config_file_not_found": "Configuration file not found: {}", - "invalid_json": "Invalid JSON in configuration file {}: {}", - "config_load_failed": "Failed to load configuration file {}: {}", - "invalid_workspace_id": "Invalid or missing workspace ID: {}", - "missing_compute_env_params": "Missing required parameters for compute environment {}", - "missing_config_args": "Configuration arguments are required for compute environment {}", - "compute_env_create_failed": ( - "Failed to create compute environment '{}'. " - "Common causes: " - "1. Seqera API token lacks required permissions (403 Forbidden) " - "2. Invalid credentials_id reference " - "3. Workspace access restrictions " - "4. Network connectivity issues" - ), - "credential_upload_failed": ( - "Failed to upload credentials to Seqera Platform. " - "Common causes: " - "1. Seqera provider not properly configured " - "2. Invalid workspace ID " - "3. Network connectivity issues to api.cloud.seqera.io " - "4. Invalid AWS credentials format" - ), -} - -# Required Environment Variables -REQUIRED_ENV_VARS = [ - "TOWER_ACCESS_TOKEN", - "TOWER_WORKSPACE_ID", - "GITHUB_TOKEN", -] - -# Optional Environment Variables with Defaults -DEFAULT_ENV_VARS = { - "TOWER_WORKSPACE_ID": "59994744926013", # Fallback workspace ID -} diff --git a/pulumi/AWSMegatests/src/utils/logging.py b/pulumi/AWSMegatests/src/utils/logging.py deleted file mode 100644 index 2cb342bd..00000000 --- a/pulumi/AWSMegatests/src/utils/logging.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Logging utilities for AWS Megatests infrastructure.""" - -import pulumi -from typing import Optional - - -def log_info(message: str, context: Optional[str] = None) -> None: - """Log an informational message with optional context. - - Args: - message: The message to log - context: Optional context prefix - """ - formatted_message = f"[{context}] {message}" if context else message - pulumi.log.info(formatted_message) - - -def log_error(message: str, context: Optional[str] = None) -> None: - """Log an error message with optional context. - - Args: - message: The error message to log - context: Optional context prefix - """ - formatted_message = f"[{context}] {message}" if context else message - pulumi.log.error(formatted_message) - - -def log_warning(message: str, context: Optional[str] = None) -> None: - """Log a warning message with optional context. - - Args: - message: The warning message to log - context: Optional context prefix - """ - formatted_message = f"[{context}] {message}" if context else message - pulumi.log.warn(formatted_message) - - -def log_step(step_number: int, step_name: str, description: str) -> None: - """Log a deployment step with consistent formatting. - - Args: - step_number: The step number - step_name: Short name for the step - description: Detailed description of what the step does - """ - log_info(f"Step {step_number}: {step_name} - {description}") - - -def log_resource_creation(resource_type: str, resource_name: str) -> None: - """Log resource creation with consistent formatting. - - Args: - resource_type: Type of resource being created - resource_name: Name of the resource - """ - log_info(f"Creating {resource_type}: {resource_name}", "Resource") - - -def log_resource_success(resource_type: str, resource_name: str) -> None: - """Log successful resource creation. - - Args: - resource_type: Type of resource that was created - resource_name: Name of the resource - """ - log_info(f"Successfully created {resource_type}: {resource_name}", "Resource") diff --git a/pulumi/AWSMegatests/uv.lock b/pulumi/AWSMegatests/uv.lock deleted file mode 100644 index 54620e57..00000000 --- a/pulumi/AWSMegatests/uv.lock +++ /dev/null @@ -1,459 +0,0 @@ -version = 1 -requires-python = ">=3.12" - -[[package]] -name = "arpeggio" -version = "2.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/12/c4/516bb54456f85ad1947702ea4cef543a59de66d31a9887dbc3d9df36e3e1/Arpeggio-2.0.2.tar.gz", hash = "sha256:c790b2b06e226d2dd468e4fbfb5b7f506cec66416031fde1441cf1de2a0ba700", size = 766643 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/4f/d28bf30a19d4649b40b501d531b44e73afada99044df100380fd9567e92f/Arpeggio-2.0.2-py2.py3-none-any.whl", hash = "sha256:f7c8ae4f4056a89e020c24c7202ac8df3e2bc84e416746f20b0da35bb1de0250", size = 55287 }, -] - -[[package]] -name = "attrs" -version = "25.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815 }, -] - -[[package]] -name = "aws-megatests" -version = "0.1.0" -source = { virtual = "." } -dependencies = [ - { name = "pulumi" }, - { name = "pulumi-aws" }, - { name = "pulumi-command" }, - { name = "pulumi-github" }, - { name = "pulumi-seqera" }, - { name = "requests" }, -] - -[package.dev-dependencies] -dev = [ - { name = "mypy" }, - { name = "pytest" }, -] - -[package.metadata] -requires-dist = [ - { name = "pulumi", specifier = ">=3.173.0,<4.0.0" }, - { name = "pulumi-aws", specifier = ">=6.81.0,<7.0.0" }, - { name = "pulumi-command", specifier = ">=1.0.1,<2.0.0" }, - { name = "pulumi-github", specifier = ">=6.4.0,<7.0.0" }, - { name = "pulumi-seqera", directory = "sdks/seqera" }, - { name = "requests", specifier = ">=2.28.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "mypy", specifier = ">=1.17.1" }, - { name = "pytest", specifier = ">=7.0.0" }, -] - -[[package]] -name = "certifi" -version = "2025.8.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216 }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655 }, - { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223 }, - { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366 }, - { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104 }, - { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830 }, - { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854 }, - { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670 }, - { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501 }, - { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173 }, - { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822 }, - { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543 }, - { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326 }, - { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008 }, - { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196 }, - { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819 }, - { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350 }, - { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644 }, - { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468 }, - { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187 }, - { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699 }, - { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580 }, - { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366 }, - { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342 }, - { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995 }, - { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640 }, - { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636 }, - { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939 }, - { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580 }, - { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870 }, - { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797 }, - { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224 }, - { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086 }, - { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400 }, - { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175 }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, -] - -[[package]] -name = "debugpy" -version = "1.8.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/75/087fe07d40f490a78782ff3b0a30e3968936854105487decdb33446d4b0e/debugpy-1.8.14.tar.gz", hash = "sha256:7cd287184318416850aa8b60ac90105837bb1e59531898c07569d197d2ed5322", size = 1641444 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/2a/ac2df0eda4898f29c46eb6713a5148e6f8b2b389c8ec9e425a4a1d67bf07/debugpy-1.8.14-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:8899c17920d089cfa23e6005ad9f22582fd86f144b23acb9feeda59e84405b84", size = 2501268 }, - { url = "https://files.pythonhosted.org/packages/10/53/0a0cb5d79dd9f7039169f8bf94a144ad3efa52cc519940b3b7dde23bcb89/debugpy-1.8.14-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6bb5c0dcf80ad5dbc7b7d6eac484e2af34bdacdf81df09b6a3e62792b722826", size = 4221077 }, - { url = "https://files.pythonhosted.org/packages/f8/d5/84e01821f362327bf4828728aa31e907a2eca7c78cd7c6ec062780d249f8/debugpy-1.8.14-cp312-cp312-win32.whl", hash = "sha256:281d44d248a0e1791ad0eafdbbd2912ff0de9eec48022a5bfbc332957487ed3f", size = 5255127 }, - { url = "https://files.pythonhosted.org/packages/33/16/1ed929d812c758295cac7f9cf3dab5c73439c83d9091f2d91871e648093e/debugpy-1.8.14-cp312-cp312-win_amd64.whl", hash = "sha256:5aa56ef8538893e4502a7d79047fe39b1dae08d9ae257074c6464a7b290b806f", size = 5297249 }, - { url = "https://files.pythonhosted.org/packages/4d/e4/395c792b243f2367d84202dc33689aa3d910fb9826a7491ba20fc9e261f5/debugpy-1.8.14-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:329a15d0660ee09fec6786acdb6e0443d595f64f5d096fc3e3ccf09a4259033f", size = 2485676 }, - { url = "https://files.pythonhosted.org/packages/ba/f1/6f2ee3f991327ad9e4c2f8b82611a467052a0fb0e247390192580e89f7ff/debugpy-1.8.14-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f920c7f9af409d90f5fd26e313e119d908b0dd2952c2393cd3247a462331f15", size = 4217514 }, - { url = "https://files.pythonhosted.org/packages/79/28/b9d146f8f2dc535c236ee09ad3e5ac899adb39d7a19b49f03ac95d216beb/debugpy-1.8.14-cp313-cp313-win32.whl", hash = "sha256:3784ec6e8600c66cbdd4ca2726c72d8ca781e94bce2f396cc606d458146f8f4e", size = 5254756 }, - { url = "https://files.pythonhosted.org/packages/e0/62/a7b4a57013eac4ccaef6977966e6bec5c63906dd25a86e35f155952e29a1/debugpy-1.8.14-cp313-cp313-win_amd64.whl", hash = "sha256:684eaf43c95a3ec39a96f1f5195a7ff3d4144e4a18d69bb66beeb1a6de605d6e", size = 5297119 }, - { url = "https://files.pythonhosted.org/packages/97/1a/481f33c37ee3ac8040d3d51fc4c4e4e7e61cb08b8bc8971d6032acc2279f/debugpy-1.8.14-py2.py3-none-any.whl", hash = "sha256:5cd9a579d553b6cb9759a7908a41988ee6280b961f24f63336835d9418216a20", size = 5256230 }, -] - -[[package]] -name = "dill" -version = "0.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668 }, -] - -[[package]] -name = "grpcio" -version = "1.66.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/71/d1/49a96df4eb1d805cf546247df40636515416d2d5c66665e5129c8b4162a8/grpcio-1.66.2.tar.gz", hash = "sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231", size = 12489713 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/5c/c4da36b7a77dbb15c4bc72228dff7161874752b2c6bddf7bb046d9da1b90/grpcio-1.66.2-cp312-cp312-linux_armv7l.whl", hash = "sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf", size = 5002933 }, - { url = "https://files.pythonhosted.org/packages/a0/d5/b631445dff250a5301f51ff56c5fc917c7f955cd02fa55379f158a89abeb/grpcio-1.66.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8", size = 10793953 }, - { url = "https://files.pythonhosted.org/packages/c8/1c/2179ac112152e92c02990f98183edf645df14aa3c38b39f1a3a60358b6c6/grpcio-1.66.2-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6", size = 5499791 }, - { url = "https://files.pythonhosted.org/packages/0b/53/8d7ab865fbd983309c8242930f00b28a01047f70c2b2e4c79a5c92a46a08/grpcio-1.66.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7", size = 6109606 }, - { url = "https://files.pythonhosted.org/packages/86/e9/3dfb5a3ff540636d46b8b723345e923e8c553d9b3f6a8d1b09b0d915eb46/grpcio-1.66.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd", size = 5762866 }, - { url = "https://files.pythonhosted.org/packages/f1/cb/c07493ad5dd73d51e4e15b0d483ff212dfec136ee1e4f3b49d115bdc7a13/grpcio-1.66.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee", size = 6446819 }, - { url = "https://files.pythonhosted.org/packages/ff/5f/142e19db367a34ea0ee8a8451e43215d0a1a5dbffcfdcae8801f22903301/grpcio-1.66.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c", size = 6040273 }, - { url = "https://files.pythonhosted.org/packages/5c/3b/12fcd752c55002e4b0e0a7bd5faec101bc0a4e3890be3f95a43353142481/grpcio-1.66.2-cp312-cp312-win32.whl", hash = "sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453", size = 3537988 }, - { url = "https://files.pythonhosted.org/packages/f1/70/76bfea3faa862bfceccba255792e780691ff25b8227180759c9d38769379/grpcio-1.66.2-cp312-cp312-win_amd64.whl", hash = "sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679", size = 4275553 }, - { url = "https://files.pythonhosted.org/packages/72/31/8708a8dfb3f1ac89926c27c5dd17412764157a2959dbc5a606eaf8ac71f6/grpcio-1.66.2-cp313-cp313-linux_armv7l.whl", hash = "sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d", size = 5004245 }, - { url = "https://files.pythonhosted.org/packages/8b/37/0b57c3769efb3cc9ec97fcaa9f7243046660e7ed58c0faebc4ef315df92c/grpcio-1.66.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34", size = 10756749 }, - { url = "https://files.pythonhosted.org/packages/bf/5a/425e995724a19a1b110340ed653bc7c5de8019d9fc84b3798a0f79c3eb31/grpcio-1.66.2-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed", size = 5499666 }, - { url = "https://files.pythonhosted.org/packages/2e/e4/86a5c5ec40a6b683671a1d044ebca433812d99da8fcfc2889e9c43cecbd4/grpcio-1.66.2-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7", size = 6109578 }, - { url = "https://files.pythonhosted.org/packages/2f/86/a86742f3deaa22385c3bff984c5947fc62d47d3fab26c508730037d027e5/grpcio-1.66.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46", size = 5763274 }, - { url = "https://files.pythonhosted.org/packages/c3/61/b9a2a4345dea0a354c4ed8ac7aacbdd0ff986acbc8f92680213cf3d2faa3/grpcio-1.66.2-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a", size = 6450416 }, - { url = "https://files.pythonhosted.org/packages/50/b9/ad303ce75d8cd71d855a661519aa160ce42f27498f589f1ae6d9f8c5e8ac/grpcio-1.66.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b", size = 6040045 }, - { url = "https://files.pythonhosted.org/packages/ac/b3/8db1873e3240ef1672ba87b89e949ece367089e29e4d221377bfdd288bd3/grpcio-1.66.2-cp313-cp313-win32.whl", hash = "sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75", size = 3537126 }, - { url = "https://files.pythonhosted.org/packages/a2/df/133216989fe7e17caeafd7ff5b17cc82c4e722025d0b8d5d2290c11fe2e6/grpcio-1.66.2-cp313-cp313-win_amd64.whl", hash = "sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf", size = 4278018 }, -] - -[[package]] -name = "idna" -version = "3.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, -] - -[[package]] -name = "iniconfig" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050 }, -] - -[[package]] -name = "mypy" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mypy-extensions" }, - { name = "pathspec" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295 }, - { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355 }, - { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285 }, - { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895 }, - { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025 }, - { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664 }, - { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338 }, - { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066 }, - { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473 }, - { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296 }, - { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657 }, - { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320 }, - { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037 }, - { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550 }, - { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963 }, - { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189 }, - { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322 }, - { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879 }, - { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411 }, -] - -[[package]] -name = "mypy-extensions" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963 }, -] - -[[package]] -name = "packaging" -version = "25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469 }, -] - -[[package]] -name = "parver" -version = "0.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "arpeggio" }, - { name = "attrs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cc/e5/1c774688a90f0b76e872e30f6f1ba3f5e14056cd0d96a684047d4a986226/parver-0.5.tar.gz", hash = "sha256:b9fde1e6bb9ce9f07e08e9c4bea8d8825c5e78e18a0052d02e02bf9517eb4777", size = 26908 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/4c/f98024021bef4d44dce3613feebd702c7ad8883f777ff8488384c59e9774/parver-0.5-py3-none-any.whl", hash = "sha256:2281b187276c8e8e3c15634f62287b2fb6fe0efe3010f739a6bd1e45fa2bf2b2", size = 15172 }, -] - -[[package]] -name = "pathspec" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, -] - -[[package]] -name = "pip" -version = "25.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/59/de/241caa0ca606f2ec5fe0c1f4261b0465df78d786a38da693864a116c37f4/pip-25.1.1.tar.gz", hash = "sha256:3de45d411d308d5054c2168185d8da7f9a2cd753dbac8acbfa88a8909ecd9077", size = 1940155 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/a2/d40fb2460e883eca5199c62cfc2463fd261f760556ae6290f88488c362c0/pip-25.1.1-py3-none-any.whl", hash = "sha256:2913a38a2abf4ea6b64ab507bd9e967f3b53dc1ede74b01b0931e1ce548751af", size = 1825227 }, -] - -[[package]] -name = "pluggy" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, -] - -[[package]] -name = "protobuf" -version = "4.25.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/df/01/34c8d2b6354906d728703cb9d546a0e534de479e25f1b581e4094c4a85cc/protobuf-4.25.8.tar.gz", hash = "sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd", size = 380920 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/ff/05f34305fe6b85bbfbecbc559d423a5985605cad5eda4f47eae9e9c9c5c5/protobuf-4.25.8-cp310-abi3-win32.whl", hash = "sha256:504435d831565f7cfac9f0714440028907f1975e4bed228e58e72ecfff58a1e0", size = 392745 }, - { url = "https://files.pythonhosted.org/packages/08/35/8b8a8405c564caf4ba835b1fdf554da869954712b26d8f2a98c0e434469b/protobuf-4.25.8-cp310-abi3-win_amd64.whl", hash = "sha256:bd551eb1fe1d7e92c1af1d75bdfa572eff1ab0e5bf1736716814cdccdb2360f9", size = 413736 }, - { url = "https://files.pythonhosted.org/packages/28/d7/ab27049a035b258dab43445eb6ec84a26277b16105b277cbe0a7698bdc6c/protobuf-4.25.8-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ca809b42f4444f144f2115c4c1a747b9a404d590f18f37e9402422033e464e0f", size = 394537 }, - { url = "https://files.pythonhosted.org/packages/bd/6d/a4a198b61808dd3d1ee187082ccc21499bc949d639feb948961b48be9a7e/protobuf-4.25.8-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:9ad7ef62d92baf5a8654fbb88dac7fa5594cfa70fd3440488a5ca3bfc6d795a7", size = 294005 }, - { url = "https://files.pythonhosted.org/packages/d6/c6/c9deaa6e789b6fc41b88ccbdfe7a42d2b82663248b715f55aa77fbc00724/protobuf-4.25.8-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:83e6e54e93d2b696a92cad6e6efc924f3850f82b52e1563778dfab8b355101b0", size = 294924 }, - { url = "https://files.pythonhosted.org/packages/0c/c1/6aece0ab5209981a70cd186f164c133fdba2f51e124ff92b73de7fd24d78/protobuf-4.25.8-py3-none-any.whl", hash = "sha256:15a0af558aa3b13efef102ae6e4f3efac06f1eea11afb3a57db2901447d9fb59", size = 156757 }, -] - -[[package]] -name = "pulumi" -version = "3.181.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "debugpy" }, - { name = "dill" }, - { name = "grpcio" }, - { name = "pip" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "semver" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/61/12c5fb71e25e7ce518e4989e202ea14916e0f8e50759c19d181d0430e5d9/pulumi-3.181.0-py3-none-any.whl", hash = "sha256:923c28314850ce71cde0d4010df4e9392a51cab730cf9acfbe692679d281f93a", size = 356698 }, -] - -[[package]] -name = "pulumi-aws" -version = "6.83.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "parver" }, - { name = "pulumi" }, - { name = "semver" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/c8/d16a58afdab5c4dafb8043a4370c0673c48ae314aea5ac7d1ce4eee2af76/pulumi_aws-6.83.0.tar.gz", hash = "sha256:24ce88ad5bb81eb937935d9e40466e164addef86140a32cac22ed17176812cb7", size = 7839650 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/30/0b/3ed17e6fc68d9c0b572c4b259c5e291a8267011c640a9a9bb5a26b596763/pulumi_aws-6.83.0-py3-none-any.whl", hash = "sha256:e61144b9680ae1ebd98daa7bfffce1f646796a9640f9b73119b91e72c26e3bbc", size = 10608060 }, -] - -[[package]] -name = "pulumi-command" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "parver" }, - { name = "pulumi" }, - { name = "semver" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/96/b4/03d37a61e49b636b7cd5ba72a28996f07a18d815e6ba2343f6071b77fc51/pulumi_command-1.1.0.tar.gz", hash = "sha256:509b2709938aacfb13717b7b537d728375cd90b409093bf24dacab17d9294a4a", size = 33072 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/f4/c89295ca63e2555cda4b4e16069200cf04262ac87ae263c48562f8297cc8/pulumi_command-1.1.0-py3-none-any.whl", hash = "sha256:d6a62444a1a9a26ded354b668a9cf88c0becd4faacabe4adafbc8dbb65af96c4", size = 36859 }, -] - -[[package]] -name = "pulumi-github" -version = "6.7.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "parver" }, - { name = "pulumi" }, - { name = "semver" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e6/aa/97611dbf7c0c4ea622b6eef1890b3379d4c7fc0f9be22de092b00a743124/pulumi_github-6.7.3.tar.gz", hash = "sha256:7cce624ab8800ad94ecc906f0b09081957b2a1345f46392d43e49d145ca42434", size = 203054 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/da/ca6b46064fd67a3357a4105f4d6a0ad14bb8b73932fbd56564561a1d6b2c/pulumi_github-6.7.3-py3-none-any.whl", hash = "sha256:aa154bb245bb614ed2dfd879ef9363ff10d8073337c1639280e0744097dba2b6", size = 369378 }, -] - -[[package]] -name = "pulumi-seqera" -version = "0.25.2" -source = { directory = "sdks/seqera" } -dependencies = [ - { name = "parver" }, - { name = "pulumi" }, - { name = "semver" }, -] - -[package.metadata] -requires-dist = [ - { name = "parver", specifier = ">=0.2.1" }, - { name = "pulumi", specifier = ">=3.165.0,<4.0.0" }, - { name = "semver", specifier = ">=2.8.1" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'", specifier = ">=4.11,<5" }, -] - -[[package]] -name = "pygments" -version = "2.19.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217 }, -] - -[[package]] -name = "pytest" -version = "8.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "iniconfig" }, - { name = "packaging" }, - { name = "pluggy" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474 }, -] - -[[package]] -name = "pyyaml" -version = "6.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873 }, - { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302 }, - { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154 }, - { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223 }, - { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542 }, - { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164 }, - { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611 }, - { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591 }, - { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 }, - { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309 }, - { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679 }, - { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428 }, - { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361 }, - { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523 }, - { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660 }, - { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597 }, - { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527 }, - { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446 }, -] - -[[package]] -name = "requests" -version = "2.32.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847 }, -] - -[[package]] -name = "semver" -version = "3.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/d1/d3159231aec234a59dd7d601e9dd9fe96f3afff15efd33c1070019b26132/semver-3.0.4.tar.gz", hash = "sha256:afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602", size = 269730 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912 }, -] - -[[package]] -name = "typing-extensions" -version = "4.14.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906 }, -] - -[[package]] -name = "urllib3" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795 }, -] diff --git a/pulumi/AWSMegatests/variables.tf b/pulumi/AWSMegatests/variables.tf new file mode 100644 index 00000000..29cae833 --- /dev/null +++ b/pulumi/AWSMegatests/variables.tf @@ -0,0 +1,21 @@ +variable "aws_region" { + type = string + default = "eu-west-1" +} + +variable "github_owner" { + type = string + default = "nf-core" +} + +variable "seqera_org_id" { + type = number + default = 252464779077610 +} + +variable "op_vault" { + type = string + default = "Dev" +} + + diff --git a/pulumi/AWSMegatests/versions.tf b/pulumi/AWSMegatests/versions.tf new file mode 100644 index 00000000..533995d3 --- /dev/null +++ b/pulumi/AWSMegatests/versions.tf @@ -0,0 +1,26 @@ +terraform { + required_version = ">= 1.6.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + github = { + source = "integrations/github" + version = "~> 6.0" + } + seqera = { + source = "registry.terraform.io/seqeralabs/seqera" + version = "~> 0.25" + } + onepassword = { + source = "1Password/onepassword" + version = "~> 2.0" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/pulumi/AWSMegatests/workspace.tf b/pulumi/AWSMegatests/workspace.tf new file mode 100644 index 00000000..14c9d279 --- /dev/null +++ b/pulumi/AWSMegatests/workspace.tf @@ -0,0 +1,12 @@ +import { + to = seqera_workspace.dev + id = "{\"id\": 85085054091044, \"org_id\": 252464779077610}" +} + +resource "seqera_workspace" "dev" { + name = "AWSmegatests-dev" + full_name = "AWS Megatests (dev)" + description = "Development workspace for AWS megatests — launches from dev branch" + org_id = var.seqera_org_id + visibility = "SHARED" +}