From 0985cfcf906c55d594378cc32280dd08e432dabf Mon Sep 17 00:00:00 2001 From: "Perry Z." Date: Sun, 2 Aug 2026 03:36:08 -0700 Subject: [PATCH] feat: LLM Proxy --- .../_modules/cli-proxy-api/README.md | 29 +++ .../_modules/cli-proxy-api/bootstrap.sh.tftpl | 170 ++++++++++++++++++ apps/llm-proxy/_modules/cli-proxy-api/main.tf | 69 +++++++ .../_modules/cli-proxy-api/outputs.tf | 14 ++ .../_modules/cli-proxy-api/variables.tf | 39 ++++ .../_modules/cli-proxy-api/versions.tf | 8 + apps/llm-proxy/usw2dev/main.tf | 16 ++ apps/llm-proxy/usw2dev/moved.tf | 29 +++ apps/llm-proxy/usw2dev/outputs.tf | 24 +++ apps/llm-proxy/usw2dev/providers.tf | 10 ++ apps/llm-proxy/usw2dev/remote_backend.tf | 19 ++ apps/llm-proxy/usw2dev/remote_states.tf | 10 ++ apps/llm-proxy/usw2dev/terraform.auto.tfvars | 1 + apps/llm-proxy/usw2dev/variables.tf | 67 +++++++ 14 files changed, 505 insertions(+) create mode 100644 apps/llm-proxy/_modules/cli-proxy-api/README.md create mode 100644 apps/llm-proxy/_modules/cli-proxy-api/bootstrap.sh.tftpl create mode 100644 apps/llm-proxy/_modules/cli-proxy-api/main.tf create mode 100644 apps/llm-proxy/_modules/cli-proxy-api/outputs.tf create mode 100644 apps/llm-proxy/_modules/cli-proxy-api/variables.tf create mode 100644 apps/llm-proxy/_modules/cli-proxy-api/versions.tf create mode 100644 apps/llm-proxy/usw2dev/main.tf create mode 100644 apps/llm-proxy/usw2dev/moved.tf create mode 100644 apps/llm-proxy/usw2dev/outputs.tf create mode 100644 apps/llm-proxy/usw2dev/providers.tf create mode 100644 apps/llm-proxy/usw2dev/remote_backend.tf create mode 100644 apps/llm-proxy/usw2dev/remote_states.tf create mode 100644 apps/llm-proxy/usw2dev/terraform.auto.tfvars create mode 100644 apps/llm-proxy/usw2dev/variables.tf diff --git a/apps/llm-proxy/_modules/cli-proxy-api/README.md b/apps/llm-proxy/_modules/cli-proxy-api/README.md new file mode 100644 index 0000000..f59dda7 --- /dev/null +++ b/apps/llm-proxy/_modules/cli-proxy-api/README.md @@ -0,0 +1,29 @@ +# LLM Proxy +- [CLIProxyAPI](https://github.com/router-for-me/CLIProxyAPI) +- [Caddy](https://caddyserver.com/) + +## First boot + +Retrieve the generated client key: + +```sh +sudo cat /var/lib/cliproxyapi/api-key +``` + +Test authentication locally on the server: + +```sh +API_KEY="$(sudo cat /var/lib/cliproxyapi/api-key)" +curl -fsS -H "Authorization: Bearer $API_KEY" http://127.0.0.1:8317/v1/models +unset API_KEY +``` + +## Provider login + +Provider callback ports remain closed in the Lightsail firewall, thus the login must be done through an SSH tunnel. + +```sh +sudo -u cliproxy -H /opt/cliproxyapi/cli-proxy-api --config /etc/cliproxyapi/config.yaml -no-browser --codex-login +``` + +Replace `--codex-login` with `--claude-login` or `--antigravity-login` when needed. Credentials are written to `/var/lib/cliproxyapi/auth` and detected by the running service. diff --git a/apps/llm-proxy/_modules/cli-proxy-api/bootstrap.sh.tftpl b/apps/llm-proxy/_modules/cli-proxy-api/bootstrap.sh.tftpl new file mode 100644 index 0000000..5c35183 --- /dev/null +++ b/apps/llm-proxy/_modules/cli-proxy-api/bootstrap.sh.tftpl @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec > >(tee -a /var/log/cliproxy-bootstrap.log | logger -t cliproxy-bootstrap -s 2>/dev/console) 2>&1 + +export DEBIAN_FRONTEND=noninteractive + +if ! swapon --show=NAME --noheadings | grep -qx '/swapfile'; then + if [ ! -f /swapfile ]; then + fallocate -l 1G /swapfile + chmod 600 /swapfile + mkswap /swapfile + fi + swapon /swapfile +fi +grep -q '^/swapfile ' /etc/fstab || echo '/swapfile none swap sw 0 0' >> /etc/fstab + +timedatectl set-timezone America/Vancouver + +apt-get update +apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl debian-archive-keyring debian-keyring gnupg openssl + +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' \ + | gpg --dearmor --yes -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' \ + > /etc/apt/sources.list.d/caddy-stable.list +chmod o+r /usr/share/keyrings/caddy-stable-archive-keyring.gpg /etc/apt/sources.list.d/caddy-stable.list +apt-get update +apt-get install -y --no-install-recommends caddy + +if ! id cliproxy >/dev/null 2>&1; then + useradd --system --home-dir /var/lib/cliproxyapi --create-home --shell /usr/sbin/nologin cliproxy +fi + +install -d -o root -g root -m 0755 /opt/cliproxyapi +install -d -o root -g cliproxy -m 0750 /etc/cliproxyapi +install -d -o cliproxy -g cliproxy -m 0750 /var/lib/cliproxyapi/auth + +cat > /usr/local/sbin/update-cliproxyapi <<'UPDATE_SCRIPT' +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 VERSION_WITHOUT_LEADING_V" >&2 + exit 2 +fi + +VERSION="$${1#v}" +case "$(uname -m)" in + x86_64) ARCH="amd64" ;; + aarch64|arm64) ARCH="aarch64" ;; + *) echo "Unsupported architecture: $(uname -m)" >&2; exit 1 ;; +esac + +ARCHIVE="CLIProxyAPI_$${VERSION}_linux_$${ARCH}.tar.gz" +RELEASE_URL="https://github.com/router-for-me/CLIProxyAPI/releases/download/v$${VERSION}" +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$${WORK_DIR}"' EXIT + +curl -fsSL "$${RELEASE_URL}/checksums.txt" -o "$${WORK_DIR}/checksums.txt" +curl -fsSL "$${RELEASE_URL}/$${ARCHIVE}" -o "$${WORK_DIR}/$${ARCHIVE}" +( + cd "$${WORK_DIR}" + grep " $${ARCHIVE}$" checksums.txt | sha256sum --check --strict - + tar -xzf "$${ARCHIVE}" +) + +install -o root -g root -m 0755 "$${WORK_DIR}/cli-proxy-api" /opt/cliproxyapi/cli-proxy-api.new +mv -f /opt/cliproxyapi/cli-proxy-api.new /opt/cliproxyapi/cli-proxy-api + +if systemctl is-active --quiet cliproxyapi.service; then + systemctl restart cliproxyapi.service +fi +UPDATE_SCRIPT +chmod 0755 /usr/local/sbin/update-cliproxyapi +# Terraform expands this template expression before the script runs. +# shellcheck disable=SC2016 +/usr/local/sbin/update-cliproxyapi '${cliproxyapi_version}' + +umask 077 +if [ ! -s /var/lib/cliproxyapi/api-key ]; then + openssl rand -hex 32 > /var/lib/cliproxyapi/api-key +fi +# Terraform converts $${API_KEY} below to a shell expansion in the rendered script. +# shellcheck disable=SC2034 +API_KEY="$(cat /var/lib/cliproxyapi/api-key)" + +cat > /etc/cliproxyapi/config.yaml < /etc/systemd/system/cliproxyapi.service <<'SERVICE' +[Unit] +Description=CLIProxyAPI +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +User=cliproxy +Group=cliproxy +WorkingDirectory=/var/lib/cliproxyapi +ExecStart=/opt/cliproxyapi/cli-proxy-api --config /etc/cliproxyapi/config.yaml +Restart=on-failure +RestartSec=5s +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +ReadWritePaths=/var/lib/cliproxyapi +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +[Install] +WantedBy=multi-user.target +SERVICE + +cat > /etc/caddy/Caddyfile <<'CADDYFILE' +${server_hostname} { + reverse_proxy 127.0.0.1:8317 +} +CADDYFILE +chown root:caddy /etc/caddy/Caddyfile +chmod 0640 /etc/caddy/Caddyfile + +install -d -m 0755 /etc/systemd/journald.conf.d +cat > /etc/systemd/journald.conf.d/cliproxyapi.conf <<'JOURNALD' +[Journal] +SystemMaxUse=100M +RuntimeMaxUse=50M +MaxRetentionSec=7day +JOURNALD + +systemctl daemon-reload +systemctl restart systemd-journald +systemctl enable --now cliproxyapi.service +caddy validate --config /etc/caddy/Caddyfile +systemctl enable caddy.service +systemctl restart caddy.service diff --git a/apps/llm-proxy/_modules/cli-proxy-api/main.tf b/apps/llm-proxy/_modules/cli-proxy-api/main.tf new file mode 100644 index 0000000..1ebe2cf --- /dev/null +++ b/apps/llm-proxy/_modules/cli-proxy-api/main.tf @@ -0,0 +1,69 @@ +resource "aws_lightsail_instance" "proxy" { + name = var.instance_name + availability_zone = var.availability_zone + blueprint_id = var.lightsail_blueprint + bundle_id = var.lightsail_bundle + ip_address_type = "dualstack" + + user_data = templatefile("${path.module}/bootstrap.sh.tftpl", { + cliproxyapi_version = var.cliproxyapi_version + server_hostname = var.hostname + }) + + tags = { + Name = var.instance_name + } +} + +resource "aws_lightsail_static_ip" "proxy" { + name = var.instance_name +} + +resource "aws_lightsail_static_ip_attachment" "proxy" { + static_ip_name = aws_lightsail_static_ip.proxy.name + instance_name = aws_lightsail_instance.proxy.name +} + +resource "aws_lightsail_instance_public_ports" "proxy" { + instance_name = aws_lightsail_instance.proxy.name + + port_info { + protocol = "tcp" + from_port = 22 + to_port = 22 + cidrs = [var.admin_ipv4_cidr] + ipv6_cidrs = [] + } + + port_info { + protocol = "tcp" + from_port = 80 + to_port = 80 + cidrs = ["0.0.0.0/0"] + ipv6_cidrs = ["::/0"] + } + + port_info { + protocol = "tcp" + from_port = 443 + to_port = 443 + cidrs = ["0.0.0.0/0"] + ipv6_cidrs = ["::/0"] + } +} + +resource "aws_route53_record" "proxy_ipv4" { + zone_id = var.route53_hosted_zone + name = var.hostname + type = "A" + ttl = 60 + records = [aws_lightsail_static_ip.proxy.ip_address] +} + +resource "aws_route53_record" "proxy_ipv6" { + zone_id = var.route53_hosted_zone + name = var.hostname + type = "AAAA" + ttl = 60 + records = [aws_lightsail_instance.proxy.ipv6_addresses[0]] +} diff --git a/apps/llm-proxy/_modules/cli-proxy-api/outputs.tf b/apps/llm-proxy/_modules/cli-proxy-api/outputs.tf new file mode 100644 index 0000000..c6bbb52 --- /dev/null +++ b/apps/llm-proxy/_modules/cli-proxy-api/outputs.tf @@ -0,0 +1,14 @@ +output "static_ipv4_address" { + description = "Static public IPv4 address attached to the Lightsail instance" + value = aws_lightsail_static_ip.proxy.ip_address +} + +output "ipv6_address" { + description = "Stable public IPv6 address assigned to the Lightsail instance" + value = aws_lightsail_instance.proxy.ipv6_addresses[0] +} + +output "instance_username" { + description = "Default SSH username from the selected Lightsail blueprint" + value = aws_lightsail_instance.proxy.username +} diff --git a/apps/llm-proxy/_modules/cli-proxy-api/variables.tf b/apps/llm-proxy/_modules/cli-proxy-api/variables.tf new file mode 100644 index 0000000..b875bff --- /dev/null +++ b/apps/llm-proxy/_modules/cli-proxy-api/variables.tf @@ -0,0 +1,39 @@ +variable "instance_name" { + description = "Name assigned to the Lightsail instance and static IP" + type = string +} + +variable "availability_zone" { + description = "Lightsail availability zone" + type = string +} + +variable "lightsail_blueprint" { + description = "Lightsail operating-system blueprint ID" + type = string +} + +variable "lightsail_bundle" { + description = "Lightsail compute bundle ID" + type = string +} + +variable "admin_ipv4_cidr" { + description = "Trusted administrator IPv4 CIDR allowed to connect over SSH" + type = string +} + +variable "cliproxyapi_version" { + description = "Pinned CLIProxyAPI release version without the leading v" + type = string +} + +variable "hostname" { + description = "Public hostname served by Caddy" + type = string +} + +variable "route53_hosted_zone" { + description = "Route 53 hosted zone ID in which to create the hostname" + type = string +} diff --git a/apps/llm-proxy/_modules/cli-proxy-api/versions.tf b/apps/llm-proxy/_modules/cli-proxy-api/versions.tf new file mode 100644 index 0000000..ba9cec1 --- /dev/null +++ b/apps/llm-proxy/_modules/cli-proxy-api/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } +} diff --git a/apps/llm-proxy/usw2dev/main.tf b/apps/llm-proxy/usw2dev/main.tf new file mode 100644 index 0000000..4bf307b --- /dev/null +++ b/apps/llm-proxy/usw2dev/main.tf @@ -0,0 +1,16 @@ +locals { + fqdn = "${var.subdomain_name}.${data.terraform_remote_state.dns.outputs.domain_name}" +} + +module "cli_proxy_api" { + source = "../_modules/cli-proxy-api" + + instance_name = "${var.project_name}-${var.env_name}" + availability_zone = var.availability_zone + lightsail_blueprint = var.lightsail_blueprint_id + lightsail_bundle = var.lightsail_bundle_id + admin_ipv4_cidr = var.admin_ipv4_cidr + cliproxyapi_version = var.cliproxyapi_version + hostname = local.fqdn + route53_hosted_zone = data.terraform_remote_state.dns.outputs.domain_hosted_zone_id +} diff --git a/apps/llm-proxy/usw2dev/moved.tf b/apps/llm-proxy/usw2dev/moved.tf new file mode 100644 index 0000000..177666e --- /dev/null +++ b/apps/llm-proxy/usw2dev/moved.tf @@ -0,0 +1,29 @@ +moved { + from = aws_lightsail_instance.proxy + to = module.cli_proxy_api.aws_lightsail_instance.proxy +} + +moved { + from = aws_lightsail_static_ip.proxy + to = module.cli_proxy_api.aws_lightsail_static_ip.proxy +} + +moved { + from = aws_lightsail_static_ip_attachment.proxy + to = module.cli_proxy_api.aws_lightsail_static_ip_attachment.proxy +} + +moved { + from = aws_lightsail_instance_public_ports.proxy + to = module.cli_proxy_api.aws_lightsail_instance_public_ports.proxy +} + +moved { + from = aws_route53_record.proxy_ipv4 + to = module.cli_proxy_api.aws_route53_record.proxy_ipv4 +} + +moved { + from = aws_route53_record.proxy_ipv6 + to = module.cli_proxy_api.aws_route53_record.proxy_ipv6 +} diff --git a/apps/llm-proxy/usw2dev/outputs.tf b/apps/llm-proxy/usw2dev/outputs.tf new file mode 100644 index 0000000..d0d9e2c --- /dev/null +++ b/apps/llm-proxy/usw2dev/outputs.tf @@ -0,0 +1,24 @@ +output "api_base_url" { + description = "OpenAI-compatible API base URL" + value = "https://${local.fqdn}/v1" +} + +output "hostname" { + description = "Public proxy hostname" + value = local.fqdn +} + +output "static_ipv4_address" { + description = "Static public IPv4 address attached to the Lightsail instance" + value = module.cli_proxy_api.static_ipv4_address +} + +output "ipv6_address" { + description = "Stable public IPv6 address assigned to the Lightsail instance" + value = module.cli_proxy_api.ipv6_address +} + +output "ssh_command" { + description = "SSH command using the Lightsail default regional private key" + value = "ssh -i ~/.ssh/LightsailDefaultKey-${var.aws_region}.pem ${module.cli_proxy_api.instance_username}@${module.cli_proxy_api.static_ipv4_address}" +} diff --git a/apps/llm-proxy/usw2dev/providers.tf b/apps/llm-proxy/usw2dev/providers.tf new file mode 100644 index 0000000..453d458 --- /dev/null +++ b/apps/llm-proxy/usw2dev/providers.tf @@ -0,0 +1,10 @@ +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Project = var.project_name + Env = var.env_name + } + } +} diff --git a/apps/llm-proxy/usw2dev/remote_backend.tf b/apps/llm-proxy/usw2dev/remote_backend.tf new file mode 100644 index 0000000..3e4ea00 --- /dev/null +++ b/apps/llm-proxy/usw2dev/remote_backend.tf @@ -0,0 +1,19 @@ +terraform { + required_version = ">= 1.9.0" + + cloud { + organization = "perry-zhu-aws" + + workspaces { + project = "aws" + name = "llm-proxy-usw2dev" + } + } + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } +} diff --git a/apps/llm-proxy/usw2dev/remote_states.tf b/apps/llm-proxy/usw2dev/remote_states.tf new file mode 100644 index 0000000..974a0c0 --- /dev/null +++ b/apps/llm-proxy/usw2dev/remote_states.tf @@ -0,0 +1,10 @@ +data "terraform_remote_state" "dns" { + backend = "remote" + + config = { + organization = "perry-zhu-aws" + workspaces = { + name = "common-dns" + } + } +} diff --git a/apps/llm-proxy/usw2dev/terraform.auto.tfvars b/apps/llm-proxy/usw2dev/terraform.auto.tfvars new file mode 100644 index 0000000..4955a76 --- /dev/null +++ b/apps/llm-proxy/usw2dev/terraform.auto.tfvars @@ -0,0 +1 @@ +env_name = "usw2dev" diff --git a/apps/llm-proxy/usw2dev/variables.tf b/apps/llm-proxy/usw2dev/variables.tf new file mode 100644 index 0000000..bae9860 --- /dev/null +++ b/apps/llm-proxy/usw2dev/variables.tf @@ -0,0 +1,67 @@ +variable "aws_region" { + description = "AWS region in which to deploy the proxy" + type = string + default = "us-west-2" +} + +variable "availability_zone" { + description = "Lightsail availability zone" + type = string + default = "us-west-2a" +} + +variable "env_name" { + description = "Environment name" + type = string + default = "usw2dev" +} + +variable "project_name" { + description = "Project name used for AWS tags and resource names" + type = string + default = "llm-proxy" +} + +variable "subdomain_name" { + description = "Subdomain created beneath the shared Route 53 zone" + type = string + default = "llm" +} + +variable "lightsail_blueprint_id" { + description = "Lightsail operating-system blueprint" + type = string + default = "ubuntu_24_04" +} + +variable "lightsail_bundle_id" { + description = "Lightsail bundle; use micro_3_0 if the Nano instance is memory constrained" + type = string + default = "nano_3_0" + + validation { + condition = contains(["nano_3_0", "micro_3_0"], var.lightsail_bundle_id) + error_message = "lightsail_bundle_id must be nano_3_0 or micro_3_0." + } +} + +variable "admin_ipv4_cidr" { + description = "Trusted administrator IPv4 CIDR allowed to connect over SSH" + type = string + + validation { + condition = can(cidrnetmask(var.admin_ipv4_cidr)) + error_message = "admin_ipv4_cidr must be a valid IPv4 CIDR, for example 203.0.113.10/32." + } +} + +variable "cliproxyapi_version" { + description = "Pinned CLIProxyAPI release version without the leading v" + type = string + default = "7.2.113" + + validation { + condition = can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", var.cliproxyapi_version)) + error_message = "cliproxyapi_version must be a semantic version without a leading v." + } +}