diff --git a/packages/queue/composer.json b/packages/queue/composer.json index 6c1d88ef2..652c50b9f 100644 --- a/packages/queue/composer.json +++ b/packages/queue/composer.json @@ -48,7 +48,8 @@ }, "suggest": { "ext-swoole": "Needed to support Swoole.", - "workerman/workerman": "Needed to support Workerman." + "workerman/workerman": "Needed to support Workerman.", + "ext-pcntl": "Fallback for Adapter\\KubernetesJob graceful shutdown when Swoole is unavailable." }, "config": { "allow-plugins": { diff --git a/packages/queue/phpunit.xml b/packages/queue/phpunit.xml index 64067c8e4..d5ffb364f 100644 --- a/packages/queue/phpunit.xml +++ b/packages/queue/phpunit.xml @@ -6,12 +6,14 @@ > + ./tests/Queue/E2E/Adapter/KubernetesJobAdapterTest.php ./tests/Queue/E2E/Adapter/LockingTest.php ./tests/Queue/E2E/Adapter/RedisReconnectCallbackTest.php ./tests/Queue/E2E/Adapter/ServerTelemetryTest.php ./tests/Queue/E2E/Adapter/SwooleConcurrencyTest.php + ./tests/Queue/E2E/Adapter/KedaTest.php ./tests/Queue/E2E/Adapter/PoolTest.php ./tests/Queue/E2E/Adapter/SwooleTest.php ./tests/Queue/E2E/Adapter/SwooleRedisClusterTest.php diff --git a/packages/queue/src/Queue/Adapter.php b/packages/queue/src/Queue/Adapter.php index 6989bced0..124e77e20 100644 --- a/packages/queue/src/Queue/Adapter.php +++ b/packages/queue/src/Queue/Adapter.php @@ -82,6 +82,11 @@ public function resources(): Container return $this->resources; } + public function runsToCompletion(): bool + { + return false; + } + public function context(): Container { return $this->context ??= new Container($this->resources()); diff --git a/packages/queue/src/Queue/Adapter/KubernetesJob.php b/packages/queue/src/Queue/Adapter/KubernetesJob.php new file mode 100644 index 000000000..97867bfb0 --- /dev/null +++ b/packages/queue/src/Queue/Adapter/KubernetesJob.php @@ -0,0 +1,142 @@ +onWorkerStart as $callback) { + $callback('0'); + } + } finally { + foreach ($this->onWorkerStop as $callback) { + $callback('0'); + } + } + }; + + if (!\extension_loaded('swoole') || Coroutine::getCid() >= 0) { + $lifecycle(); + + return $this; + } + + Coroutine::set(['hook_flags' => SWOOLE_HOOK_ALL]); + + $error = null; + + Coroutine\run(function () use (&$error, $lifecycle): void { + try { + $lifecycle(); + } catch (\Throwable $thrown) { + $error = $thrown; + } + }); + + if ($error instanceof \Throwable) { + throw $error; + } + + return $this; + } + + public function stop(): self + { + $this->stopped = true; + $this->consumer->close(); + + return $this; + } + + #[\Override] + public function runsToCompletion(): bool + { + return true; + } + + /** + * Drain the queue, then return. Processes messages until a receive() times + * out (the queue is empty) or stop() is called, so the Job completes rather + * than blocking forever like the long-running adapters. + */ + #[\Override] + public function consume(callable $messageCallback, callable $successCallback, callable $errorCallback): void + { + $this->stopped = false; + + $swoole = \extension_loaded('swoole'); + + if ($swoole) { + Process::signal(SIGTERM, fn(): \Utopia\Queue\Adapter\KubernetesJob => $this->stop()); + Process::signal(SIGINT, fn(): \Utopia\Queue\Adapter\KubernetesJob => $this->stop()); + } elseif (\function_exists('pcntl_async_signals')) { + pcntl_async_signals(true); + pcntl_signal(SIGTERM, $this->stop(...)); + pcntl_signal(SIGINT, $this->stop(...)); + } + + try { + while (!$this->isStopped()) { + $message = $this->consumer->receive($this->queue, static::RECEIVE_TIMEOUT); + + if (!$message instanceof Message) { + break; + } + + $this->context = new Container($this->resources()); + $this->process($message, $messageCallback, $successCallback, $errorCallback); + } + } finally { + if ($swoole) { + Process::signal(SIGTERM, null); + Process::signal(SIGINT, null); + } + } + } + + public function workerStart(callable $callback): self + { + $this->onWorkerStart[] = $callback; + + return $this; + } + + public function workerStop(callable $callback): self + { + $this->onWorkerStop[] = $callback; + + return $this; + } +} diff --git a/packages/queue/src/Queue/Server.php b/packages/queue/src/Queue/Server.php index 52f1e024f..c2a54d76b 100644 --- a/packages/queue/src/Queue/Server.php +++ b/packages/queue/src/Queue/Server.php @@ -342,6 +342,10 @@ function (?Message $message, Throwable $th): void { foreach ($this->errorHooks as $hook) { $hook->getAction()(...$this->getArguments($this->resources(), $hook)); } + + if ($this->adapter->runsToCompletion()) { + throw $error; + } } return $this; } diff --git a/packages/queue/tests/Queue/E2E/Adapter/KedaTest.php b/packages/queue/tests/Queue/E2E/Adapter/KedaTest.php new file mode 100644 index 000000000..b9eaadf92 --- /dev/null +++ b/packages/queue/tests/Queue/E2E/Adapter/KedaTest.php @@ -0,0 +1,102 @@ +markTestSkipped('KEDA e2e requires the kind + KEDA harness (KEDA_E2E=true).'); + } + } + + private function redis(string ...$args): string + { + $command = 'kubectl exec -n ' . self::NAMESPACE . ' deploy/redis -- redis-cli ' + . implode(' ', array_map(escapeshellarg(...), $args)) . ' 2>/dev/null'; + + return trim((string) shell_exec($command)); + } + + /** @phpstan-impure reads live Redis state, so the value changes between calls */ + private function queueLength(): int + { + return (int) $this->redis('LLEN', self::QUEUE_KEY); + } + + /** @phpstan-impure reads live Redis state, so the value changes between calls */ + private function failedLength(): int + { + return (int) $this->redis('LLEN', self::FAILED_KEY); + } + + private function jobCount(): int + { + $out = shell_exec('kubectl get jobs -n ' . self::NAMESPACE . ' --no-headers 2>/dev/null'); + if ($out === null || trim($out) === '') { + return 0; + } + + return \count(explode("\n", trim($out))); + } + + private function enqueue(array $payload): void + { + // Mirrors Redis::enqueue()'s envelope. The in-cluster Redis isn't exposed + // to the host, so we push via kubectl rather than the broker directly; + // keep this in sync if the envelope shape changes. + $message = json_encode([ + 'pid' => uniqid(more_entropy: true), + 'queue' => 'keda', + 'timestamp' => time(), + 'payload' => $payload, + ], JSON_THROW_ON_ERROR); + + $this->redis('LPUSH', self::QUEUE_KEY, $message); + } + + public function testKedaSpawnsJobsThatDrainTheQueue(): void + { + $payloads = [ + ['type' => 'test_string', 'value' => 'lorem ipsum'], + ['type' => 'test_number', 'value' => 123], + ['type' => 'test_assoc', 'value' => ['string' => 'ipsum', 'number' => 123, 'bool' => true, 'null' => null]], + ]; + + foreach ($payloads as $payload) { + $this->enqueue($payload); + } + + $this->assertSame(\count($payloads), $this->queueLength(), 'messages should be queued in Redis'); + + $deadline = time() + self::TIMEOUT; + while (time() < $deadline && $this->queueLength() > 0) { + sleep(3); + } + + $this->assertSame(0, $this->queueLength(), 'KEDA-spawned workers should drain the queue'); + // receive() pops from the main list before handling, so a drained main + // queue alone doesn't prove success — assert nothing landed in .failed.*. + $this->assertSame(0, $this->failedLength(), 'no messages should have failed'); + $this->assertGreaterThanOrEqual(1, $this->jobCount(), 'KEDA should have spawned at least one worker Job'); + } +} diff --git a/packages/queue/tests/Queue/E2E/Adapter/KubernetesJobAdapterTest.php b/packages/queue/tests/Queue/E2E/Adapter/KubernetesJobAdapterTest.php new file mode 100644 index 000000000..09e91fcf4 --- /dev/null +++ b/packages/queue/tests/Queue/E2E/Adapter/KubernetesJobAdapterTest.php @@ -0,0 +1,85 @@ +job()->inject('message')->action($action); + + return $server; + } + + public function testDrainsQueueThenReturns(): void + { + $connection = new InMemoryConnection(); + $broker = new Redis($connection, $connection); + $queue = new Queue(self::QUEUE, self::NAMESPACE); + + foreach (range(1, 5) as $n) { + $broker->enqueue($queue, ['n' => $n]); + } + + $processed = []; + $this->server($broker, function (Message $message) use (&$processed): void { + $processed[] = $message->getPayload()['n']; + })->start(); + + $this->assertSame([1, 2, 3, 4, 5], $processed, 'every queued message is processed once, in order'); + $this->assertSame(0, $broker->getQueueSize($queue), 'the queue is drained'); + } + + public function testReturnsImmediatelyWhenQueueEmpty(): void + { + $connection = new InMemoryConnection(); + $broker = new Redis($connection, $connection); + + $processed = 0; + $this->server($broker, function () use (&$processed): void { + $processed++; + })->start(); + + $this->assertSame(0, $processed, 'an empty queue processes nothing and the worker exits'); + } + + public function testFailedMessageIsRejectedAndDrainContinues(): void + { + $connection = new InMemoryConnection(); + $broker = new Redis($connection, $connection); + $queue = new Queue(self::QUEUE, self::NAMESPACE); + + $broker->enqueue($queue, ['ok' => false]); + $broker->enqueue($queue, ['ok' => true]); + + $succeeded = 0; + $this->server($broker, function (Message $message) use (&$succeeded): void { + if ($message->getPayload()['ok'] === false) { + throw new \RuntimeException('boom'); + } + $succeeded++; + })->start(); + + $this->assertSame(1, $succeeded, 'the drain continues past a failing message'); + $this->assertSame(0, $broker->getQueueSize($queue), 'the main queue is drained'); + $this->assertSame(1, $broker->getQueueSize($queue, failedJobs: true), 'the failed message lands on the failed queue'); + } +} diff --git a/packages/queue/tests/Queue/servers/Keda/Dockerfile b/packages/queue/tests/Queue/servers/Keda/Dockerfile new file mode 100644 index 000000000..10dd7b8d2 --- /dev/null +++ b/packages/queue/tests/Queue/servers/Keda/Dockerfile @@ -0,0 +1,16 @@ +FROM php:8.4-cli-alpine + +WORKDIR /app + +RUN apk add --no-cache --virtual .build-deps $PHPIZE_DEPS \ + && pecl install redis \ + && docker-php-ext-enable redis \ + && docker-php-ext-install pcntl \ + && apk del .build-deps + +COPY composer.json composer.lock ./ +COPY vendor ./vendor +COPY src ./src +COPY tests ./tests + +CMD ["php", "tests/Queue/servers/Keda/worker.php"] diff --git a/packages/queue/tests/Queue/servers/Keda/README.md b/packages/queue/tests/Queue/servers/Keda/README.md new file mode 100644 index 000000000..fd72baa2f --- /dev/null +++ b/packages/queue/tests/Queue/servers/Keda/README.md @@ -0,0 +1,62 @@ +# KEDA ScaledJob POC + +Runs queue jobs as Kubernetes Jobs the K8s-native way: the payload stays in the +**Redis queue**, and [KEDA](https://keda.sh) scales worker Jobs off the queue +depth. No custom broker, no payload on any Kubernetes object. + +Contrast with the env-var `KubernetesJob` broker (branch `feat/queue-kubernetes-job`), +which creates one Job per message with the payload inlined in an environment +variable. + +## How it works + +- Producers enqueue with the ordinary `Utopia\Queue\Broker\Redis` broker — no + Kubernetes involvement. +- A KEDA `ScaledJob` (`k8s.yaml`) watches the queue's Redis list length + (`{namespace}.queue.{name}`) and spawns worker Jobs, up to `maxReplicaCount`. +- Each worker (`worker.php`) runs the `Utopia\Queue\Adapter\KubernetesJob` + adapter: a run-to-completion adapter that drains the queue with the same Redis + broker (`receive` → handle → `commit`) and returns, so the Job completes. + +The only library code the approach needs is that adapter (`src/Queue/Adapter/KubernetesJob.php`, +covered by `KubernetesJobAdapterTest`); everything else is deployment config. + +## Run it + +```sh +tests/keda-e2e.sh # needs docker + kind + kubectl + helm +``` + +It stands up kind, installs KEDA, deploys Redis + the ScaledJob, loads the worker +image, then runs `KedaTest`, which enqueues messages and asserts KEDA spawns Jobs +that drain the queue. + +## env-var `KubernetesJob` vs KEDA `ScaledJob` + +| | env-var KubernetesJob | KEDA ScaledJob | +|---|---|---| +| Payload | inlined in the Job's env var — etcd (~1.5 MB) / `ARG_MAX` limits, visible in the pod spec, duplicated per Pod | stays in Redis; never on a K8s object | +| Custom broker | yes (`KubernetesJob` publisher) | none — reuses the `Redis` broker | +| Producer needs cluster access | yes (creates Jobs) | no (just Redis) | +| Scaling | one Job per message | KEDA scales N Jobs off queue depth; workers batch-drain | +| Extra dependency | `appwrite-labs/php-k8s` in the app | KEDA operator in the cluster | +| Secrets/PII in payload | exposed via pod spec | fine (stays in Redis) | + +Trade-off: KEDA needs its operator installed in the cluster, but it keeps the +queue a queue (payload in Redis) and is the de-facto standard for event-driven +Kubernetes Jobs — which is why it's the recommended path. + +> Note: the ScaledJob trigger `address` must be the Redis Service FQDN +> (`redis..svc.cluster.local:6379`) — KEDA evaluates triggers from the +> `keda` namespace, so a short name won't resolve to the workload's namespace. + +## Crash recovery + +`receive()` claims a message by atomically popping it from the main queue into +the broker's processing list before the worker handles it. If a worker pod is +hard-killed (OOM, node eviction) after the claim but before `commit`/`reject`, +that message is stranded in the processing list — it's no longer in the main +queue, so KEDA won't spawn a Job for it, and `backoffLimit` can't help (a new pod +only drains the main queue). Production deployments should run a periodic reaper +(`Publisher::retry()`) to requeue stale processing entries. This POC does not +configure one. diff --git a/packages/queue/tests/Queue/servers/Keda/k8s.yaml b/packages/queue/tests/Queue/servers/Keda/k8s.yaml new file mode 100644 index 000000000..35040e02e --- /dev/null +++ b/packages/queue/tests/Queue/servers/Keda/k8s.yaml @@ -0,0 +1,83 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: utopia-queue-keda +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: utopia-queue-keda +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: utopia-queue-keda +spec: + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 +--- +# KEDA scales worker Jobs off the Redis queue depth (up to maxReplicaCount); +# each Job pulls from Redis and drains messages until the queue is empty, then +# exits. No payload is carried on any Kubernetes object. +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: queue-worker + namespace: utopia-queue-keda +spec: + pollingInterval: 5 + maxReplicaCount: 5 + successfulJobsHistoryLimit: 20 + failedJobsHistoryLimit: 20 + jobTargetRef: + parallelism: 1 + completions: 1 + # The worker always exits 0 (it rejects failed messages and continues), so + # backoffLimit rarely fires. Note: if a pod is hard-killed after receive() + # claims a message but before commit/reject, that message is stranded in the + # broker's processing list (not the main queue), so KEDA won't re-trigger it; + # production needs a periodic reaper (Publisher::retry) to requeue stale + # processing entries. See servers/Keda/README.md. + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: worker + image: utopia-queue-keda-worker:e2e + imagePullPolicy: Never + env: + - name: REDIS_HOST + value: redis + - name: QUEUE_NAME + value: keda + - name: QUEUE_NAMESPACE + value: utopia-queue + triggers: + - type: redis + metadata: + # FQDN: KEDA evaluates the trigger from the keda namespace, so a short + # name won't resolve to this namespace's Redis Service. + address: redis.utopia-queue-keda.svc.cluster.local:6379 + listName: utopia-queue.queue.keda + listLength: "1" diff --git a/packages/queue/tests/Queue/servers/Keda/worker.php b/packages/queue/tests/Queue/servers/Keda/worker.php new file mode 100644 index 000000000..7696739e1 --- /dev/null +++ b/packages/queue/tests/Queue/servers/Keda/worker.php @@ -0,0 +1,29 @@ +job() + ->inject('message') + ->action(fn(Message $message) => handleRequest($message)); +$server->error() + ->inject('error') + ->action(fn(\Throwable $error): int|false => fwrite(STDERR, $error->getMessage() . "\n")); +$server->start(); diff --git a/packages/queue/tests/e2e.sh b/packages/queue/tests/e2e.sh index d10185a74..edb288f69 100755 --- a/packages/queue/tests/e2e.sh +++ b/packages/queue/tests/e2e.sh @@ -1,9 +1,12 @@ #!/bin/sh -# Run the e2e suite against local workers. Expects the compose services -# (redis, redis-cluster) to be up — see docker-compose.yml. +# Run the e2e suite against local workers plus a KEDA-driven kind cluster. +# Expects the compose services (redis, redis-cluster) to be up — see +# docker-compose.yml — and provisions kind + KEDA + the ScaledJob for KedaTest. set -e cd "$(dirname "$0")/.." +. tests/keda-lib.sh + php tests/Queue/servers/Swoole/worker.php & SWOOLE=$! php tests/Queue/servers/SwooleRedisCluster/worker.php & CLUSTER=$! php tests/Queue/servers/Workerman/worker.php start & WORKERMAN=$! @@ -11,9 +14,20 @@ php tests/Queue/servers/Workerman/worker.php start & WORKERMAN=$! cleanup() { kill -INT "$SWOOLE" "$CLUSTER" "$WORKERMAN" 2> /dev/null || true wait 2> /dev/null || true + keda_down } trap cleanup EXIT INT TERM +# KEDA is required in CI: under `set -e` any provisioning failure aborts so KEDA +# regressions surface instead of KedaTest silently skipping. Locally it's +# best-effort — without Docker the rest of the suite still runs and KedaTest +# skips itself. +if [ -n "$CI" ] || { command -v docker > /dev/null 2>&1 && docker info > /dev/null 2>&1; }; then + keda_up +else + echo "e2e.sh: Docker unavailable, skipping KEDA setup — KedaTest will skip" >&2 +fi + sleep 3 phpunit --testsuite e2e diff --git a/packages/queue/tests/keda-e2e.sh b/packages/queue/tests/keda-e2e.sh new file mode 100755 index 000000000..07fb8ff8c --- /dev/null +++ b/packages/queue/tests/keda-e2e.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Standalone KEDA ScaledJob e2e: stand up kind + KEDA + Redis + the ScaledJob, +# then run KedaTest, which pushes messages and asserts KEDA spawns worker Jobs +# that drain the queue. Needs docker + kind + kubectl + helm (auto-installed if +# missing). tests/e2e.sh runs the same setup as part of the full suite in CI. +set -e +cd "$(dirname "$0")/.." + +. tests/keda-lib.sh + +trap keda_down EXIT INT TERM +keda_up + +../../vendor/bin/phpunit tests/Queue/E2E/Adapter/KedaTest.php diff --git a/packages/queue/tests/keda-lib.sh b/packages/queue/tests/keda-lib.sh new file mode 100644 index 000000000..122eb56b8 --- /dev/null +++ b/packages/queue/tests/keda-lib.sh @@ -0,0 +1,75 @@ +# Sourced helpers to stand up / tear down the KEDA e2e environment (kind + KEDA + +# Redis + the ScaledJob). Call keda_up (exports KUBECONFIG + KEDA_E2E) then +# keda_down. Assumes cwd is the package root and vendor/ is installed (the worker +# image copies it). Missing CLI tools are downloaded at pinned, checksum-verified +# versions; kubectl/helm are usually already on CI runners. + +KIND_CLUSTER="${KIND_CLUSTER:-utopia-queue-keda}" +KUBECONFIG_FILE="${KUBECONFIG_FILE:-$(pwd)/.kubeconfig.keda}" +KEDA_TOOLS="${KEDA_TOOLS:-$(pwd)/.e2e-bin}" + +KIND_VERSION='v0.30.0' +KIND_SHA256='517ab7fc89ddeed5fa65abf71530d90648d9638ef0c4cde22c2c11f8097b8889' +KUBECTL_VERSION='v1.31.4' +KUBECTL_SHA256='298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f' +HELM_VERSION='v3.16.4' +HELM_SHA256='fc307327959aa38ed8f9f7e66d45492bb022a66c3e5da6063958254b9767d179' + +# Only tear down a cluster this script created, never a pre-existing one. +KEDA_CREATED_CLUSTER=0 + +keda_verify() { + # $1 = file, $2 = expected sha256 + echo "$2 $1" | sha256sum -c - > /dev/null 2>&1 || { echo "checksum mismatch for $1"; exit 1; } +} + +keda_up() { + mkdir -p "$KEDA_TOOLS" + PATH="$KEDA_TOOLS:$PATH" + + if ! command -v kind > /dev/null 2>&1; then + echo "installing kind $KIND_VERSION..." + curl -fsSL -o "$KEDA_TOOLS/kind" "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64" + keda_verify "$KEDA_TOOLS/kind" "$KIND_SHA256" + chmod +x "$KEDA_TOOLS/kind" + fi + if ! command -v kubectl > /dev/null 2>&1; then + echo "installing kubectl $KUBECTL_VERSION..." + curl -fsSL -o "$KEDA_TOOLS/kubectl" "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl" + keda_verify "$KEDA_TOOLS/kubectl" "$KUBECTL_SHA256" + chmod +x "$KEDA_TOOLS/kubectl" + fi + if ! command -v helm > /dev/null 2>&1; then + echo "installing helm $HELM_VERSION..." + curl -fsSL -o "$KEDA_TOOLS/helm.tar.gz" "https://get.helm.sh/helm-$HELM_VERSION-linux-amd64.tar.gz" + keda_verify "$KEDA_TOOLS/helm.tar.gz" "$HELM_SHA256" + tar -xz -C "$KEDA_TOOLS" --strip-components=1 -f "$KEDA_TOOLS/helm.tar.gz" linux-amd64/helm + rm -f "$KEDA_TOOLS/helm.tar.gz" + fi + + if ! kind get clusters 2> /dev/null | grep -qx "$KIND_CLUSTER"; then + kind create cluster --name "$KIND_CLUSTER" --wait 120s + KEDA_CREATED_CLUSTER=1 + fi + kind get kubeconfig --name "$KIND_CLUSTER" > "$KUBECONFIG_FILE" + export KUBECONFIG="$KUBECONFIG_FILE" + + helm repo add kedacore https://kedacore.github.io/charts > /dev/null 2>&1 || true + helm repo update kedacore > /dev/null + helm upgrade --install keda kedacore/keda --namespace keda --create-namespace --wait --timeout 5m + + docker build -t utopia-queue-keda-worker:e2e -f tests/Queue/servers/Keda/Dockerfile . + kind load docker-image utopia-queue-keda-worker:e2e --name "$KIND_CLUSTER" + + kubectl apply -f tests/Queue/servers/Keda/k8s.yaml + kubectl rollout status deploy/redis -n utopia-queue-keda --timeout=120s + + export KEDA_E2E=true +} + +keda_down() { + if [ "$KEDA_CREATED_CLUSTER" = "1" ]; then + kind delete cluster --name "$KIND_CLUSTER" > /dev/null 2>&1 || true + fi + rm -f "$KUBECONFIG_FILE" +}