diff --git a/packages/queue/phpunit.xml b/packages/queue/phpunit.xml index 64067c8e4..b72583673 100644 --- a/packages/queue/phpunit.xml +++ b/packages/queue/phpunit.xml @@ -6,6 +6,7 @@ > + ./tests/Queue/E2E/Adapter/BackgroundTest.php ./tests/Queue/E2E/Adapter/LockingTest.php ./tests/Queue/E2E/Adapter/RedisReconnectCallbackTest.php ./tests/Queue/E2E/Adapter/ServerTelemetryTest.php diff --git a/packages/queue/src/Queue/Broker/Background.php b/packages/queue/src/Queue/Broker/Background.php new file mode 100644 index 000000000..4d73807bb --- /dev/null +++ b/packages/queue/src/Queue/Broker/Background.php @@ -0,0 +1,165 @@ +channel = new Channel(max(1, $capacity)); + $this->waitGroup = new WaitGroup(); + $this->coroutines = max(1, $coroutines); + + $telemetry->createObservableGauge( + 'messaging.publisher.buffer.depth', + '{message}', + 'Publishes buffered awaiting background dispatch.', + )->observe(function (callable $observe): void { + $observe($this->channel->length(), []); + }); + } + + /** + * Spawn the reader coroutines that drain the channel into the wrapped + * publisher. Call once from within a coroutine runtime; until then + * enqueue() publishes synchronously. + */ + public function start(): void + { + if ($this->started) { + return; + } + + $this->started = true; + + for ($i = 0; $i < $this->coroutines; $i++) { + $this->waitGroup->add(); + + Coroutine::create(function (): void { + try { + while (($task = $this->channel->pop()) instanceof \Closure) { + $task(); + } + } finally { + $this->waitGroup->done(); + } + }); + } + } + + /** + * Drain the channel and stop the readers, blocking until they have finished. + * Messages already enqueued are published before the readers exit. + */ + public function shutdown(): void + { + if (!$this->started) { + return; + } + + for ($i = 0; $i < $this->coroutines; $i++) { + $this->channel->push(null); // one sentinel per reader; pop() returns non-Closure → loop ends + } + + $this->waitGroup->wait(); + $this->started = false; + } + + /** + * Publish synchronously, blocking until the broker accepts the message. + */ + public function publish(Queue $queue, array $payload, bool $priority = false): bool + { + return $this->publisher->publish($queue, $payload, $priority); + } + + /** + * Hand the publish to the background reader via the channel. Blocks when the + * channel is full (back pressure), up to the configured timeout, then throws + * BufferFullException if no slot frees in time. Falls back to a synchronous + * publish when no reader loop is running. + * + * @throws BufferFullException when the buffer stays full past the timeout. + */ + public function enqueue(Queue $queue, array $payload, bool $priority = false): void + { + if (!$this->started || Coroutine::getCid() === -1) { + $this->publish($queue, $payload, $priority); + + return; + } + + $accepted = $this->channel->push(function () use ($queue, $payload, $priority): void { + try { + $this->publisher->publish($queue, $payload, $priority); + } catch (\Throwable $error) { + // Fire-and-forget: no producer to surface to, so log and move on. + error_log('Uncaught error while publishing queue message: ' . $error->getMessage()); + } + }, $this->timeout); + + if ($accepted === false) { + throw new BufferFullException('Publisher buffer full; enqueue timed out.'); + } + } + + public function retry(Queue $queue, ?int $limit = null): void + { + $this->publisher->retry($queue, $limit); + } + + public function getQueueSize(Queue $queue, bool $failedJobs = false): int + { + return $this->publisher->getQueueSize($queue, $failedJobs); + } +} diff --git a/packages/queue/src/Queue/Broker/Pool.php b/packages/queue/src/Queue/Broker/Pool.php index 2f27d045f..8bff7ee0e 100644 --- a/packages/queue/src/Queue/Broker/Pool.php +++ b/packages/queue/src/Queue/Broker/Pool.php @@ -5,17 +5,17 @@ use Utopia\Pools\Pool as UtopiaPool; use Utopia\Queue\Consumer; use Utopia\Queue\Message; -use Utopia\Queue\Publisher; +use Utopia\Queue\Publisher\Synchronous; use Utopia\Queue\Queue; -readonly class Pool implements Publisher, Consumer +readonly class Pool implements Synchronous, Consumer { public function __construct( private ?UtopiaPool $publisher = null, private ?UtopiaPool $consumer = null, ) {} - public function enqueue(Queue $queue, array $payload, bool $priority = false): bool + public function publish(Queue $queue, array $payload, bool $priority = false): bool { return $this->delegate($this->publisher, __FUNCTION__, \func_get_args()); } @@ -55,6 +55,6 @@ public function close(): void */ protected function delegate(?UtopiaPool $pool, string $method, array $args): mixed { - return $pool?->use(fn(Publisher|Consumer $adapter) => $adapter->$method(...$args)); + return $pool?->use(fn(Synchronous|Consumer $adapter) => $adapter->$method(...$args)); } } diff --git a/packages/queue/src/Queue/Broker/Redis.php b/packages/queue/src/Queue/Broker/Redis.php index 65d38535b..0d26976df 100644 --- a/packages/queue/src/Queue/Broker/Redis.php +++ b/packages/queue/src/Queue/Broker/Redis.php @@ -5,10 +5,10 @@ use Utopia\Queue\Connection; use Utopia\Queue\Consumer; use Utopia\Queue\Message; -use Utopia\Queue\Publisher; +use Utopia\Queue\Publisher\Synchronous; use Utopia\Queue\Queue; -class Redis implements Publisher, Consumer +class Redis implements Synchronous, Consumer { private const int POP_TIMEOUT = 2; private const int RECONNECT_BACKOFF_MS = 100; @@ -155,7 +155,7 @@ private function triggerReconnectSuccessCallback(Queue $queue, int $attempts): v } } - public function enqueue(Queue $queue, array $payload, bool $priority = false): bool + public function publish(Queue $queue, array $payload, bool $priority = false): bool { $payload = [ 'pid' => uniqid(more_entropy: true), @@ -203,7 +203,7 @@ public function retry(Queue $queue, ?int $limit = null): void break; } - $this->enqueue($queue, $job->getPayload()); + $this->publish($queue, $job->getPayload()); $processed++; } } diff --git a/packages/queue/src/Queue/Publisher.php b/packages/queue/src/Queue/Publisher.php deleted file mode 100644 index 78ce4786a..000000000 --- a/packages/queue/src/Queue/Publisher.php +++ /dev/null @@ -1,23 +0,0 @@ -queueDepth->observe(function (callable $observe): void { - if (!$this->adapter->consumer instanceof Publisher) { + if (!$this->adapter->consumer instanceof Synchronous) { return; } diff --git a/packages/queue/tests/Queue/E2E/Adapter/BackgroundTest.php b/packages/queue/tests/Queue/E2E/Adapter/BackgroundTest.php new file mode 100644 index 000000000..0904ec9e7 --- /dev/null +++ b/packages/queue/tests/Queue/E2E/Adapter/BackgroundTest.php @@ -0,0 +1,212 @@ +recordingPublisher($published)); + + $result = $background->publish(new Queue('emails'), ['id' => 1]); + + $this->assertTrue($result); + $this->assertSame([['id' => 1]], $published); + } + + public function testEnqueueFallsBackToSyncWhenNotStarted(): void + { + $published = []; + $background = new Background($this->recordingPublisher($published)); + + $background->enqueue(new Queue('emails'), ['id' => 1]); + + $this->assertSame([['id' => 1]], $published, 'no reader loop running → publish synchronously'); + } + + public function testReaderDrainsChannelIntoPublisher(): void + { + $published = []; + $background = new Background($this->recordingPublisher($published)); + + Coroutine\run(function () use ($background): void { + $background->start(); + + for ($i = 1; $i <= 5; $i++) { + $background->enqueue(new Queue('emails'), ['id' => $i]); + } + + $background->shutdown(); // drains queued publishes, then waits for the reader + }); + + $this->assertSame([1, 2, 3, 4, 5], array_column($published, 'id')); + } + + public function testBackPressureBoundDeliversEveryMessageInOrder(): void + { + $published = []; + // Capacity 1 forces enqueue() to block on nearly every push, so the + // producer only advances as the reader drains — exercising back pressure. + $background = new Background($this->recordingPublisher($published), capacity: 1); + + Coroutine\run(function () use ($background): void { + $background->start(); + + for ($i = 1; $i <= 20; $i++) { + $background->enqueue(new Queue('emails'), ['id' => $i]); + } + + $background->shutdown(); + }); + + $this->assertSame(range(1, 20), array_column($published, 'id')); + } + + public function testConcurrentCoroutinesDeliverEveryMessage(): void + { + $published = []; + $background = new Background($this->recordingPublisher($published), coroutines: 4); + + Coroutine\run(function () use ($background): void { + $background->start(); + + for ($i = 1; $i <= 20; $i++) { + $background->enqueue(new Queue('emails'), ['id' => $i]); + } + + $background->shutdown(); + }); + + // Four readers dispatch concurrently, so order isn't guaranteed — but + // every message must land exactly once. + $ids = array_column($published, 'id'); + sort($ids); + $this->assertSame(range(1, 20), $ids); + } + + public function testDelegatesManagementCalls(): void + { + $published = [['id' => 1], ['id' => 2]]; + $background = new Background($this->recordingPublisher($published)); + + $this->assertSame(2, $background->getQueueSize(new Queue('emails'))); + } + + public function testEnqueueThrowsWhenBufferStaysFull(): void + { + // A publisher that parks in publish() until the test releases the gate, + // so the single reader can't drain the channel. + $gate = new Channel(2); + $publisher = new readonly class ($gate) implements Synchronous { + public function __construct(private Channel $gate) {} + + public function publish(Queue $queue, array $payload, bool $priority = false): bool + { + $this->gate->pop(); + + return true; + } + + public function retry(Queue $queue, ?int $limit = null): void {} + + public function getQueueSize(Queue $queue, bool $failedJobs = false): int + { + return 0; + } + }; + + $background = new Background($publisher, capacity: 1, timeout: 0.05); + $threw = false; + + Coroutine\run(function () use ($background, $gate, &$threw): void { + $background->start(); + + $background->enqueue(new Queue('emails'), ['id' => 1]); // reader pops it, parks in publish() + $background->enqueue(new Queue('emails'), ['id' => 2]); // fills the one slot + + try { + $background->enqueue(new Queue('emails'), ['id' => 3]); // full + parked → back pressure + } catch (BufferFullException) { + $threw = true; + } + + $gate->push(true); // release the parked publishes so the run can finish + $gate->push(true); + $background->shutdown(); + }); + + $this->assertTrue($threw, 'a full buffer past the timeout throws BufferFullException'); + } + + public function testReportsBufferDepthGauge(): void + { + $telemetry = new TestTelemetry(); + $buffer = []; + new Background($this->recordingPublisher($buffer), telemetry: $telemetry); + + // An idle buffer observes a depth of zero; the gauge is wired to the channel. + $this->assertArrayHasKey('messaging.publisher.buffer.depth', $telemetry->observableGauges); + $this->assertSame([0], $this->collectObservations($telemetry, 'messaging.publisher.buffer.depth')); + } + + /** + * Reads an observable gauge by invoking its registered callbacks. + * + * @return array + */ + private function collectObservations(TestTelemetry $telemetry, string $name): array + { + /** @var object{callbacks: array} $gauge */ + $gauge = $telemetry->observableGauges[$name]; + + $values = []; + foreach ($gauge->callbacks as $callback) { + $callback(function (float|int $value, iterable $attributes = []) use (&$values): void { + $values[] = $value; + }); + } + + return $values; + } + + /** + * A synchronous publisher that records published payloads into the buffer. + * + * @param array> $buffer + */ + private function recordingPublisher(array &$buffer): Synchronous + { + return new class ($buffer) implements Synchronous { + /** + * @param array> $buffer + */ + public function __construct(private array &$buffer) {} + + public function publish(Queue $queue, array $payload, bool $priority = false): bool + { + $this->buffer[] = $payload; + + return true; + } + + public function retry(Queue $queue, ?int $limit = null): void {} + + public function getQueueSize(Queue $queue, bool $failedJobs = false): int + { + return \count($this->buffer); + } + }; + } +} diff --git a/packages/queue/tests/Queue/E2E/Adapter/Base.php b/packages/queue/tests/Queue/E2E/Adapter/Base.php index 12884040b..043e985ca 100644 --- a/packages/queue/tests/Queue/E2E/Adapter/Base.php +++ b/packages/queue/tests/Queue/E2E/Adapter/Base.php @@ -8,7 +8,7 @@ use PHPUnit\Framework\Attributes\Depends; use PHPUnit\Framework\TestCase; -use Utopia\Queue\Publisher; +use Utopia\Queue\Publisher\Synchronous; use Utopia\Queue\Queue; abstract class Base extends TestCase @@ -57,7 +57,7 @@ public function setUp(): void ]; } - abstract protected function getPublisher(): Publisher; + abstract protected function getPublisher(): Synchronous; abstract protected function getQueue(): Queue; @@ -66,7 +66,7 @@ public function testEvents(): void $publisher = $this->getPublisher(); foreach ($this->payloads as $payload) { - $this->assertTrue($publisher->enqueue($this->getQueue(), $payload)); + $this->assertTrue($publisher->publish($this->getQueue(), $payload)); } sleep(1); @@ -78,7 +78,7 @@ public function testConcurrency(): void $publisher = $this->getPublisher(); go(function () use ($publisher): void { foreach ($this->payloads as $payload) { - $this->assertTrue($publisher->enqueue($this->getQueue(), $payload)); + $this->assertTrue($publisher->publish($this->getQueue(), $payload)); } sleep(1); @@ -90,7 +90,7 @@ public function testEnqueuePriority(): void { $publisher = $this->getPublisher(); - $result = $publisher->enqueue($this->getQueue(), ['type' => 'test_string', 'value' => 'priority'], priority: true); + $result = $publisher->publish($this->getQueue(), ['type' => 'test_string', 'value' => 'priority'], priority: true); $this->assertTrue($result); } @@ -100,28 +100,28 @@ public function testParamAliases(): void $publisher = $this->getPublisher(); // Resolves via canonical key - $this->assertTrue($publisher->enqueue($this->getQueue(), [ + $this->assertTrue($publisher->publish($this->getQueue(), [ 'type' => 'test_alias', 'aliasValue' => 'canonical', 'value' => 'canonical', ])); // Resolves via first alias when canonical absent - $this->assertTrue($publisher->enqueue($this->getQueue(), [ + $this->assertTrue($publisher->publish($this->getQueue(), [ 'type' => 'test_alias', 'alias_value' => 'first-alias', 'value' => 'first-alias', ])); // Falls through to later alias when earlier ones absent - $this->assertTrue($publisher->enqueue($this->getQueue(), [ + $this->assertTrue($publisher->publish($this->getQueue(), [ 'type' => 'test_alias', 'aliased' => 'second-alias', 'value' => 'second-alias', ])); // Canonical key wins when both canonical and aliases are present - $this->assertTrue($publisher->enqueue($this->getQueue(), [ + $this->assertTrue($publisher->publish($this->getQueue(), [ 'type' => 'test_alias', 'aliasValue' => 'canonical-wins', 'alias_value' => 'should-lose', @@ -137,28 +137,28 @@ public function testRetry(): void { $publisher = $this->getPublisher(); - $published = $publisher->enqueue($this->getQueue(), [ + $published = $publisher->publish($this->getQueue(), [ 'type' => 'test_exception', 'id' => 1, ]); $this->assertTrue($published); - $published = $publisher->enqueue($this->getQueue(), [ + $published = $publisher->publish($this->getQueue(), [ 'type' => 'test_exception', 'id' => 2, ]); $this->assertTrue($published); - $published = $publisher->enqueue($this->getQueue(), [ + $published = $publisher->publish($this->getQueue(), [ 'type' => 'test_exception', 'id' => 3, ]); $this->assertTrue($published); - $published = $publisher->enqueue($this->getQueue(), [ + $published = $publisher->publish($this->getQueue(), [ 'type' => 'test_exception', 'id' => 4, ]); diff --git a/packages/queue/tests/Queue/E2E/Adapter/PoolTest.php b/packages/queue/tests/Queue/E2E/Adapter/PoolTest.php index 325bbddd7..ecbd617ff 100644 --- a/packages/queue/tests/Queue/E2E/Adapter/PoolTest.php +++ b/packages/queue/tests/Queue/E2E/Adapter/PoolTest.php @@ -9,12 +9,12 @@ use Utopia\Queue\Broker\Pool; use Utopia\Queue\Broker\Redis as RedisBroker; use Utopia\Queue\Connection\Redis; -use Utopia\Queue\Publisher; +use Utopia\Queue\Publisher\Synchronous; use Utopia\Queue\Queue; final class PoolTest extends Base { - protected function getPublisher(): Publisher + protected function getPublisher(): Synchronous { $pool = new UtopiaPool(new Stack(), 'redis', 1, fn(): \Utopia\Queue\Broker\Redis => new RedisBroker(new Redis('127.0.0.1', 16379), new Redis('127.0.0.1', 16379))); diff --git a/packages/queue/tests/Queue/E2E/Adapter/ServerTelemetryTest.php b/packages/queue/tests/Queue/E2E/Adapter/ServerTelemetryTest.php index 8a381ec6d..809708e9f 100644 --- a/packages/queue/tests/Queue/E2E/Adapter/ServerTelemetryTest.php +++ b/packages/queue/tests/Queue/E2E/Adapter/ServerTelemetryTest.php @@ -9,7 +9,7 @@ use Utopia\Queue\Adapter; use Utopia\Queue\Consumer; use Utopia\Queue\Message; -use Utopia\Queue\Publisher; +use Utopia\Queue\Publisher\Synchronous; use Utopia\Queue\Queue; use Utopia\Queue\Server; use Utopia\Telemetry\Adapter\Test as TestTelemetry; @@ -277,14 +277,14 @@ public function reject(Queue $queue, Message $message): void {} public function close(): void {} } -final class ServerTelemetryPublisherConsumer extends ServerTelemetryConsumer implements Publisher +final class ServerTelemetryPublisherConsumer extends ServerTelemetryConsumer implements Synchronous { /** * @param int[] $queueSizes */ public function __construct(private array $queueSizes) {} - public function enqueue(Queue $queue, array $payload, bool $priority = false): bool + public function publish(Queue $queue, array $payload, bool $priority = false): bool { return true; } @@ -297,9 +297,9 @@ public function getQueueSize(Queue $queue, bool $failedJobs = false): int } } -final class ServerTelemetryFailingPublisherConsumer extends ServerTelemetryConsumer implements Publisher +final class ServerTelemetryFailingPublisherConsumer extends ServerTelemetryConsumer implements Synchronous { - public function enqueue(Queue $queue, array $payload, bool $priority = false): bool + public function publish(Queue $queue, array $payload, bool $priority = false): bool { return true; } diff --git a/packages/queue/tests/Queue/E2E/Adapter/SwooleConcurrencyTest.php b/packages/queue/tests/Queue/E2E/Adapter/SwooleConcurrencyTest.php index 7a6d9fbd0..d35286511 100644 --- a/packages/queue/tests/Queue/E2E/Adapter/SwooleConcurrencyTest.php +++ b/packages/queue/tests/Queue/E2E/Adapter/SwooleConcurrencyTest.php @@ -48,7 +48,7 @@ private function runWorker(int $messages, int $maxCoroutines): array \Swoole\Coroutine\run(function () use ($broker, $queue, $messages, $maxCoroutines, &$active, &$maxActive, &$processed): void { for ($i = 0; $i < $messages; $i++) { - $broker->enqueue($queue, ['n' => $i]); + $broker->publish($queue, ['n' => $i]); } $adapter = new Swoole($broker, 1, self::QUEUE, self::NAMESPACE, maxCoroutines: $maxCoroutines); diff --git a/packages/queue/tests/Queue/E2E/Adapter/SwooleRedisClusterTest.php b/packages/queue/tests/Queue/E2E/Adapter/SwooleRedisClusterTest.php index 4f8902812..aeabfde6e 100644 --- a/packages/queue/tests/Queue/E2E/Adapter/SwooleRedisClusterTest.php +++ b/packages/queue/tests/Queue/E2E/Adapter/SwooleRedisClusterTest.php @@ -6,7 +6,7 @@ use Utopia\Queue\Broker\Redis; use Utopia\Queue\Connection\RedisCluster; -use Utopia\Queue\Publisher; +use Utopia\Queue\Publisher\Synchronous; use Utopia\Queue\Queue; final class SwooleRedisClusterTest extends Base @@ -20,7 +20,7 @@ private function getConnection(): RedisCluster ]); } - protected function getPublisher(): Publisher + protected function getPublisher(): Synchronous { return new Redis($this->getConnection(), $this->getConnection()); } @@ -46,12 +46,12 @@ public function testPriorityJobIsConsumedBeforeNormalJobs(): void } // Enqueue three normal jobs (pushed to head/left). - $this->getPublisher()->enqueue($queue, ['order' => 'normal-1']); - $this->getPublisher()->enqueue($queue, ['order' => 'normal-2']); - $this->getPublisher()->enqueue($queue, ['order' => 'normal-3']); + $this->getPublisher()->publish($queue, ['order' => 'normal-1']); + $this->getPublisher()->publish($queue, ['order' => 'normal-2']); + $this->getPublisher()->publish($queue, ['order' => 'normal-3']); // Enqueue one priority job (pushed to tail/right — same end BRPOP reads from). - $this->getPublisher()->enqueue($queue, ['order' => 'priority'], priority: true); + $this->getPublisher()->publish($queue, ['order' => 'priority'], priority: true); // The first pop should yield the priority job. $first = $connection->rightPopArray($key, 1); diff --git a/packages/queue/tests/Queue/E2E/Adapter/SwooleTest.php b/packages/queue/tests/Queue/E2E/Adapter/SwooleTest.php index 39fcbb35c..090cd028c 100644 --- a/packages/queue/tests/Queue/E2E/Adapter/SwooleTest.php +++ b/packages/queue/tests/Queue/E2E/Adapter/SwooleTest.php @@ -6,7 +6,7 @@ use Utopia\Queue\Broker\Redis as RedisBroker; use Utopia\Queue\Connection\Redis; -use Utopia\Queue\Publisher; +use Utopia\Queue\Publisher\Synchronous; use Utopia\Queue\Queue; final class SwooleTest extends Base @@ -16,7 +16,7 @@ private function getConnection(): Redis return new Redis('127.0.0.1', 16379); } - protected function getPublisher(): Publisher + protected function getPublisher(): Synchronous { return new RedisBroker($this->getConnection(), $this->getConnection()); } @@ -42,12 +42,12 @@ public function testPriorityJobIsConsumedBeforeNormalJobs(): void } // Enqueue three normal jobs (pushed to head/left). - $this->getPublisher()->enqueue($queue, ['order' => 'normal-1']); - $this->getPublisher()->enqueue($queue, ['order' => 'normal-2']); - $this->getPublisher()->enqueue($queue, ['order' => 'normal-3']); + $this->getPublisher()->publish($queue, ['order' => 'normal-1']); + $this->getPublisher()->publish($queue, ['order' => 'normal-2']); + $this->getPublisher()->publish($queue, ['order' => 'normal-3']); // Enqueue one priority job (pushed to tail/right — same end BRPOP reads from). - $this->getPublisher()->enqueue($queue, ['order' => 'priority'], priority: true); + $this->getPublisher()->publish($queue, ['order' => 'priority'], priority: true); // The first pop should yield the priority job. $first = $connection->rightPopArray($key, 1); diff --git a/packages/queue/tests/Queue/E2E/Adapter/WorkermanTest.php b/packages/queue/tests/Queue/E2E/Adapter/WorkermanTest.php index d3ca07587..8b4422331 100644 --- a/packages/queue/tests/Queue/E2E/Adapter/WorkermanTest.php +++ b/packages/queue/tests/Queue/E2E/Adapter/WorkermanTest.php @@ -6,12 +6,12 @@ use Utopia\Queue\Broker\Redis as RedisPublisher; use Utopia\Queue\Connection\Redis; -use Utopia\Queue\Publisher; +use Utopia\Queue\Publisher\Synchronous; use Utopia\Queue\Queue; final class WorkermanTest extends Base { - protected function getPublisher(): Publisher + protected function getPublisher(): Synchronous { return new RedisPublisher(new Redis('127.0.0.1', 16379), new Redis('127.0.0.1', 16379)); }