From 0c5f9c8fbbc89335405c6ba1de97f6d04b6a590e Mon Sep 17 00:00:00 2001 From: Diego Braga Date: Fri, 10 Jul 2026 18:53:26 +0200 Subject: [PATCH] feat(shutdown): graceful drain via unstructured-runtime v1.3.2 Bump unstructured-runtime to v1.3.2 (bounded graceful drain of in-flight reconciles on SIGTERM) and add a --graceful-shutdown-timeout flag (COMPOSITION_CONTROLLER_GRACEFUL_SHUTDOWN_TIMEOUT, default 30s) wired through builder.WithGracefulShutdownTimeout. In-flight reconciles now finish under a live context after SIGTERM instead of being severed, closing the external-create-pending stranding window (roadmap#231) and unblocking CDC HA (#233/#242). The value must be set below the pod's terminationGracePeriodSeconds; a paired helm-chart change (terminationGracePeriodSeconds >= 40s) follows. Co-Authored-By: Claude Opus 4.8 (1M context) --- go.mod | 2 +- go.sum | 4 ++-- main.go | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ee230ad..5b6033d 100644 --- a/go.mod +++ b/go.mod @@ -163,7 +163,7 @@ require ( // span (#5, tag v1.3.0). v1.3.1 drops the plumbing/slogs/pretty test dependency so the module // builds against plumbing's current main line. // The fork keeps the upstream module path, so pin it via replace. -replace github.com/krateoplatformops/unstructured-runtime => github.com/braghettos/unstructured-runtime v1.3.1 +replace github.com/krateoplatformops/unstructured-runtime => github.com/braghettos/unstructured-runtime v1.3.2 // Source plumbing from the braghettos fork (v1.10.0, main line): carries the jqutil int64/int32 // gojq-panic fix, crdgen array-default markers, the krateo.io/traceparent child-manifest diff --git a/go.sum b/go.sum index 256a6e7..526791c 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/braghettos/plumbing v1.10.3 h1:IZBIZXILY48lkl325D+P6OV1B1gVfwEzUf3Fh2dlQlg= github.com/braghettos/plumbing v1.10.3/go.mod h1:Z8Xd2ZR/F4mYhtGtr+fH2ZN9+ifqaXESHEmaBT+NiuI= -github.com/braghettos/unstructured-runtime v1.3.1 h1:ZwuLl1VP7Ysqk/JSrYHIC0GOYVSN7tV0XdiFkZAIa3Q= -github.com/braghettos/unstructured-runtime v1.3.1/go.mod h1:FhrvKaArNeocZaW0TDQuvR2nmPo+fgbFHFNP2l7H/i4= +github.com/braghettos/unstructured-runtime v1.3.2 h1:sXjKsdX7MbjvejZfO+eq7bf2y6p0unbVeZQZGdin6wM= +github.com/braghettos/unstructured-runtime v1.3.2/go.mod h1:FhrvKaArNeocZaW0TDQuvR2nmPo+fgbFHFNP2l7H/i4= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= diff --git a/main.go b/main.go index 5325299..2034c83 100644 --- a/main.go +++ b/main.go @@ -104,6 +104,9 @@ func main() { env.Int("COMPOSITION_CONTROLLER_MAX_ERROR_RETRIES", 5), "How many times to retry the processing of a resource when an error occurs before giving up and dropping the resource.") metricsServerPort := flag.Int("metrics-server-port", env.Int("COMPOSITION_CONTROLLER_METRICS_SERVER_PORT", 0), "The address to bind the metrics server to. If empty, metrics server is disabled.") + gracefulShutdownTimeout := flag.Duration("graceful-shutdown-timeout", + env.Duration("COMPOSITION_CONTROLLER_GRACEFUL_SHUTDOWN_TIMEOUT", 30*time.Second), + "Max time to let in-flight reconciles finish after SIGTERM before the process exits. Must be set below the pod's terminationGracePeriodSeconds. 0 disables the drain (abrupt shutdown); negative waits indefinitely.") safeReleaseName := flag.Bool("safe-release-name", env.Bool("COMPOSITION_CONTROLLER_SAFE_RELEASE_NAME", true), "If disabled the randmom suffix is not appended in the Helm release name. This can be useful for avoid having problems with complex helm charts. The use of this option is highly discouraged, as it can lead to release name collisions.") otelEnabled := flag.Bool("otel-enabled", env.Bool("OTEL_ENABLED", false), "Enable OTLP metrics export for provider-runtime telemetry.") @@ -338,6 +341,7 @@ func main() { builder.WithResyncInterval(*resyncInterval), builder.WithGlobalRateLimiter(workqueue.NewExponentialTimedFailureRateLimiter[any](*minErrorRetryInterval, *maxErrorRetryInterval)), builder.WithMaxRetries(*maxErrorRetry), + builder.WithGracefulShutdownTimeout(*gracefulShutdownTimeout), builder.WithListWatcher(controller.ListWatcherConfiguration{ LabelSelector: ptr.To(labelselector.String()), }),