From 223149b1537c8e6f6ad06f00c9a49f0a8dc3e1da Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:41:29 +0000 Subject: [PATCH 1/2] Add LocalDNS LPS bootstrap patching support Add AgentBaker LocalDNS live-patching support for the LPS bootstrap path and runtime knead dispatcher. The bootstrap path fetches LocalDNS nodeConfig from LPS, renders it through aks-node-controller, and feeds the generated Corefile into the existing updated.localdns.corefile flow before kubelet starts. The runtime path applies dispatched LocalDNS payloads with apply-localdns-config. Update focused unit, shellspec, and E2E coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/app.go | 29 + aks-node-controller/go.mod | 16 +- aks-node-controller/go.sum | 51 +- aks-node-controller/localdnsconfig.go | 522 ++++++++++++++++++ aks-node-controller/localdnsconfig_test.go | 221 ++++++++ aks-node-controller/parser/helper.go | 6 +- aks-node-controller/parser/helper_test.go | 4 + .../parser/templates/localdns.toml.gtpl | 2 + e2e/scenario_localdns_hosts_test.go | 182 ++++++ parts/linux/cloud-init/artifacts/localdns.sh | 150 ++++- .../ubuntu/ubuntu-snapshot-update.sh | 55 ++ pkg/agent/baker.go | 2 + pkg/agent/baker_test.go | 10 + .../cloud-init/artifacts/localdns_spec.sh | 76 +++ .../artifacts/ubuntu-snapshot-update_spec.sh | 57 +- 15 files changed, 1375 insertions(+), 8 deletions(-) create mode 100644 aks-node-controller/localdnsconfig.go create mode 100644 aks-node-controller/localdnsconfig_test.go diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index cb0e3477779..b9453fcbb83 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -66,6 +66,8 @@ type App struct { // Authorization header for the check-hotfix LPS fetch. When nil, the real IMDS endpoint // is queried. fetchAttestedToken func(ctx context.Context) (string, error) + // fetchLocalDNSConfigFn overrides the real LPS LocalDNS config fetch for tests. + fetchLocalDNSConfigFn localDNSConfigFetcher } // provision.json values are emitted as strings by the shell jq invocation. @@ -168,6 +170,33 @@ func (a *App) Run(ctx context.Context, args []string) int { return a.runCheckHotfixCommand(ctx) }, }, + { + Name: "fetch-localdns-config", + Usage: "Read the LocalDNS config from the live-patching-service and update the Corefile (fail-open)", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "output", Usage: "path to write the LocalDNS Corefile"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if extra := cmd.Args().Slice(); len(extra) > 0 { + slog.Warn("ignoring unexpected fetch-localdns-config arguments", "args", strings.Join(extra, " ")) + } + return a.runFetchLocalDNSConfigCommand(ctx, cmd.String("output")) + }, + }, + { + Name: "apply-localdns-config", + Usage: "Apply a dispatched LocalDNS live-patching config slice to the Corefile", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "config-file", Usage: "path to the LocalDNS config JSON; reads stdin when omitted or '-'"}, + &cli.StringFlag{Name: "output", Usage: "path to write the LocalDNS Corefile"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if extra := cmd.Args().Slice(); len(extra) > 0 { + return fmt.Errorf("unexpected apply-localdns-config arguments: %s", strings.Join(extra, " ")) + } + return a.runApplyLocalDNSConfigCommand(ctx, cmd.String("config-file"), cmd.String("output"), cmd.Root().Writer) + }, + }, }, } diff --git a/aks-node-controller/go.mod b/aks-node-controller/go.mod index faf133532f5..a2f9e0c202f 100644 --- a/aks-node-controller/go.mod +++ b/aks-node-controller/go.mod @@ -3,18 +3,32 @@ module github.com/Azure/agentbaker/aks-node-controller go 1.25.11 require ( + github.com/Azure/agentbaker/aks-live-patching v0.0.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 github.com/Masterminds/semver/v3 v3.5.0 github.com/fsnotify/fsnotify v1.8.0 github.com/google/go-cmp v0.7.0 github.com/stretchr/testify v1.11.1 github.com/urfave/cli/v3 v3.8.0 - google.golang.org/protobuf v1.36.7 + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) + +replace github.com/Azure/agentbaker => ../ + +replace github.com/Azure/agentbaker/aks-live-patching => ../aks-live-patching + +replace github.com/coreos/ignition/v2 => github.com/flatcar/ignition/v2 v2.0.0-20250903113522-05b8a773288c diff --git a/aks-node-controller/go.sum b/aks-node-controller/go.sum index c58a6462885..4ab566d5771 100644 --- a/aks-node-controller/go.sum +++ b/aks-node-controller/go.sum @@ -2,23 +2,68 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyg github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/urfave/cli/v3 v3.8.0 h1:XqKPrm0q4P0q5JpoclYoCAv0/MIvH/jZ2umzuf8pNTI= github.com/urfave/cli/v3 v3.8.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/aks-node-controller/localdnsconfig.go b/aks-node-controller/localdnsconfig.go new file mode 100644 index 00000000000..771d60b27c0 --- /dev/null +++ b/aks-node-controller/localdnsconfig.go @@ -0,0 +1,522 @@ +package main + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + akslivepatchingv1 "github.com/Azure/agentbaker/aks-live-patching/pkg/gen/akslivepatching/v1" + "github.com/Azure/agentbaker/aks-node-controller/helpers" + "github.com/Azure/agentbaker/aks-node-controller/parser" + aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" + "github.com/Azure/agentbaker/aks-node-controller/pkg/nodeconfigutils" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/encoding/protojson" +) + +const ( + localDNSLivePatchingComponentName = "localDNS" + defaultLocalDNSCorefilePath = "/opt/azure/containers/localdns/livepatched.localdns.corefile" + localDNSHostsFilePath = "/etc/localdns/hosts" + localDNSAgentPoolLabel = "kubernetes.azure.com/agentpool" + localDNSLPSALPNProto = "aks-live-patching" + localDNSALPNH2Proto = "h2" +) + +type localDNSConfigFetcher func(context.Context) (string, error) + +type localDNSConfigOutcome string + +const ( + outcomeLocalDNSConfigApplied localDNSConfigOutcome = "applied" + outcomeLocalDNSConfigAlreadyCurrent localDNSConfigOutcome = "alreadyCurrent" + outcomeLocalDNSConfigNotFound localDNSConfigOutcome = "notFound" + outcomeLocalDNSConfigNoCorefileData localDNSConfigOutcome = "noCorefileData" + outcomeLocalDNSConfigFailed localDNSConfigOutcome = "failed" +) + +type localDNSConfigPayload struct { + Corefile string `json:"corefile"` + CorefileBase64 string `json:"corefileBase64"` + CorefileBase64Alt string `json:"corefile_base64"` + CoreFile string `json:"coreFile"` + LocalDNSProfile json.RawMessage `json:"localDnsProfile"` + LocalDNSProfileAlt json.RawMessage `json:"local_dns_profile"` + AgentPools map[string]localDNSAgentPoolConfig `json:"agentPools"` + Profiles map[string]localDNSAgentPoolConfig `json:"profiles"` +} + +type localDNSAgentPoolConfig struct { + CorefileVersion string `json:"corefileVersion"` + ConfigChecksum string `json:"configChecksum"` + Corefile string `json:"corefile"` + CorefileBase64 string `json:"corefileBase64"` + CorefileBase64Alt string `json:"corefile_base64"` + CoreFile string `json:"coreFile"` + LocalDNSProfile json.RawMessage `json:"localDnsProfile"` + LocalDNSProfileAlt json.RawMessage `json:"local_dns_profile"` +} + +type localDNSCorefileUpdate struct { + corefile string + desiredVersion string + hasCorefile bool +} + +func (a *App) runApplyLocalDNSConfigCommand(ctx context.Context, configPath string, outputPath string, writer io.Writer) error { + config, err := readLocalDNSConfigInput(configPath) + if err != nil { + return err + } + outcome, err := a.applyLocalDNSConfig(ctx, config, outputPath) + if writer != nil { + _, _ = fmt.Fprintf(writer, "%s\n", outcome) + } + if err != nil { + return err + } + return nil +} + +func readLocalDNSConfigInput(configPath string) (string, error) { + if configPath == "" || configPath == "-" { + data, err := io.ReadAll(os.Stdin) + if err != nil { + return "", fmt.Errorf("reading localDNS config from stdin: %w", err) + } + return string(data), nil + } + data, err := os.ReadFile(configPath) + if err != nil { + return "", fmt.Errorf("reading localDNS config %s: %w", configPath, err) + } + return string(data), nil +} + +func (a *App) applyLocalDNSConfig(ctx context.Context, config string, outputPath string) (localDNSConfigOutcome, error) { + return a.fetchAndApplyLocalDNSConfigWithFetcher(ctx, outputPath, func(context.Context) (string, error) { + return config, nil + }) +} + +func (a *App) runFetchLocalDNSConfigCommand(ctx context.Context, outputPath string) (err error) { + slog.Info("aks-node-controller fetch-localdns-config started", "outputPath", outputPath) + startTime := time.Now() + defer func() { + if r := recover(); r != nil { + slog.Error("fetch-localdns-config panicked (fail-open)", "panic", r) + if a.eventLogger != nil { + a.eventLogger.LogEvent("FetchLocalDNSConfig", + fmt.Sprintf("fetch-localdns-config outcome=%s panic=%v", outcomeLocalDNSConfigFailed, r), + helpers.EventLevelError, startTime, time.Now()) + } + err = nil + } + }() + + outcome, err := a.fetchAndApplyLocalDNSConfig(ctx, outputPath) + level := helpers.EventLevelInformational + if outcome == outcomeLocalDNSConfigFailed { + level = helpers.EventLevelError + } + message := fmt.Sprintf("fetch-localdns-config outcome=%s", outcome) + if err != nil { + message = fmt.Sprintf("%s error=%s", message, err.Error()) + slog.Warn("fetch-localdns-config completed with error (fail-open)", "outcome", outcome, "error", err) + } else { + slog.Info("fetch-localdns-config completed", "outcome", outcome) + } + if a.eventLogger != nil { + a.eventLogger.LogEvent("FetchLocalDNSConfig", message, level, startTime, time.Now()) + } + return nil +} + +func (a *App) fetchAndApplyLocalDNSConfig(ctx context.Context, outputPath string) (localDNSConfigOutcome, error) { + return a.fetchAndApplyLocalDNSConfigWithFetcher(ctx, outputPath, a.fetchLocalDNSConfig) +} + +func (a *App) fetchAndApplyLocalDNSConfigWithFetcher(ctx context.Context, outputPath string, fetcher localDNSConfigFetcher) (localDNSConfigOutcome, error) { + if outputPath == "" { + outputPath = defaultLocalDNSCorefilePath + } + config, err := fetcher(ctx) + if err != nil { + if isLPSUnavailable(err) { + return outcomeLocalDNSConfigNotFound, nil + } + return outcomeLocalDNSConfigFailed, err + } + update, err := a.localDNSCorefileUpdateFromConfig(config) + if err != nil { + return outcomeLocalDNSConfigFailed, err + } + versionPath := localDNSCorefileVersionPath(outputPath) + if !update.hasCorefile { + return outcomeLocalDNSConfigNoCorefileData, nil + } + current, readErr := os.ReadFile(outputPath) + if readErr != nil && !os.IsNotExist(readErr) { + return outcomeLocalDNSConfigFailed, fmt.Errorf("reading localDNS corefile %s: %w", outputPath, readErr) + } + contentMatches := readErr == nil && bytes.Equal(current, []byte(update.corefile)) + if update.desiredVersion != "" { + currentVersion, err := readLocalDNSCorefileVersion(versionPath) + if err != nil { + return outcomeLocalDNSConfigFailed, err + } + if currentVersion == update.desiredVersion && contentMatches { + return outcomeLocalDNSConfigAlreadyCurrent, nil + } + } else if contentMatches { + return outcomeLocalDNSConfigAlreadyCurrent, nil + } + if err := writeLocalDNSCorefile(outputPath, update.corefile); err != nil { + return outcomeLocalDNSConfigFailed, err + } + if update.desiredVersion != "" { + if err := writeLocalDNSCorefileVersion(versionPath, update.desiredVersion); err != nil { + return outcomeLocalDNSConfigFailed, err + } + } + return outcomeLocalDNSConfigApplied, nil +} + +func (a *App) fetchLocalDNSConfig(ctx context.Context) (string, error) { + if a.fetchLocalDNSConfigFn != nil { + return a.fetchLocalDNSConfigFn(ctx) + } + return a.fetchLocalDNSConfigFromLPS(ctx) +} + +func (a *App) fetchLocalDNSConfigFromLPS(ctx context.Context) (string, error) { + fqdn, caPEM, err := a.lpsTargetFromNodeConfig() + if err != nil { + return "", fmt.Errorf("resolving LPS endpoint from node config: %w", err) + } + token, err := a.attestedToken(ctx) + if err != nil { + return "", fmt.Errorf("imds attested token: %w", err) + } + rootCAs, err := certPoolFromPEM(caPEM) + if err != nil { + return "", err + } + host := fqdn + if h, _, splitErr := net.SplitHostPort(fqdn); splitErr == nil { + host = h + } + target := net.JoinHostPort(host, lpsAPIServerPort) + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: rootCAs, + NextProtos: []string{localDNSLPSALPNProto, localDNSALPNH2Proto}, + InsecureSkipVerify: true, //nolint:gosec // SNI stays on the apiserver FQDN for ALPN routing; chain and hostname are verified below. + VerifyPeerCertificate: localDNSVerifyChainAgainstPool(rootCAs, lpsSNIHost), + } + ctx, cancel := context.WithTimeout(ctx, lpsFetchTimeout) + defer cancel() + + conn, err := grpc.NewClient( + target, + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + ) + if err != nil { + return "", fmt.Errorf("creating LPS client: %w", err) + } + defer conn.Close() + + client := akslivepatchingv1.NewLivePatchingServiceClient(conn) + rpcCtx := metadata.AppendToOutgoingContext(ctx, "authorization", token) + resp, err := client.GetComponentConfig(rpcCtx, &akslivepatchingv1.GetComponentConfigRequest{ + ComponentName: localDNSLivePatchingComponentName, + }) + if err != nil { + if statusCode, ok := localDNSLPSUnavailableStatusCode(status.Code(err)); ok { + return "", &lpsUnavailableError{statusCode: statusCode} + } + return "", fmt.Errorf("get %s component config: %w", localDNSLivePatchingComponentName, err) + } + return resp.GetConfig(), nil +} + +func localDNSLPSUnavailableStatusCode(code codes.Code) (int, bool) { + switch code { + case codes.NotFound: + return http.StatusNotFound, true + case codes.PermissionDenied: + return http.StatusForbidden, true + case codes.Unauthenticated: + return http.StatusUnauthorized, true + default: + return 0, false + } +} + +func localDNSVerifyChainAgainstPool(pool *x509.CertPool, serverName string) func([][]byte, [][]*x509.Certificate) error { + return func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return fmt.Errorf("server presented no certificates") + } + leaf, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return fmt.Errorf("failed to parse server certificate: %w", err) + } + intermediates := x509.NewCertPool() + for _, raw := range rawCerts[1:] { + cert, err := x509.ParseCertificate(raw) + if err != nil { + return fmt.Errorf("failed to parse intermediate certificate: %w", err) + } + intermediates.AddCert(cert) + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: pool, Intermediates: intermediates, DNSName: serverName}); err != nil { + return fmt.Errorf("server certificate verification failed: %w", err) + } + return nil + } +} + +func certPoolFromPEM(caPEM []byte) (*x509.CertPool, error) { + if len(caPEM) == 0 { + return nil, fmt.Errorf("cluster CA unavailable from provision-config; refusing to fetch over unverified TLS") + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("failed to parse cluster CA PEM") + } + return pool, nil +} + +func (a *App) localDNSCorefileUpdateFromConfig(config string) (localDNSCorefileUpdate, error) { + config = strings.TrimSpace(config) + if config == "" { + return localDNSCorefileUpdate{}, nil + } + + var payload localDNSConfigPayload + if err := json.Unmarshal([]byte(config), &payload); err != nil { + return localDNSCorefileUpdate{}, fmt.Errorf("parsing localDNS LPS config: %w", err) + } + + selected, found, err := a.selectLocalDNSAgentPoolConfig(payload) + if err != nil || !found { + return localDNSCorefileUpdate{}, err + } + + update := localDNSCorefileUpdate{ + desiredVersion: firstNonEmpty(selected.CorefileVersion, selected.ConfigChecksum), + } + return a.localDNSCorefileUpdateFromAgentPoolConfig(selected, update) +} + +func (a *App) selectLocalDNSAgentPoolConfig(payload localDNSConfigPayload) (localDNSAgentPoolConfig, bool, error) { + selected := localDNSAgentPoolConfig{ + Corefile: payload.Corefile, + CorefileBase64: payload.CorefileBase64, + CorefileBase64Alt: payload.CorefileBase64Alt, + CoreFile: payload.CoreFile, + LocalDNSProfile: payload.LocalDNSProfile, + LocalDNSProfileAlt: payload.LocalDNSProfileAlt, + } + if len(payload.AgentPools) == 0 && len(payload.Profiles) == 0 { + return selected, true, nil + } + + agentPool, err := a.nodeAgentPoolName() + if err != nil { + return localDNSAgentPoolConfig{}, false, err + } + if selected, ok := payload.AgentPools[agentPool]; ok { + return selected, true, nil + } + if selected, ok := payload.Profiles[agentPool]; ok { + return selected, true, nil + } + return localDNSAgentPoolConfig{}, false, nil +} + +func (a *App) localDNSCorefileUpdateFromAgentPoolConfig(selected localDNSAgentPoolConfig, update localDNSCorefileUpdate) (localDNSCorefileUpdate, error) { + switch { + case strings.TrimSpace(selected.Corefile) != "": + update.corefile = selected.Corefile + update.hasCorefile = true + return update, nil + case strings.TrimSpace(selected.CoreFile) != "": + update.corefile = selected.CoreFile + update.hasCorefile = true + return update, nil + case strings.TrimSpace(selected.CorefileBase64) != "": + return update.withCorefileBase64(selected.CorefileBase64) + case strings.TrimSpace(selected.CorefileBase64Alt) != "": + return update.withCorefileBase64(selected.CorefileBase64Alt) + } + profileJSON := selected.LocalDNSProfile + if len(profileJSON) == 0 { + profileJSON = selected.LocalDNSProfileAlt + } + if len(profileJSON) == 0 { + if selected.CorefileVersion != "" || selected.ConfigChecksum != "" { + slog.Info("localDNS LPS config has only version/checksum; Corefile content is required for bootstrap mutation", + "corefileVersion", selected.CorefileVersion, "configChecksum", selected.ConfigChecksum) + } + return update, nil + } + profile := &aksnodeconfigv1.LocalDnsProfile{} + unmarshalOptions := protojson.UnmarshalOptions{DiscardUnknown: true} + if err := unmarshalOptions.Unmarshal(profileJSON, profile); err != nil { + return localDNSCorefileUpdate{}, fmt.Errorf("parsing localDNS profile: %w", err) + } + if !profile.GetEnableLocalDns() { + return update, nil + } + nodeConfig, err := a.nodeConfigWithLocalDNSProfile(profile) + if err != nil { + return localDNSCorefileUpdate{}, err + } + includeHostsPlugin := profile.GetEnableHostsPlugin() + if includeHostsPlugin { + if _, statErr := os.Stat(localDNSHostsFilePath); statErr != nil { + includeHostsPlugin = false + } + } + corefile, err := parser.GenerateLocalDNSCorefileFromAKSNodeConfig(nodeConfig, includeHostsPlugin) + if err != nil { + return localDNSCorefileUpdate{}, err + } + update.corefile = corefile + update.hasCorefile = true + return update, nil +} + +func (u localDNSCorefileUpdate) withCorefileBase64(v string) (localDNSCorefileUpdate, error) { + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(v)) + if err != nil { + return localDNSCorefileUpdate{}, fmt.Errorf("decoding localDNS corefileBase64: %w", err) + } + if len(strings.TrimSpace(string(decoded))) == 0 { + return u, nil + } + u.corefile = string(decoded) + u.hasCorefile = true + return u, nil +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func (a *App) nodeAgentPoolName() (string, error) { + path := a.getNodeConfigPath() + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("reading node config %s: %w", path, err) + } + cfg, perr := nodeconfigutils.UnmarshalConfigurationV1(raw) + if perr != nil { + slog.Info("node config parsed with errors, continuing with partial config", "error", perr) + } + if cfg == nil { + return "", fmt.Errorf("node config %s could not be parsed", path) + } + agentPool := cfg.GetKubeletConfig().GetKubeletNodeLabels()[localDNSAgentPoolLabel] + if agentPool == "" { + return "", fmt.Errorf("node config has no %s kubelet node label", localDNSAgentPoolLabel) + } + return agentPool, nil +} + +func (a *App) nodeConfigWithLocalDNSProfile(profile *aksnodeconfigv1.LocalDnsProfile) (*aksnodeconfigv1.Configuration, error) { + path := a.getNodeConfigPath() + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading node config %s: %w", path, err) + } + cfg, perr := nodeconfigutils.UnmarshalConfigurationV1(raw) + if perr != nil { + slog.Info("node config parsed with errors, continuing with partial config", "error", perr) + } + if cfg == nil { + return nil, fmt.Errorf("node config %s could not be parsed", path) + } + cfg.LocalDnsProfile = profile + return cfg, nil +} + +func writeLocalDNSCorefile(path string, corefile string) error { + if strings.TrimSpace(corefile) == "" { + return fmt.Errorf("localDNS corefile is empty") + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("create parent directory: %w", err) + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".localdns-corefile-*") + if err != nil { + return fmt.Errorf("creating temp file: %w", err) + } + tmpPath := tmp.Name() + if _, err := io.WriteString(tmp, corefile); err != nil { + tmp.Close() + os.Remove(tmpPath) + return fmt.Errorf("writing temp file: %w", err) + } + if err := tmp.Close(); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("closing temp file: %w", err) + } + if err := os.Chmod(tmpPath, 0644); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("chmod temp file: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("renaming temp file: %w", err) + } + return nil +} + +func localDNSCorefileVersionPath(corefilePath string) string { + return corefilePath + ".version" +} + +func readLocalDNSCorefileVersion(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", fmt.Errorf("reading localDNS corefile version %s: %w", path, err) + } + return strings.TrimSpace(string(data)), nil +} + +func writeLocalDNSCorefileVersion(path string, version string) error { + if strings.TrimSpace(version) == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("create parent directory: %w", err) + } + return os.WriteFile(path, []byte(strings.TrimSpace(version)+"\n"), 0600) +} diff --git a/aks-node-controller/localdnsconfig_test.go b/aks-node-controller/localdnsconfig_test.go new file mode 100644 index 00000000000..ddfbf2beefe --- /dev/null +++ b/aks-node-controller/localdnsconfig_test.go @@ -0,0 +1,221 @@ +package main + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" +) + +func writeLocalDNSTestNodeConfig(t *testing.T, app *App) { + t.Helper() + p := filepath.Join(t.TempDir(), "aks-node-controller-config.json") + require.NoError(t, os.WriteFile(p, []byte(fmt.Sprintf(`{ + "version": "v1", + "kubelet_config": { + "kubelet_node_labels": { + "kubernetes.azure.com/agentpool": %q + } + } +}`, "pool1")), 0o600)) + app.nodeConfigPath = p +} + +func TestFetchAndApplyLocalDNSConfig(t *testing.T) { + t.Run("corefileBase64 rewrites output", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + want := ".:53 {\n forward . 168.63.129.16\n}\n" + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"corefileBase64":"` + base64.StdEncoding.EncodeToString([]byte(want)) + `"}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + }) + + t.Run("agent pool corefileBase64 rewrites output", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + want := ".:53 {\n forward . 168.63.129.16\n reload\n}\n" + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123","corefileBase64":"` + + base64.StdEncoding.EncodeToString([]byte(want)) + `"},"pool2":{"corefileBase64":"ignored"}}}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + version, err := os.ReadFile(localDNSCorefileVersionPath(out)) + require.NoError(t, err) + assert.Equal(t, "abc123\n", string(version)) + }) + + t.Run("already current version and content skips rewrite", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + out := filepath.Join(t.TempDir(), "localdns.corefile") + original := ".:53 {\n forward . 1.1.1.1\n}\n" + require.NoError(t, os.WriteFile(out, []byte(original), 0o644)) + require.NoError(t, writeLocalDNSCorefileVersion(localDNSCorefileVersionPath(out), "abc123")) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123","corefileBase64":"` + + base64.StdEncoding.EncodeToString([]byte(original)) + `"}}}`, nil + } + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigAlreadyCurrent, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, original, string(got)) + }) + + t.Run("matching version with stale corefile rewrites output", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + out := filepath.Join(t.TempDir(), "localdns.corefile") + require.NoError(t, os.WriteFile(out, []byte("stale-corefile"), 0o644)) + require.NoError(t, writeLocalDNSCorefileVersion(localDNSCorefileVersionPath(out), "abc123")) + want := ".:53 {\n forward . 168.63.129.16\n}\n" + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123","corefileBase64":"` + + base64.StdEncoding.EncodeToString([]byte(want)) + `"}}}`, nil + } + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + }) + + t.Run("agent pool version only config is no-op", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool1":{"corefileVersion":"abc123"}}}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigNoCorefileData, outcome) + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr)) + }) + + t.Run("agent pool localDnsProfile renders corefile", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{ + "agentPools": { + "pool1": { + "corefileVersion": "profile-hash", + "localDnsProfile": { + "enableLocalDns": true, + "vnetDnsOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDnsOverrides": { + "cluster.local": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } + } +}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigApplied, outcome) + got, err := os.ReadFile(out) + require.NoError(t, err) + assert.Contains(t, string(got), "health-check.localdns.local:53") + assert.Contains(t, string(got), "cluster.local:53") + version, err := os.ReadFile(localDNSCorefileVersionPath(out)) + require.NoError(t, err) + assert.Equal(t, "profile-hash\n", string(version)) + }) + + t.Run("other agent pool config is no-op", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + writeLocalDNSTestNodeConfig(t, tt.App) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return `{"agentPools":{"pool2":{"corefileBase64":"` + base64.StdEncoding.EncodeToString([]byte("ignored")) + `"}}}`, nil + } + out := filepath.Join(t.TempDir(), "localdns.corefile") + + outcome, err := tt.App.fetchAndApplyLocalDNSConfig(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, outcomeLocalDNSConfigNoCorefileData, outcome) + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr)) + }) + + t.Run("cli action fails open", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.fetchLocalDNSConfigFn = func(context.Context) (string, error) { + return "", assert.AnError + } + exitCode := tt.App.Run(context.Background(), []string{"aks-node-controller", "fetch-localdns-config", "--output", filepath.Join(t.TempDir(), "localdns.corefile")}) + assert.Equal(t, 0, exitCode) + }) +} + +func TestLocalDNSLPSUnavailableStatusCode(t *testing.T) { + tests := []struct { + name string + code codes.Code + statusCode int + ok bool + }{ + {name: "not found", code: codes.NotFound, statusCode: http.StatusNotFound, ok: true}, + {name: "permission denied", code: codes.PermissionDenied, statusCode: http.StatusForbidden, ok: true}, + {name: "unauthenticated", code: codes.Unauthenticated, statusCode: http.StatusUnauthorized, ok: true}, + {name: "internal", code: codes.Internal, ok: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + statusCode, ok := localDNSLPSUnavailableStatusCode(tt.code) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.statusCode, statusCode) + }) + } +} diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 9cc9e7eecca..0f348a40f42 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -894,7 +894,7 @@ type localDnsCorefileTemplateData struct { // Corefile is created using localdns.toml.gtpl template and aksnodeconfig values. // includeHostsPlugin controls whether the hosts plugin block is included in the generated Corefile. -func generateLocalDnsCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, includeHostsPlugin bool) (string, error) { +func GenerateLocalDNSCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, includeHostsPlugin bool) (string, error) { var corefileBuffer bytes.Buffer templateData := localDnsCorefileTemplateData{ Config: aksnodeconfig, @@ -906,6 +906,10 @@ func generateLocalDnsCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Co return corefileBuffer.String(), nil } +func generateLocalDnsCorefileFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, includeHostsPlugin bool) (string, error) { + return GenerateLocalDNSCorefileFromAKSNodeConfig(aksnodeconfig, includeHostsPlugin) +} + // getLocalDnsClusterListenerIp returns APIPA-IP address that will be used in localdns systemd unit. func getLocalDnsClusterListenerIp() string { return localDnsClusterListenerIp diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 567431d225b..65382dbd732 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -1687,6 +1687,7 @@ health-check.localdns.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -1714,6 +1715,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -1731,6 +1733,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -1756,6 +1759,7 @@ testdomain456.com:53 { max_concurrent 2000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 diff --git a/aks-node-controller/parser/templates/localdns.toml.gtpl b/aks-node-controller/parser/templates/localdns.toml.gtpl index 818b23aa421..191cee3cbcb 100644 --- a/aks-node-controller/parser/templates/localdns.toml.gtpl +++ b/aks-node-controller/parser/templates/localdns.toml.gtpl @@ -47,6 +47,7 @@ health-check.localdns.local:53 { max_concurrent {{$override.MaxConcurrent}} } ready {{getLocalDnsNodeListenerIp}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 @@ -112,6 +113,7 @@ health-check.localdns.local:53 { max_concurrent {{$override.MaxConcurrent}} } ready {{getLocalDnsClusterListenerIp}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index c83eba792a9..32b598c7df6 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -1,12 +1,30 @@ package e2e import ( + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "os" + "strings" "testing" + "time" aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + desiredLocalDNSVersion = "e2e-localdns-corefile-version" + localDNSPayloadPath = "/opt/azure/containers/localdns/e2e-localdns-lps-payload.json" + localDNSFetcherStamp = "/opt/azure/containers/localdns/e2e-localdns-lps-fetcher-called" + localDNSBranchScriptArchivePath = "/opt/azure/containers/localdns/e2e-localdns.sh.gz.b64" + localDNSFetcherPath = "/opt/azure/containers/localdns/e2e-fetch-localdns-config" ) // Test_LocalDNSHostsPlugin tests the localdns hosts plugin across all supported distros @@ -56,3 +74,167 @@ func Test_LocalDNSHostsPlugin(t *testing.T) { }) } } + +// Test_LocalDNSLPSBootstrapPatch validates the node-side LocalDNS live-patching +// bootstrap path. It simulates LPS by temporarily wrapping aks-node-controller's +// fetch-localdns-config command so it returns a LocalDNS nodeConfig payload, then +// delegates to the real apply-localdns-config implementation. The test verifies that +// localdns.sh: +// 1. invokes the fetcher before CoreDNS starts, +// 2. renders the supplied LocalDNS profile payload into updated.localdns.corefile, +// 3. persists the paired corefileVersion, and +// 4. stamps components.localDNS.current after kubeconfig/node registration. +func Test_LocalDNSLPSBootstrapPatch(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests LocalDNS LPS bootstrap patching applies Corefile and reports corefileVersion", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.LocalDNSProfile.EnableLocalDNS = true + }, + CustomDataWriteFiles: []CustomDataWriteFile{ + { + Path: localDNSBranchScriptArchivePath, + Permissions: "0644", + Owner: "root", + Content: mustReadCompressedLocalDNSArtifact(t), + }, + { + Path: "/etc/systemd/system/localdns.service.d/00-e2e-branch-localdns.conf", + Permissions: "0644", + Owner: "root", + Content: localDNSBranchScriptDropIn(), + }, + { + Path: localDNSPayloadPath, + Permissions: "0644", + Owner: "root", + Content: localDNSLPSPayload(desiredLocalDNSVersion), + }, + { + Path: localDNSFetcherPath, + Permissions: "0755", + Owner: "root", + Content: localDNSLPSFetcherWrapper(), + }, + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.LocalDnsProfile.EnableLocalDns = true + }, + Validator: validateLocalDNSLPSBootstrapPatch, + }, + }) +} + +func mustReadCompressedLocalDNSArtifact(t *testing.T) string { + t.Helper() + data, err := os.ReadFile("../parts/linux/cloud-init/artifacts/localdns.sh") + require.NoError(t, err) + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err = zw.Write(data) + require.NoError(t, err) + require.NoError(t, zw.Close()) + content := strings.ReplaceAll(string(data), `AKS_NODE_CONTROLLER_BINARY="/opt/azure/containers/aks-node-controller"`, `AKS_NODE_CONTROLLER_BINARY="`+localDNSFetcherPath+`"`) + buf.Reset() + zw = gzip.NewWriter(&buf) + _, err = zw.Write([]byte(content)) + require.NoError(t, err) + require.NoError(t, zw.Close()) + return base64.StdEncoding.EncodeToString(buf.Bytes()) +} + +func localDNSBranchScriptDropIn() string { + return `[Service] +ExecStartPre=/bin/bash -c 'base64 -d ` + localDNSBranchScriptArchivePath + ` | gzip -d > /opt/azure/containers/localdns.sh && chmod 0544 /opt/azure/containers/localdns.sh' +` +} + +func localDNSLPSPayload(version string) string { + return `{ + "agentPools": { + "nodepool2": { + "corefileVersion": "` + version + `", + "localDnsProfile": { + "enableLocalDns": true, + "vnetDnsOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDnsOverrides": { + "cluster.local": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } + } +}` +} + +func localDNSLPSFetcherWrapper() string { + return `#!/bin/bash +set -euo pipefail +if [ "${1:-}" != "fetch-localdns-config" ]; then + exec /opt/azure/containers/aks-node-controller "$@" +fi +output="" +while [ "$#" -gt 0 ]; do + case "$1" in + --output) + output="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +if [ -z "$output" ]; then + echo "missing --output" >&2 + exit 1 +fi +touch ` + localDNSFetcherStamp + ` +exec /opt/azure/containers/aks-node-controller apply-localdns-config --config-file ` + localDNSPayloadPath + ` --output "$output" +` +} + +func validateLocalDNSLPSBootstrapPatch(ctx context.Context, s *Scenario) { + const ( + updatedCorefile = "/opt/azure/containers/localdns/updated.localdns.corefile" + livepatchedCorefile = "/opt/azure/containers/localdns/livepatched.localdns.corefile" + ) + + ValidateFileExists(ctx, s, localDNSFetcherStamp) + ValidateFileHasContent(ctx, s, updatedCorefile, "health-check.localdns.local:53") + ValidateFileHasContent(ctx, s, updatedCorefile, "cluster.local:53") + ValidateFileHasContent(ctx, s, livepatchedCorefile+".version", desiredLocalDNSVersion) + ValidateLocalDNSService(ctx, s, "enabled") + ValidateLocalDNSResolution(ctx, s, "169.254.10.10") + + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + status := node.Annotations["kubernetes.azure.com/live-patching-status"] + return strings.Contains(status, `"localDNS":{"current":"`+desiredLocalDNSVersion+`"}`), nil + }) + require.NoError(s.T, err, "node did not report LocalDNS live-patching current version %q", desiredLocalDNSVersion) +} diff --git a/parts/linux/cloud-init/artifacts/localdns.sh b/parts/linux/cloud-init/artifacts/localdns.sh index 6f07c1ee455..0156e276201 100644 --- a/parts/linux/cloud-init/artifacts/localdns.sh +++ b/parts/linux/cloud-init/artifacts/localdns.sh @@ -24,6 +24,8 @@ LOCALDNS_CGROUP_DIR="/sys/fs/cgroup/localdns.slice/localdns.service" LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" # This is the localdns corefile that has updated UpstreamDNSServerIPs and will be used by the localdns systemd unit. +LIVEPATCHED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/livepatched.localdns.corefile" + UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" # This is slice file used by localdns systemd unit. @@ -69,6 +71,10 @@ START_LOCALDNS_TIMEOUT=10 DNS_HEALTH_CHECK_TIMEOUT=2 DNS_HEALTH_CHECK_TRIES=2 +AKS_NODE_CONTROLLER_BINARY="/opt/azure/containers/aks-node-controller" +LOCALDNS_LIVE_PATCHING_COMPONENT_NAME="localDNS" +LOCALDNS_LIVE_PATCHING_STATUS_ANNOTATION="kubernetes.azure.com/live-patching-status" + # Function definitions used in this file. # functions defined until "${__SOURCED__:+return}" are sourced and tested in - # spec/parts/linux/cloud-init/artifacts/localdns_spec.sh. @@ -157,6 +163,14 @@ regenerate_localdns_corefile() { return 0 } +localdns_source_corefile() { + if [ -s "${LIVEPATCHED_LOCALDNS_CORE_FILE:-}" ]; then + echo "${LIVEPATCHED_LOCALDNS_CORE_FILE}" + return 0 + fi + echo "${LOCALDNS_CORE_FILE}" +} + # Replace AzureDNSIP in corefile with VNET DNS ServerIPs if necessary. replace_azurednsip_in_corefile() { if [ -z "${RESOLV_CONF:-}" ]; then @@ -187,8 +201,10 @@ replace_azurednsip_in_corefile() { # and also not equal to the localdns node listener IP to avoid creating a circular dependency. # Corefile will have 168.63.129.16 when user input has VnetDNS value for forwarddestination. # Note - For root domain under VnetDNSOverrides, all DNS traffic should be forwarded to VnetDNS. - cp "${LOCALDNS_CORE_FILE}" "${UPDATED_LOCALDNS_CORE_FILE}" || { - echo "Failed to copy ${LOCALDNS_CORE_FILE} to ${UPDATED_LOCALDNS_CORE_FILE}" + local source_corefile + source_corefile="$(localdns_source_corefile)" + cp "${source_corefile}" "${UPDATED_LOCALDNS_CORE_FILE}" || { + echo "Failed to copy ${source_corefile} to ${UPDATED_LOCALDNS_CORE_FILE}" return 1 } @@ -304,6 +320,119 @@ replace_azurednsip_in_corefile() { return 0 } +refresh_localdns_corefile_from_lps() { + if [ -z "${LIVEPATCHED_LOCALDNS_CORE_FILE:-}" ]; then + echo "LIVEPATCHED_LOCALDNS_CORE_FILE is not set or is empty." + return 1 + fi + + if [ ! -x "${AKS_NODE_CONTROLLER_BINARY}" ]; then + echo "AKS node controller binary not found at ${AKS_NODE_CONTROLLER_BINARY}; skipping LocalDNS LPS config fetch." + return 0 + fi + + # Write the LPS-provided Corefile to the livepatched source file; VNET DNS replacement + # later derives UPDATED_LOCALDNS_CORE_FILE from this file before CoreDNS starts. + if "${AKS_NODE_CONTROLLER_BINARY}" fetch-localdns-config --output "${LIVEPATCHED_LOCALDNS_CORE_FILE}"; then + echo "Completed LocalDNS LPS config fetch." + return 0 + fi + + echo "LocalDNS LPS config fetch failed; continuing with existing corefile." + return 0 +} + +localdns_corefile_version_file() { + echo "${LIVEPATCHED_LOCALDNS_CORE_FILE}.version" +} + +wait_for_kubeconfig_and_node() { + if [ ! -x /opt/bin/kubectl ]; then + echo "kubectl binary not found at /opt/bin/kubectl, skipping annotation." >&2 + return 1 + fi + + local kubeconfig="${KUBECONFIG:-/var/lib/kubelet/kubeconfig}" + local wait_count=0 + local max_wait="${KUBECONFIG_WAIT_ATTEMPTS:-60}" + while [ ! -f "${kubeconfig}" ]; do + if [ $wait_count -ge $max_wait ]; then + echo "Timeout waiting for kubeconfig at ${kubeconfig} after ${max_wait} attempts, skipping annotation." >&2 + return 1 + fi + echo "Waiting for TLS bootstrapping to complete (attempt $((wait_count + 1))/${max_wait})..." >&2 + sleep 3 + wait_count=$((wait_count + 1)) + done + echo "Kubeconfig found at ${kubeconfig}" >&2 + + local node_name + node_name=$(hostname) + if [ -z "${node_name}" ]; then + echo "Cannot get node name, skipping annotation." >&2 + return 1 + fi + node_name=$(echo "$node_name" | tr '[:upper:]' '[:lower:]') + + echo "Waiting for node ${node_name} to be registered in the cluster..." >&2 + local node_wait_count=0 + local max_node_wait="${NODE_REGISTRATION_WAIT_ATTEMPTS:-30}" + while [ $node_wait_count -lt $max_node_wait ]; do + if /opt/bin/kubectl --kubeconfig "${kubeconfig}" get node "${node_name}" >/dev/null 2>&1; then + echo "${kubeconfig}|${node_name}" + return 0 + fi + echo "Waiting for node registration (attempt $((node_wait_count + 1))/${max_node_wait})..." >&2 + sleep 3 + node_wait_count=$((node_wait_count + 1)) + done + + echo "Timeout waiting for node ${node_name} to be registered after ${max_node_wait} attempts, skipping annotation." >&2 + return 1 +} + +annotate_node_with_localdns_livepatch_status() { + local version_file + version_file="$(localdns_corefile_version_file)" + if [ ! -s "${version_file}" ]; then + echo "LocalDNS corefile version file not found at ${version_file}, skipping live patching status annotation." + return 0 + fi + + local corefile_version + corefile_version="$(tr -d '[:space:]' < "${version_file}")" + if [ -z "${corefile_version}" ]; then + echo "LocalDNS corefile version file is empty, skipping live patching status annotation." + return 0 + fi + + local kube_node + kube_node="$(wait_for_kubeconfig_and_node)" || return 0 + local kubeconfig="${kube_node%%|*}" + local node_name="${kube_node#*|}" + local current_status + current_status=$(/opt/bin/kubectl --kubeconfig "${kubeconfig}" get node "${node_name}" -o "jsonpath={.metadata.annotations['kubernetes\.azure\.com/live-patching-status']}" 2>/dev/null || true) + if [ -z "${current_status}" ]; then + current_status='{}' + fi + + local updated_status + if ! updated_status="$(printf '%s' "${current_status}" | jq -c \ + --arg component "${LOCALDNS_LIVE_PATCHING_COMPONENT_NAME}" \ + --arg current "${corefile_version}" \ + '.components = (.components // {}) | .components[$component].current = $current')"; then + echo "Failed to render LocalDNS live patching status annotation." + return 0 + fi + + echo "Setting LocalDNS live patching current version ${corefile_version} for node ${node_name}." + if /opt/bin/kubectl --kubeconfig "${kubeconfig}" annotate --overwrite node "${node_name}" "${LOCALDNS_LIVE_PATCHING_STATUS_ANNOTATION}=${updated_status}"; then + echo "Successfully set LocalDNS live patching status annotation." + else + echo "Warning: Failed to set LocalDNS live patching status annotation (this is non-fatal)." + fi +} + # Build iptables rules to skip conntrack for DNS traffic to localdns. build_localdns_iptable_rules() { # These rules skip conntrack for DNS traffic to the local DNS service IPs to save conntrack table space. @@ -1026,6 +1155,12 @@ if ! wait_for_localdns_removed_from_resolv_conf 5; then exit $ERR_LOCALDNS_FAIL fi +# Fetch LocalDNS config from LPS if present. This is fail-open: no config or fetch errors keep the +# locally generated corefile. If LPS returns a usable profile/corefile, LIVEPATCHED_LOCALDNS_CORE_FILE +# is written before VNET DNS replacement builds UPDATED_LOCALDNS_CORE_FILE for CoreDNS. +# --------------------------------------------------------------------------------------------------------------------- +refresh_localdns_corefile_from_lps + # Replace AzureDNSIP in corefile with VNET DNS ServerIPs. # --------------------------------------------------------------------------------------------------------------------- replace_azurednsip_in_corefile || exit $ERR_LOCALDNS_FAIL @@ -1073,6 +1208,17 @@ echo "Startup complete - serving node and pod DNS traffic." # Export initial resource metrics so the exporter has data before the first watchdog tick. export_resource_metrics +# The generic knead live-patching loop owns kubernetes.azure.com/live-patching-status at runtime. +# Keep this legacy/bootstrap writer opt-in to avoid racing knead's status update. +# -------------------------------------------------------------------------------------------------------------------- +if [ "${LOCALDNS_ENABLE_LEGACY_LIVEPATCH_STATUS:-false}" = "true" ]; then + annotate_node_with_localdns_livepatch_status & + LOCALDNS_LIVEPATCH_ANNOTATION_PID=$! + echo "Started LocalDNS live-patching status annotation in background (PID: ${LOCALDNS_LIVEPATCH_ANNOTATION_PID})" +else + echo "Skipping LocalDNS live-patching status annotation; knead owns live-patching-status." +fi + # Set node annotation to indicate hosts plugin is in use (if applicable). # -------------------------------------------------------------------------------------------------------------------- # Only run when hosts plugin is currently enabled or was previously enabled (marker exists). diff --git a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh index b2e986a2d41..a23e4c80b37 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @@ -237,6 +237,10 @@ knead_apply_components() { component_comparator=securityPatchIsCurrent component_handler=updateSecurityPatch ;; + localDNS) + component_comparator=localDNSIsCurrent + component_handler=updateLocalDNS + ;; *) echo "unsupported component: ${component}" component_index=$((component_index + 1)) @@ -278,6 +282,57 @@ knead_apply_components() { } # Records the processed hash and per-component results in the node status annotation. +localDNSIsCurrent() { + local desired_payload="$1" + local current_payload="$2" + + [ "${desired_payload}" = "${current_payload}" ] +} + +updateLocalDNS() { + local component_payload="${1:-}" + local outcome + + if [ ! -x /opt/azure/containers/aks-node-controller ]; then + echo "aks-node-controller binary is required for localDNS live patching" + return 1 + fi + + if ! outcome="$(printf '%s' "${component_payload}" | /opt/azure/containers/aks-node-controller apply-localdns-config \ + --config-file - \ + --output /opt/azure/containers/localdns/livepatched.localdns.corefile)"; then + echo "localDNS config apply failed" + return 1 + fi + printf '%s +' "${outcome}" + + case "$(printf '%s +' "${outcome}" | tail -n 1)" in + applied) + if ! systemctl restart localdns.service; then + echo "failed to restart localdns.service" + return 1 + fi + echo "localDNS update completed successfully" + ;; + alreadyCurrent) + echo "localDNS is already current" + ;; + notFound) + echo "localDNS LPS config is not available" + ;; + noCorefileData) + echo "localDNS LPS config has no node-applicable payload" + return 1 + ;; + *) + echo "unexpected localDNS apply outcome: ${outcome}" + return 1 + ;; + esac +} + knead_write_status() { local node_name="$1" local goal="$2" diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 873790f8eff..f0801a0b70a 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -2310,6 +2310,7 @@ health-check.localdns.local:53 { max_concurrent {{$override.MaxConcurrent}} } ready {{$.NodeListenerIP}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 @@ -2375,6 +2376,7 @@ health-check.localdns.local:53 { max_concurrent {{$override.MaxConcurrent}} } ready {{$.ClusterListenerIP}}:8181 + reload cache {{$override.CacheDurationInSeconds}} { success 9984 denial 9984 diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 9ea69ecdc4e..67c4a2a7adc 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -474,6 +474,7 @@ health-check.localdns.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -501,6 +502,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -518,6 +520,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -543,6 +546,7 @@ testdomain456.com:53 { max_concurrent 2000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 @@ -663,6 +667,7 @@ health-check.localdns.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -690,6 +695,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -707,6 +713,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.10:8181 + reload cache 3600 { success 9984 denial 9984 @@ -732,6 +739,7 @@ testdomain456.com:53 { max_concurrent 1000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 @@ -759,6 +767,7 @@ cluster.local:53 { max_concurrent 1000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 @@ -776,6 +785,7 @@ testdomain567.com:53 { max_concurrent 1000 } ready 169.254.10.11:8181 + reload cache 3600 { success 9984 denial 9984 diff --git a/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh b/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh index 65fbaae2cb9..6b061521fe2 100644 --- a/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/localdns_spec.sh @@ -23,6 +23,7 @@ Describe 'localdns.sh' TEST_DIR="/tmp/localdnstest" LOCALDNS_SCRIPT_PATH="${TEST_DIR}/opt/azure/containers/localdns" LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" + LIVEPATCHED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/livepatched.localdns.corefile" UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" mkdir -p "$LOCALDNS_SCRIPT_PATH" # Use production-realistic corefile format with brace syntax @@ -391,6 +392,34 @@ EOF The stdout should include "Successfully exported forward IPs to ${LOCALDNS_SCRIPT_PATH}/forward_ips.prom" End + It 'should fetch LocalDNS LPS config through aks-node-controller when binary exists' + AKS_NODE_CONTROLLER_BINARY="${TEST_DIR}/aks-node-controller" + cat > "${AKS_NODE_CONTROLLER_BINARY}" <<'EOF' +#!/bin/bash +echo "anc args: $*" +exit 0 +EOF + chmod +x "${AKS_NODE_CONTROLLER_BINARY}" + + When run refresh_localdns_corefile_from_lps + The status should be success + The output should include "anc args: fetch-localdns-config --output ${LIVEPATCHED_LOCALDNS_CORE_FILE}" + The output should include "Completed LocalDNS LPS config fetch." + End + + It 'should skip LocalDNS LPS config fetch when aks-node-controller binary is missing' + AKS_NODE_CONTROLLER_BINARY="${TEST_DIR}/missing-aks-node-controller" + When run refresh_localdns_corefile_from_lps + The status should be success + The output should include "AKS node controller binary not found at ${AKS_NODE_CONTROLLER_BINARY}; skipping LocalDNS LPS config fetch." + End + + It 'should skip LocalDNS live patching status annotation when version file is missing' + When run annotate_node_with_localdns_livepatch_status + The status should be success + The output should include "LocalDNS corefile version file not found at ${LIVEPATCHED_LOCALDNS_CORE_FILE}.version, skipping live patching status annotation." + End + It 'should set correct permissions on forward_ips.prom file' When run replace_azurednsip_in_corefile The status should be success @@ -1960,3 +1989,50 @@ KUBECTL_EOF End End End + + + Describe 'livepatched corefile source selection' + setup() { + Include "./parts/linux/cloud-init/artifacts/localdns.sh" + TEST_DIR="/tmp/localdns-livepatched-test" + LOCALDNS_SCRIPT_PATH="${TEST_DIR}/opt/azure/containers/localdns" + LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" + LIVEPATCHED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/livepatched.localdns.corefile" + UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" + RESOLV_CONF="${TEST_DIR}/run/systemd/resolve/resolv.conf" + mkdir -p "${LOCALDNS_SCRIPT_PATH}" "$(dirname "${RESOLV_CONF}")" + echo 'nameserver 10.0.0.1' > "${RESOLV_CONF}" + } + cleanup() { + rm -rf "${TEST_DIR}" + } + BeforeEach 'setup' + AfterEach 'cleanup' + + It 'uses the livepatched Corefile when present' + printf '.:53 { + forward . 9.9.9.9 +} +' > "${LOCALDNS_CORE_FILE}" + printf '.:53 { + forward . 168.63.129.16 +} +' > "${LIVEPATCHED_LOCALDNS_CORE_FILE}" + When run replace_azurednsip_in_corefile + The status should be success + The output should include 'Successfully updated' + The contents of file "${UPDATED_LOCALDNS_CORE_FILE}" should include '10.0.0.1' + The contents of file "${UPDATED_LOCALDNS_CORE_FILE}" should not include '9.9.9.9' + End + + It 'falls back to the generated Corefile when no livepatched Corefile exists' + printf '.:53 { + forward . 168.63.129.16 +} +' > "${LOCALDNS_CORE_FILE}" + When run replace_azurednsip_in_corefile + The status should be success + The output should include 'Successfully updated' + The contents of file "${UPDATED_LOCALDNS_CORE_FILE}" should include '10.0.0.1' + End + End diff --git a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh index 78692c3f109..07e748a00f3 100644 --- a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @@ -21,9 +21,10 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' TEST_REPO_SERVICE="" printf '%s' '{"components":[]}' > "${TEST_COMPONENTS_JSON_FILE}" TEST_SECURITY_STATUS=0 + TEST_LOCALDNS_STATUS=0 TEST_ANNOTATE_STATUS=0 export KUBECTL KNEAD_COMPONENT_STATE_FILE TEST_COMPONENTS_JSON_FILE TEST_KUBECTL_ARGS_FILE - export TEST_STATUS TEST_GOAL TEST_AGENT_POOL TEST_REPO_SERVICE TEST_SECURITY_STATUS TEST_ANNOTATE_STATUS + export TEST_STATUS TEST_GOAL TEST_AGENT_POOL TEST_REPO_SERVICE TEST_SECURITY_STATUS TEST_LOCALDNS_STATUS TEST_ANNOTATE_STATUS } cleanup() { @@ -66,6 +67,12 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' return "${TEST_SECURITY_STATUS}" } + + updateLocalDNS() { + echo "updateLocalDNS called with args: $*" + return "${TEST_LOCALDNS_STATUS}" + } + securityPatchIsCurrent() { local desired_payload="$1" local current_payload="$2" @@ -157,6 +164,54 @@ Describe 'ubuntu-snapshot-update.sh generic reconciliation' The contents of file "${TEST_KUBECTL_ARGS_FILE}" should equal 'get cm -n kube-system live-patching-config -o jsonpath={.data.live-patching-config\.json}' End + + It 'dispatches localDNS and writes successful status' + mkdir -p /opt/azure/containers + cat > /opt/azure/containers/aks-node-controller <<'EOF' +#!/bin/bash +echo "aks-node-controller called with args: $*" +cat > /tmp/localdns-livepatch-payload +echo applied +EOF + chmod +x /opt/azure/containers/aks-node-controller + Mock systemctl + echo "systemctl called with args: $*" + End + set_payload_goal '{"components":[{"name":"localDNS","nodeConfig":"{\"profiles\":{\"ap1\":{\"configChecksum\":\"localdns-v1\"}}}"}]}' + + When call knead_main + The status should be success + The output should include 'applying component: localDNS' + The output should include 'aks-node-controller called with args: apply-localdns-config --config-file - --output /opt/azure/containers/localdns/livepatched.localdns.corefile' + The contents of file "/tmp/localdns-livepatch-payload" should include '"configChecksum":"localdns-v1"' + The output should include 'systemctl called with args: restart localdns.service' + The output should include 'localDNS update completed successfully' + The output should include 'annotate mock called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-status={"currentHash":"' + The output should include '"components":{"localDNS":{"code":"Succeeded"}}}' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"localDNS"' + End + + It 'marks localDNS failed when service restart fails' + mkdir -p /opt/azure/containers + cat > /opt/azure/containers/aks-node-controller <<'EOF' +#!/bin/bash +echo applied +EOF + chmod +x /opt/azure/containers/aks-node-controller + Mock systemctl + echo "systemctl called with args: $*" + exit 1 + End + set_payload_goal '{"components":[{"name":"localDNS","nodeConfig":"{\"profiles\":{\"ap1\":{\"configChecksum\":\"localdns-v1\"}}}"}]}' + + When call knead_main + The status should be failure + The output should include 'applying component: localDNS' + The output should include 'failed to restart localdns.service' + The output should include 'component failed: localDNS' + The output should include '"components":{"localDNS":{"code":"Failed"}}}' + End + It 'fails before dispatch when the goal hash does not match the ConfigMap payload' printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"}]}' > "${TEST_COMPONENTS_JSON_FILE}" TEST_GOAL="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" From ca116f78aed223371879011abeae858ca8c0f179 Mon Sep 17 00:00:00 2001 From: Saewon Kwak <23280628+saewoni@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:44:16 +0000 Subject: [PATCH 2/2] Refresh PR after clean rebuild Force GitHub to rebuild the stale PR merge ref after replacing the stacked branch with a clean LocalDNS commit on top of feature/knead-security-patching. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>