diff --git a/k8up/.helmignore b/k8up/.helmignore new file mode 100644 index 00000000..3b832e50 --- /dev/null +++ b/k8up/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj + +*kustomize*/ +Makefile +*gotmpl* +test/ diff --git a/k8up/Chart.yaml b/k8up/Chart.yaml new file mode 100644 index 00000000..5e770b26 --- /dev/null +++ b/k8up/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +description: Kubernetes and OpenShift Backup Operator based on restic +home: https://k8up.io/ +keywords: +- backup +- operator +- restic +maintainers: +- email: info@appuio.ch + name: K8up Authors +name: k8up +sources: +- https://github.com/k8up-io/k8up +version: 4.4.1 diff --git a/k8up/README.md b/k8up/README.md new file mode 100644 index 00000000..c8b2a749 --- /dev/null +++ b/k8up/README.md @@ -0,0 +1,142 @@ +# k8up + +![Version: 4.4.1](https://img.shields.io/badge/Version-4.4.1-informational?style=flat-square) + +Kubernetes and OpenShift Backup Operator based on restic + +**Homepage:** + +## Installation + +```bash +helm repo add k8up-io https://k8up-io.github.io/k8up +helm install k8up k8up-io/k8up +``` +```bash +kubectl apply -f https://github.com/k8up-io/k8up/releases/download/k8up-4.4.1/k8up-crd.yaml +``` + + + +## Handling CRDs + +* Always upgrade the CRDs before upgrading the Helm release. +* Watch out for breaking changes in the K8up release notes. + +## Source Code + +* + + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| image.pullPolicy | string | `"IfNotPresent"` | Operator image pull policy | +| image.registry | string | `"ghcr.io"` | Operator image registry | +| image.repository | string | `"k8up-io/k8up"` | Operator image repository | +| image.tag | string | `"v2.7.1"` | Operator image tag (version) | +| imagePullSecrets | list | `[]` | | +| k8up.backupImage.repository | string | `""` | The backup runner image repository. Defaults to `{image.registry}/{image.repository}`. Specify an image repository including registry, e.g. `example.com/repo/image` | +| k8up.backupImage.tag | string | `""` | The backup runner image tag Defaults to `{image.tag}` | +| k8up.enableLeaderElection | bool | `true` | Specifies whether leader election should be enabled. | +| k8up.envVars | list | `[]` | envVars allows the specification of additional environment variables. See [values.yaml](values.yaml) how to specify See documentation which variables are supported. | +| k8up.globalResources | object | empty values | Specify the resource requests and limits that the Pods should have when they are scheduled by K8up. You are still able to override those via K8up resources, but this gives cluster administrators custom defaults. | +| k8up.globalResources.limits.cpu | string | `""` | Global CPU resource limit applied to jobs. See [supported units][resource-units]. | +| k8up.globalResources.limits.memory | string | `""` | Global Memory resource limit applied to jobs. See [supported units][resource-units]. | +| k8up.globalResources.requests.cpu | string | `""` | Global CPU resource requests applied to jobs. See [supported units][resource-units]. | +| k8up.globalResources.requests.memory | string | `""` | Global Memory resource requests applied to jobs. See [supported units][resource-units]. | +| k8up.operatorNamespace | string | `""` | Specifies the namespace in which K8up's `EffectiveSchedules` are stored. Defaults to release namespace if left empty. | +| k8up.timezone | string | `""` | Specifies the timezone K8up is using for scheduling. Empty value defaults to the timezone in which Kubernetes is deployed. Accepts `tz database` compatible entries, e.g. `Europe/Zurich` | +| metrics.prometheusRule.additionalLabels | object | `{}` | Add labels to the PrometheusRule object | +| metrics.prometheusRule.additionalRules | list | `[]` | Provide additional alert rules in addition to the defaults | +| metrics.prometheusRule.createDefaultRules | bool | `true` | Whether the default rules should be installed | +| metrics.prometheusRule.enabled | bool | `false` | Whether to enable PrometheusRule manifest for [Prometheus Operator][prometheus-operator] | +| metrics.prometheusRule.jobFailedRulesFor | list | `["archive","backup","check","prune","restore"]` | Create default rules for the given job types. Valid values are "archive", "backup", "check", "prune", and "restore". | +| metrics.prometheusRule.namespace | string | `""` | If the object should be installed in a different namespace than operator | +| metrics.service.annotations | object | `{}` | Annotations to add to the service | +| metrics.service.nodePort | int | `0` | Service node port of the metrics endpoint, requires `metrics.service.type=NodePort` | +| metrics.service.port | int | `8080` | | +| metrics.service.type | string | `"ClusterIP"` | | +| metrics.serviceMonitor.additionalLabels | object | `{}` | Add labels to the ServiceMonitor object | +| metrics.serviceMonitor.enabled | bool | `false` | Whether to enable ServiceMonitor manifests for [Prometheus Operator][prometheus-operator] | +| metrics.serviceMonitor.namespace | string | `""` | If the object should be installed in a different namespace than operator | +| metrics.serviceMonitor.scrapeInterval | string | `"60s"` | Scrape interval to collect metrics | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | Annotations to add to the Pod spec. | +| podSecurityContext | object | `{}` | Security context to add to the Pod spec. | +| rbac.create | bool | `true` | Create cluster roles and rolebinding. May need elevated permissions to create cluster roles and -bindings. | +| replicaCount | int | `1` | How many operator pods should run. Note: Operator features leader election for K8s 1.16 and later, so that only 1 pod is reconciling/scheduling jobs. Follower pods reduce interruption time as they're on hot standby when leader is unresponsive. | +| resources.limits.memory | string | `"256Mi"` | Memory limit of K8up operator. See [supported units][resource-units]. | +| resources.requests.cpu | string | `"20m"` | CPU request of K8up operator. See [supported units][resource-units]. | +| resources.requests.memory | string | `"128Mi"` | Memory request of K8up operator. See [supported units][resource-units]. | +| securityContext | object | `{}` | Container security context | +| serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | +| serviceAccount.create | bool | `true` | Specifies whether a service account should be created | +| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | +| tolerations | list | `[]` | | + +## Upgrading from Charts v0 to v1 + +* In `image.repository` the registry domain was moved into its own parameter `image.registry`. +* K8up 1.x features leader election, this enables rolling updates and multiple replicas. + `k8up.enableLeaderElection` defaults to `true`. Disable this for older Kubernetes versions (<= 1.15) +* `replicaCount` is now configurable, defaults to `1`. +* Note: Deployment strategy type has changed from `Recreate` to `RollingUpdate`. +* CRDs need to be installed separately, they are no longer included in this chart. + +## Upgrading from Charts v1 to v2 + +* Note: `image.repository` changed from `vshn/k8up` to `k8up-io/k8up`. +* Note: `image.registry` changed from `quay.io` to `ghcr.io`. +* Note: `image.tag` changed from `v1.x` to `v2.x`. Please see the [full changelog](https://github.com/k8up-io/k8up/releases/tag/v2.0.0). +* `metrics.prometheusRule.legacyRules` has been removed (no support for OpenShift 3.11 anymore). +* Note: `k8up.backupImage.repository` changed from `quay.io/vshn/wrestic` to `ghcr.io/k8up-io/k8up` (`wrestic` is not needed anymore in K8up v2). + +## Upgrading from Charts v2 to v3 + +Due to the migration of the chart from [APPUiO](https://github.com/appuio/charts/tree/master/appuio/k8up) to this repo, we decided to make a breaking change for the chart. +Only chart archives from version 3.x can be downloaded from the https://k8up-io.github.io/k8up index. +No 2.x chart releases will be migrated from the APPUiO Helm repo. + +Some RBAC roles and role bindings have change the name. +In most cases this shouldn't be an issue and Helm should be able to cleanup the old resources without impact on the RBAC permissions. + +* New parameter: `podAnnotations`, default `{}`. +* New parameter: `service.annotations`, default `{}`. +* Parameter changed: `image.tag` now defaults to `v2` instead of a pinned version. +* Parameter changed: `image.pullPolicy` now defaults to `Always` instead of `IfNotPresent`. +* Note: Renamed ClusterRole `${release-name}-manager-role` to `${release-name}-manager`. +* Note: Spec of ClusterRole `${release-name}-leader-election-role` moved to `${release-name}-manager`. +* Note: Renamed ClusterRoleBinding `${release-name}-manager-rolebinding` to `${release-name}`. +* Note: ClusterRoleBinding `${release-name}-leader-election-rolebinding` removed (not needed anymore). +* Note: Renamed ClusterRole `${release-name}-k8up-view` to `${release-name}-view`. +* Note: Renamed ClusterRole `${release-name}-k8up-edit` to `${release-name}-edit`. + +## Upgrading from Charts v3 to v4 + +The image tag is now pinned again and not using a floating tag. + +* Parameter changed: `image.tag` now defaults to a pinned version. Each new K8up version now requires also a new chart version. +* Parameter changed: `image.pullPolicy` now defaults to `IfNotPresent` instead of `Always`. +* Parameter changed: `k8up.backupImage.repository` is now unset, which defaults to the same image as defined in `image.{registry/repository}`. +* Parameter changed: `k8up.backupImage.tag` is now unset, which defaults to the same image tag as defined in `image.tag`. + +## Source Code + +* + + +[resource-units]: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes +[prometheus-operator]: https://github.com/coreos/prometheus-operator diff --git a/k8up/templates/NOTES.txt b/k8up/templates/NOTES.txt new file mode 100644 index 00000000..9105b0ad --- /dev/null +++ b/k8up/templates/NOTES.txt @@ -0,0 +1,8 @@ +##################### +! Attention ! +##################### + +This Helm chart does not include CRDs. +Please make sure you have installed or upgraded the necessary CRDs as instructed in the Chart README. + +##################### diff --git a/k8up/templates/_helpers.tpl b/k8up/templates/_helpers.tpl new file mode 100644 index 00000000..58828ffd --- /dev/null +++ b/k8up/templates/_helpers.tpl @@ -0,0 +1,82 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8up.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8up.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8up.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "k8up.labels" -}} +helm.sh/chart: {{ include "k8up.chart" . }} +app.kubernetes.io/name: {{ include "k8up.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "k8up.selectorLabels" -}} +app.kubernetes.io/name: {{ include "k8up.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Static labels +*/}} +{{- define "k8up.staticLabels" -}} +app.kubernetes.io/name: {{ include "k8up.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8up.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8up.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Backup Image +*/}} +{{- define "k8up.backupImage" -}} +{{- with .Values -}} +{{ if .k8up.backupImage.repository }}{{ .k8up.backupImage.repository }}{{ else }}{{ .image.registry}}/{{ .image.repository }}{{ end }}:{{ if .k8up.backupImage.tag }}{{ .k8up.backupImage.tag }}{{ else }}{{ .image.tag }}{{ end }} +{{- end -}} +{{- end -}} diff --git a/k8up/templates/cleanup-hook.yaml b/k8up/templates/cleanup-hook.yaml new file mode 100644 index 00000000..39ebb436 --- /dev/null +++ b/k8up/templates/cleanup-hook.yaml @@ -0,0 +1,100 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cleanup-service-account + annotations: + "helm.sh/hook": post-install,post-upgrade,post-delete + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: +{{ include "k8up.labels" . | indent 4 }} + +--- + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8up-cleanup-roles + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade,post-delete + "helm.sh/hook-weight": "2" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "k8up.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - apiGroups: + - "rbac.authorization.k8s.io" + resources: + - rolebindings + - roles + verbs: + - delete + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cleanup-rolebinding + annotations: + "helm.sh/hook": post-install,post-upgrade,post-delete + "helm.sh/hook-weight": "3" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "k8up.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8up-cleanup-roles +subjects: +- kind: ServiceAccount + name: cleanup-service-account + namespace: {{ .Release.Namespace }} + +--- + +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ .Release.Name }}-cleanup" + labels: + {{- include "k8up.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install,post-upgrade,post-delete + "helm.sh/hook-weight": "4" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + template: + metadata: + name: "{{ .Release.Name }}" + labels: + {{- include "k8up.selectorLabels" . | nindent 8 }} + spec: + restartPolicy: Never + serviceAccountName: cleanup-service-account + containers: + - name: "{{ .Release.Name }}-cleanup" + image: "bitnami/kubectl:latest" + command: + - sh + - -c + args: + - | + #!/bin/bash + + NAMESPACES=$(kubectl get namespace -ojson | jq -r '.items[].metadata.name') + + for ns in $NAMESPACES + do + kubectl -n "$ns" delete rolebinding pod-executor-namespaced --ignore-not-found=true + kubectl -n "$ns" delete role pod-executor --ignore-not-found=true + done diff --git a/k8up/templates/clusterrolebinding.yaml b/k8up/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..99b2af82 --- /dev/null +++ b/k8up/templates/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.serviceAccount.create .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "k8up.fullname" . }} + labels: + {{- include "k8up.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "k8up.fullname" . }}-manager +subjects: +- kind: ServiceAccount + name: {{ include "k8up.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/k8up/templates/deployment.yaml b/k8up/templates/deployment.yaml new file mode 100644 index 00000000..6f0a45cc --- /dev/null +++ b/k8up/templates/deployment.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "k8up.fullname" . }} + labels: +{{ include "k8up.labels" . | indent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "k8up.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "k8up.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: k8up-operator + image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: [ operator ] + env: + - name: BACKUP_IMAGE + value: "{{ include "k8up.backupImage" . }}" + {{- with .Values.k8up.timezone }} + - name: TZ + value: {{ . }} + {{- end }} + - name: BACKUP_ENABLE_LEADER_ELECTION + value: "{{ .Values.k8up.enableLeaderElection }}" + - name: BACKUP_OPERATOR_NAMESPACE + {{- if .Values.k8up.operatorNamespace }} + value: "{{ .Values.k8up.operatorNamespace }}" + {{- else }} + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + {{- with .Values.k8up.globalResources.requests.cpu }} + - name: BACKUP_GLOBALCPU_REQUEST + value: {{ . }} + {{- end }} + {{- with .Values.k8up.globalResources.requests.memory }} + - name: BACKUP_GLOBALMEMORY_REQUEST + value: {{ . }} + {{- end }} + {{- with .Values.k8up.globalResources.limits.cpu }} + - name: BACKUP_GLOBALCPU_LIMIT + value: {{ . }} + {{- end }} + {{- with .Values.k8up.globalResources.limits.memory }} + - name: BACKUP_GLOBALMEMORY_LIMIT + value: {{ . }} + {{- end }} + {{- if .Values.k8up.envVars }} + {{- toYaml .Values.k8up.envVars | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + livenessProbe: + httpGet: + path: /metrics + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + serviceAccountName: {{ template "k8up.serviceAccountName" . }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/k8up/templates/executor-clusterrole.yaml b/k8up/templates/executor-clusterrole.yaml new file mode 100644 index 00000000..b050daf9 --- /dev/null +++ b/k8up/templates/executor-clusterrole.yaml @@ -0,0 +1,35 @@ +{{- if .Values.rbac.create -}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: 'k8up-executor' + labels: + {{- include "k8up.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - apiGroups: + - k8up.io + resources: + - snapshots + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +{{- end -}} diff --git a/k8up/templates/operator-clusterrole.yaml b/k8up/templates/operator-clusterrole.yaml new file mode 100644 index 00000000..f2193543 --- /dev/null +++ b/k8up/templates/operator-clusterrole.yaml @@ -0,0 +1,289 @@ +{{- if .Values.rbac.create -}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: '{{ include "k8up.fullname" . }}-manager' + labels: + {{- include "k8up.labels" . | nindent 4 }} +rules: + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - update + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - watch + - apiGroups: + - k8up.io + resources: + - archives + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - k8up.io + resources: + - archives/finalizers + - archives/status + verbs: + - get + - patch + - update + - apiGroups: + - k8up.io + resources: + - backups + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - k8up.io + resources: + - backups/finalizers + - backups/status + verbs: + - get + - patch + - update + - apiGroups: + - k8up.io + resources: + - checks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - k8up.io + resources: + - checks/finalizers + - checks/status + verbs: + - get + - patch + - update + - apiGroups: + - k8up.io + resources: + - effectiveschedules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - k8up.io + resources: + - effectiveschedules/finalizers + verbs: + - update + - apiGroups: + - k8up.io + resources: + - prebackuppods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - k8up.io + resources: + - prebackuppods/finalizers + - prebackuppods/status + verbs: + - get + - patch + - update + - apiGroups: + - k8up.io + resources: + - prunes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - k8up.io + resources: + - prunes/finalizers + - prunes/status + verbs: + - get + - patch + - update + - apiGroups: + - k8up.io + resources: + - restores + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - k8up.io + resources: + - restores/finalizers + - restores/status + verbs: + - get + - patch + - update + - apiGroups: + - k8up.io + resources: + - schedules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - k8up.io + resources: + - schedules/finalizers + - schedules/status + verbs: + - get + - patch + - update + - apiGroups: + - k8up.io + resources: + - snapshots + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - k8up.io + resources: + - snapshots/finalizers + - snapshots/status + verbs: + - get + - patch + - update + - apiGroups: + - rbac.authorization.k8s.io + resourceNames: + - k8up-executor + resources: + - clusterroles + verbs: + - bind + - apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - create + - delete + - get + - list + - update + - watch +{{- end -}} diff --git a/k8up/templates/prometheus/prometheusrule.yaml b/k8up/templates/prometheus/prometheusrule.yaml new file mode 100644 index 00000000..5e7e3b17 --- /dev/null +++ b/k8up/templates/prometheus/prometheusrule.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.metrics.prometheusRule.enabled (or .Values.metrics.prometheusRule.createDefaultRules .Values.metrics.prometheusRule.additionalRules) -}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "k8up.fullname" . }}-rule + namespace: {{ default .Release.Namespace .Values.metrics.prometheusRule.namespace }} + labels: + {{- include "k8up.labels" . | nindent 4 }} + {{- with .Values.metrics.prometheusRule.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: K8up + rules: + {{- if .Values.metrics.prometheusRule.createDefaultRules }} + - alert: K8upResticErrors + expr: k8up_backup_restic_last_errors > 0 + for: 1m + labels: + severity: critical + annotations: + summary: Amount of errors of last restic backup + description: This alert is fired when error number is > 0 + runbook_url: https://k8up.io/k8up/explanations/runbooks/K8upResticErrors.html + - alert: K8upBackupNotRunning + expr: sum(rate(k8up_jobs_total[25h])) == 0 and on(namespace) k8up_schedules_gauge > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "No K8up jobs were run in {{ "{{ $labels.namespace }}" }} within the last 24 hours. Check the operator, there might be a deadlock" + runbook_url: https://k8up.io/k8up/explanations/runbooks/K8upBackupNotRunning.html + {{- range .Values.metrics.prometheusRule.jobFailedRulesFor }} + - alert: K8up{{- . | title -}}Failed + expr: (sum(kube_job_status_failed) by(job_name, namespace) * on(job_name, namespace) group_right() kube_job_labels{label_k8up_syn_tools_type="{{- . -}}"}) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Job in {{ "{{ $labels.namespace }}" }} of type {{ "{{ $labels.label_k8up_syn_tools_type }}" }} failed" + runbook_url: https://k8up.io/k8up/explanations/runbooks/K8up{{- . | title -}}Failed.html + {{- end }} + {{- end }} + {{- with .Values.metrics.prometheusRule.additionalRules }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/k8up/templates/prometheus/servicemonitor.yaml b/k8up/templates/prometheus/servicemonitor.yaml new file mode 100644 index 00000000..e9d0fe6f --- /dev/null +++ b/k8up/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,24 @@ +{{- if .Values.metrics.serviceMonitor.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "k8up.fullname" . }}-monitor + namespace: {{ default .Release.Namespace .Values.metrics.serviceMonitor.namespace }} + labels: + {{- include "k8up.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: http + interval: {{ .Values.metrics.serviceMonitor.scrapeInterval }} + selector: + matchLabels: + {{- include "k8up.selectorLabels" . | nindent 6 }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + {{- end }} +{{- end }} diff --git a/k8up/templates/service.yaml b/k8up/templates/service.yaml new file mode 100644 index 00000000..4f44cd02 --- /dev/null +++ b/k8up/templates/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "k8up.fullname" . }}-metrics + labels: + {{- include "k8up.labels" . | nindent 4 }} + {{- with .Values.metrics.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.metrics.service.type }} + ports: + - name: http + port: {{ .Values.metrics.service.port }} + targetPort: http + {{- if eq .Values.metrics.service.type "NodePort" }} + nodePort: {{ .Values.metrics.service.nodePort }} + {{- end }} + selector: + {{- include "k8up.selectorLabels" . | nindent 4 }} diff --git a/k8up/templates/serviceaccount.yaml b/k8up/templates/serviceaccount.yaml new file mode 100644 index 00000000..43cc7acf --- /dev/null +++ b/k8up/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "k8up.serviceAccountName" . }} + labels: +{{ include "k8up.labels" . | indent 4 }} +{{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} +{{- end }} + +{{- end -}} diff --git a/k8up/templates/user-clusterrole.yaml b/k8up/templates/user-clusterrole.yaml new file mode 100644 index 00000000..93056563 --- /dev/null +++ b/k8up/templates/user-clusterrole.yaml @@ -0,0 +1,41 @@ +{{- if .Values.rbac.create -}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- include "k8up.staticLabels" . | nindent 4 }} + name: {{ include "k8up.fullname" . }}-edit +rules: +- apiGroups: + - k8up.io + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- include "k8up.staticLabels" . | nindent 4 }} + name: {{ include "k8up.fullname" . }}-view +rules: +- apiGroups: + - k8up.io + resources: + - '*' + verbs: + - get + - list + - watch +{{- end -}} diff --git a/k8up/values.yaml b/k8up/values.yaml new file mode 100644 index 00000000..2018412b --- /dev/null +++ b/k8up/values.yaml @@ -0,0 +1,134 @@ +# -- How many operator pods should run. +# Note: Operator features leader election for K8s 1.16 and later, so that only 1 pod is reconciling/scheduling jobs. +# Follower pods reduce interruption time as they're on hot standby when leader is unresponsive. +replicaCount: 1 +image: + # -- Operator image pull policy + pullPolicy: IfNotPresent + # -- Operator image registry + registry: ghcr.io + # -- Operator image repository + repository: k8up-io/k8up + # -- Operator image tag (version) + tag: v2.7.1 + +imagePullSecrets: [] +serviceAccount: + # -- Specifies whether a service account should be created + create: true + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + # -- Annotations to add to the service account. + annotations: {} + +k8up: + # -- envVars allows the specification of additional environment variables. + # See [values.yaml](values.yaml) how to specify + # See documentation which variables are supported. + envVars: [] + # - name: BACKUP_GLOBALACCESSKEYID + # valueFrom: + # secretKeyRef: + # name: global-s3-credentials + # key: access-key-id + + backupImage: + # -- The backup runner image repository. + # Defaults to `{image.registry}/{image.repository}`. + # Specify an image repository including registry, e.g. `example.com/repo/image` + repository: "" + # -- The backup runner image tag + # Defaults to `{image.tag}` + tag: "" + + # -- Specifies the timezone K8up is using for scheduling. + # Empty value defaults to the timezone in which Kubernetes is deployed. + # Accepts `tz database` compatible entries, e.g. `Europe/Zurich` + timezone: "" + + # -- Specifies whether leader election should be enabled. + enableLeaderElection: true + + # -- Specifies the namespace in which K8up's `EffectiveSchedules` are stored. + # Defaults to release namespace if left empty. + operatorNamespace: "" + + # -- Specify the resource requests and limits that the Pods should + # have when they are scheduled by K8up. You are still able to override those + # via K8up resources, but this gives cluster administrators custom defaults. + # @default -- empty values + globalResources: + requests: + # -- Global CPU resource requests applied to jobs. See [supported units][resource-units]. + cpu: "" + # -- Global Memory resource requests applied to jobs. See [supported units][resource-units]. + memory: "" + limits: + # -- Global CPU resource limit applied to jobs. See [supported units][resource-units]. + cpu: "" + # -- Global Memory resource limit applied to jobs. See [supported units][resource-units]. + memory: "" + +# -- Annotations to add to the Pod spec. +podAnnotations: {} +# -- Security context to add to the Pod spec. +podSecurityContext: {} +# -- Container security context +securityContext: {} + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +metrics: + service: + port: 8080 + type: ClusterIP + # -- Service node port of the metrics endpoint, requires `metrics.service.type=NodePort` + nodePort: 0 + # -- Annotations to add to the service + annotations: {} + + serviceMonitor: + # -- Whether to enable ServiceMonitor manifests for + # [Prometheus Operator][prometheus-operator] + enabled: false + # -- Scrape interval to collect metrics + scrapeInterval: 60s + # -- If the object should be installed in a different namespace than operator + namespace: "" + # -- Add labels to the ServiceMonitor object + additionalLabels: {} + prometheusRule: + # -- Whether to enable PrometheusRule manifest for + # [Prometheus Operator][prometheus-operator] + enabled: false + # -- If the object should be installed in a different namespace than operator + namespace: "" + # -- Add labels to the PrometheusRule object + additionalLabels: {} + # -- Whether the default rules should be installed + createDefaultRules: true + # -- Create default rules for the given job types. + # Valid values are "archive", "backup", "check", "prune", and "restore". + jobFailedRulesFor: ["archive", "backup", "check", "prune", "restore"] + # -- Provide additional alert rules in addition to the defaults + additionalRules: [] + +rbac: + # -- Create cluster roles and rolebinding. + # May need elevated permissions to create cluster roles and -bindings. + create: true + +resources: + limits: + # -- Memory limit of K8up operator. See [supported units][resource-units]. + memory: 256Mi + requests: + # -- CPU request of K8up operator. See [supported units][resource-units]. + cpu: 20m + # -- Memory request of K8up operator. See [supported units][resource-units]. + memory: 128Mi diff --git a/robusta_krr/strategies/__init__.py b/robusta_krr/strategies/__init__.py index 05e029bb..c3341ebb 100644 --- a/robusta_krr/strategies/__init__.py +++ b/robusta_krr/strategies/__init__.py @@ -1 +1,2 @@ from .simple import SimpleStrategy +from .limit import LimitStrategy diff --git a/robusta_krr/strategies/limit.py b/robusta_krr/strategies/limit.py new file mode 100644 index 00000000..354c078a --- /dev/null +++ b/robusta_krr/strategies/limit.py @@ -0,0 +1,138 @@ +import numpy as np +import pydantic as pd + +from robusta_krr.core.abstract.strategies import ( + BaseStrategy, + K8sObjectData, + MetricsPodData, + PodsTimeData, + ResourceRecommendation, + ResourceType, + RunResult, + StrategySettings, +) +from robusta_krr.core.integrations.prometheus.metrics import ( + CPUAmountLoader, + MaxMemoryLoader, + MemoryAmountLoader, + PercentileCPULoader, + PrometheusMetric, +) + + +class LimitStrategySettings(StrategySettings): + cpu_percentile: float = pd.Field(99, gt=0, le=100, description="The percentile to use for the CPU recommendation.") + memory_buffer_percentage: float = pd.Field( + 15, gt=0, description="The percentage of added buffer to the peak memory usage for memory recommendation." + ) + points_required: int = pd.Field( + 100, ge=1, description="The number of data points required to make a recommendation for a resource." + ) + + def calculate_memory_request(self, data: PodsTimeData) -> float: + data_ = [np.max(values[:, 1]) for values in data.values()] + if len(data_) == 0: + return float("NaN") + return np.median(data_) * (1 + self.memory_buffer_percentage / 100) + + def calculate_memory_limit(self, data: PodsTimeData) -> float: + data_ = [np.max(values[:, 1]) for values in data.values()] + if len(data_) == 0: + return float("NaN") + return np.max(data_) * (1 + self.memory_buffer_percentage / 100) + + + + def calculate_cpu_request(self, data: PodsTimeData) -> float: + if len(data) == 0: + return float("NaN") + if len(data) > 1: + data_ = np.concatenate([values[:, 1] for values in data.values()]) + else: + data_ = list(data.values())[0][:, 1] + return np.median(data_) + + def calculate_cpu_limit(self, data: PodsTimeData) -> float: + if len(data) == 0: + return float("NaN") + if len(data) > 1: + data_ = np.concatenate([values[:, 1] for values in data.values()]) + else: + data_ = list(data.values())[0][:, 1] + return np.max(data_) + + + +class LimitStrategy(BaseStrategy[LimitStrategySettings]): + """ + CPU request: {cpu_percentile}% percentile, limit: unset + Memory request: max + {memory_buffer_percentage}%, limit: max + {memory_buffer_percentage}% + History: {history_duration} hours + Step: {timeframe_duration} minutes + + This strategy does not work with objects with HPA defined (Horizontal Pod Autoscaler). + If HPA is defined for CPU or Memory, the strategy will return "?" for that resource. + + Learn more: [underline]https://github.com/robusta-dev/krr#algorithm[/underline] + """ + + display_name = "limit" + rich_console = True + + @property + def metrics(self) -> list[type[PrometheusMetric]]: + return [PercentileCPULoader(self.settings.cpu_percentile), MaxMemoryLoader, CPUAmountLoader, MemoryAmountLoader] + + def __calculate_cpu_proposal( + self, history_data: MetricsPodData, object_data: K8sObjectData + ) -> ResourceRecommendation: + data = history_data["PercentileCPULoader"] + + if len(data) == 0: + return ResourceRecommendation.undefined(info="No data") + + data_count = {pod: values[0, 1] for pod, values in history_data["CPUAmountLoader"].items()} + # Here we filter out pods from calculation that have less than `points_required` data points + filtered_data = { + pod: values for pod, values in data.items() if data_count.get(pod, 0) >= self.settings.points_required + } + + if len(filtered_data) == 0: + return ResourceRecommendation.undefined(info="Not enough data") + + if object_data.hpa is not None and object_data.hpa.target_cpu_utilization_percentage is not None: + return ResourceRecommendation.undefined(info="HPA detected") + + cpu_usage_request = self.settings.calculate_cpu_request(filtered_data) + cpu_usage_limit = self.settings.calculate_cpu_limit(filtered_data) + return ResourceRecommendation(request=cpu_request, limit=limit) + + def __calculate_memory_proposal( + self, history_data: MetricsPodData, object_data: K8sObjectData + ) -> ResourceRecommendation: + data = history_data["MaxMemoryLoader"] + + if len(data) == 0: + return ResourceRecommendation.undefined(info="No data") + + data_count = {pod: values[0, 1] for pod, values in history_data["MemoryAmountLoader"].items()} + # Here we filter out pods from calculation that have less than `points_required` data points + filtered_data = { + pod: value for pod, value in data.items() if data_count.get(pod, 0) >= self.settings.points_required + } + + if len(filtered_data) == 0: + return ResourceRecommendation.undefined(info="Not enough data") + + if object_data.hpa is not None and object_data.hpa.target_memory_utilization_percentage is not None: + return ResourceRecommendation.undefined(info="HPA detected") + + memory_request = self.settings.calculate_memory_request(filtered_data) + memory_limit = self.settings.calculate_memory_limit(filtered_data) + return ResourceRecommendation(request=memory_request, limit=memory_limit) + + def run(self, history_data: MetricsPodData, object_data: K8sObjectData) -> RunResult: + return { + ResourceType.CPU: self.__calculate_cpu_proposal(history_data, object_data), + ResourceType.Memory: self.__calculate_memory_proposal(history_data, object_data), + }