Copy all files from kubernetes repo into this one

This commit is a like-for-like copy of the [kubernetes repo](https://github.com/python-discord/kubernetes) check that repo for comit history prioir to this commit.

Co-authored-by: Amrou Bellalouna <[email protected]>
Co-authored-by: Bradley Reynolds <[email protected]>
Co-authored-by: Chris <[email protected]>
Co-authored-by: Chris Lovering <[email protected]>
Co-authored-by: ChrisJL <[email protected]>
Co-authored-by: Den4200 <[email protected]>
Co-authored-by: GDWR <[email protected]>
Co-authored-by: Hassan Abouelela <[email protected]>
Co-authored-by: Hassan Abouelela <[email protected]>
Co-authored-by: jchristgit <[email protected]>
Co-authored-by: Joe Banks <[email protected]>
Co-authored-by: Joe Banks <[email protected]>
Co-authored-by: Joe Banks <[email protected]>
Co-authored-by: Johannes Christ <[email protected]>
Co-authored-by: Kieran Siek <[email protected]>
Co-authored-by: kosayoda <[email protected]>
Co-authored-by: ks129 <[email protected]>
Co-authored-by: Leon Sand├©y <[email protected]>
Co-authored-by: Leon Sand├©y <[email protected]>
Co-authored-by: MarkKoz <[email protected]>
Co-authored-by: Matteo Bertucci <[email protected]>
Co-authored-by: Sebastiaan Zeeff <[email protected]>
Co-authored-by: Sebastiaan Zeeff <[email protected]>
Co-authored-by: vcokltfre <[email protected]>

22 files changed, 512 insertions, 0 deletions

diff --git a/kubernetes/namespaces/monitoring/alerts/Makefile b/kubernetes/namespaces/monitoring/alerts/Makefile
new file mode 100644
index 0000000..c599ee6
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/Makefile
@@ -0,0 +1,11 @@
+.PHONY: alerts alertmanager
+
+all: alerts alertmanager
+
+# Upload the alerting rules to the Kubernetes cluster
+alerts:
+	kubectl create configmap -n monitoring prometheus-alert-rules --from-file=alerts.d/ -o yaml --dry-run=client | kubectl apply -f -
+
+# Upload the alertmanager configuration to the Kubernetes cluster
+alertmanager:
+	kubectl create configmap -n monitoring alertmanager-config --from-file=alertmanager.yaml=alertmanager.yaml -o yaml --dry-run=client | kubectl apply -f -
diff --git a/kubernetes/namespaces/monitoring/alerts/README.md b/kubernetes/namespaces/monitoring/alerts/README.md
new file mode 100644
index 0000000..75f70ac
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/README.md
@@ -0,0 +1,5 @@
+# Alerts
+
+This directory contains alerting rules and routing configuration for production.
+
+To build and upload this configuration, see the annotated `Makefile` in this directory.
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager.yaml
new file mode 100644
index 0000000..bef166a
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alertmanager.yaml
@@ -0,0 +1,24 @@
+route:
+  group_by: ['alertname', 'cluster', 'service']
+
+  group_wait: 15s
+
+  group_interval: 1m
+
+  receiver: devops-team
+
+receivers:
+- name: devops-team
+  slack_configs:
+  - api_url_file: "/opt/pydis/alertmanager/webhooks/DEVOPS_HOOK"
+    send_resolved: true
+    title: '{{ if eq .Status "firing" }}[FIRING]{{ else }}[RESOLVED]{{ end }}'
+    text: |
+      {{ if eq .Status "firing" }}{{ range .Alerts }}
+      **{{ .Annotations.summary }}:**
+      {{ .Annotations.description }} [(Link)]({{.GeneratorURL}})
+
+      {{ end }}{{ else }}Alert has resolved.{{ end }}
+    fields:
+    - title: Alert
+      value: "{{ .GroupLabels.alertname }}"
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager/deployment.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager/deployment.yaml
new file mode 100644
index 0000000..4f1c322
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alertmanager/deployment.yaml
@@ -0,0 +1,92 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alertmanager
+  namespace: monitoring
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: alertmanager
+  template:
+    metadata:
+      labels:
+        app: alertmanager
+    spec:
+      serviceAccountName: prometheus
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - alertmanager
+              namespaces:
+              - monitoring
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+      initContainers:
+      - image: debian:bullseye-slim
+        imagePullPolicy: Always
+        name: alertmanager-peering-setup
+        command: [
+          '/opt/pydis/alertmanager/init.d/find-pods.sh'
+        ]
+        volumeMounts:
+        - name: alertmanager-init
+          mountPath: /opt/pydis/alertmanager/init.d
+        - name: alertmanager-tmp
+          mountPath: /tmp
+        securityContext:
+          runAsUser: 0
+      containers:
+      - image: prom/alertmanager:latest
+        imagePullPolicy: Always
+        name: alertmanager
+        command:
+          - /bin/sh
+          - -c
+          - |
+            exec /bin/alertmanager \
+              --config.file=/opt/pydis/alertmanager/config.d/alertmanager.yaml \
+              --web.external-url=https://alertmanager.pythondiscord.com \
+              --storage.path=/data/alertmanager \
+              $(cat /tmp/peers)
+        ports:
+        - name: am
+          containerPort: 9093
+        - name: am-peering
+          containerPort: 9094
+        volumeMounts:
+        - name: alertmanager-config
+          mountPath: /opt/pydis/alertmanager/config.d
+        - name: alertmanager-webhooks
+          mountPath: /opt/pydis/alertmanager/webhooks
+        - name: alertmanager-tmp-data
+          mountPath: /data
+        - name: alertmanager-tmp
+          mountPath: /tmp
+        securityContext:
+          readOnlyRootFilesystem: true
+      restartPolicy: Always
+      volumes:
+        - name: alertmanager-config
+          configMap:
+            name: alertmanager-config
+        - name: alertmanager-webhooks
+          secret:
+            secretName: alert-manager-hook
+        - name: alertmanager-tmp-data
+          emptyDir: {}
+        - name: alertmanager-tmp
+          emptyDir: {}
+        - name: alertmanager-init
+          configMap:
+            name: alertmanager-init
+            defaultMode: 0777
+      securityContext:
+        fsGroup: 1000
+        runAsUser: 1000
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager/ingress.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager/ingress.yaml
new file mode 100644
index 0000000..fc99e52
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alertmanager/ingress.yaml
@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/auth-tls-verify-client: "on"
+    nginx.ingress.kubernetes.io/auth-tls-secret: "kube-system/mtls-client-crt-bundle"
+    nginx.ingress.kubernetes.io/auth-tls-error-page: "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
+  name: alertmanager
+  namespace: monitoring
+spec:
+  tls:
+  - hosts:
+      - "*.pythondiscord.com"
+  rules:
+  - host: alertmanager.pythondiscord.com
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: alertmanager
+            port:
+              number: 9093
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager/initscript.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager/initscript.yaml
new file mode 100644
index 0000000..f1f36e2
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alertmanager/initscript.yaml
@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alertmanager-init
+  namespace: monitoring
+data:
+  find-pods.sh: |
+    #!/bin/sh
+
+    # Install curl and jq for JSON parsing
+    apt update && apt install -y curl jq
+
+    # Find the template hash
+    echo Finding template hash...
+    TEMPLATE_HASH=$(echo $HOSTNAME | cut -d- -f2)
+
+    # Query kubernetes API for all matching pods
+    echo Querying Kubernetes API for pods...
+    PODS=$(curl \
+      -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
+      https://kubernetes.default/api/v1/namespaces/monitoring/pods\?labelSelector=pod-template-hash=$TEMPLATE_HASH\&pretty=false -sk -o /tmp/peers.json)
+
+    echo Finding Alertmanager IPs...
+    AM_IPS=$(jq '.items[].status.podIP' /tmp/peers.json -r)
+
+    echo Generating CLI flags for Alertmanager...
+    PEER_ARGS=$(echo $AM_IPS | sed 's/ /\n/g' | awk '{ print "--cluster.peer="$1":9094" }')
+
+    echo Writing CLI flags to /tmp/peers...
+    echo $PEER_ARGS > /tmp/peers
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager/sd-service.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager/sd-service.yaml
new file mode 100644
index 0000000..8ec901a
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alertmanager/sd-service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alertmanager-sd
+  namespace: monitoring
+spec:
+  selector:
+    app: alertmanager
+  clusterIP: None
+  ports:
+  - port: 9093
+    targetPort: 9093
+    name: am
+  - port: 9094
+    targetPort: 9094
+    name: am-peering
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager/secrets.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager/secrets.yaml
new file mode 100644
index 0000000..7cc1d95
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alertmanager/secrets.yaml
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager/service-account.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager/service-account.yaml
new file mode 100644
index 0000000..3f26311
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alertmanager/service-account.yaml
@@ -0,0 +1,28 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: alertmanager
+rules:
+- apiGroups: [""]
+  resources: ["pods", "endpoints"]
+  verbs: ["get", "list"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: alertmanager
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: alertmanager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: alertmanager
+subjects:
+  - kind: ServiceAccount
+    name: alertmanager
+    namespace: monitoring
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager/service.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager/service.yaml
new file mode 100644
index 0000000..145b1e2
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alertmanager/service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alertmanager
+  namespace: monitoring
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "9093"
+spec:
+  selector:
+    app: alertmanager
+  ports:
+  - port: 9093
+    targetPort: 9093
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/alertmanager.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/alertmanager.yaml
new file mode 100644
index 0000000..b3fcad9
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/alertmanager.yaml
@@ -0,0 +1,21 @@
+groups:
+- name: alertmanager
+  rules:
+
+  - alert: AlertManagerClusterFailedPeers
+    expr: alertmanager_cluster_failed_peers > 0
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: "An Alertmanager node is reporting failed peers"
+      description: "AM {{ $labels.instance }} is reporting that {{ $value }} of it's peers is invalid."
+
+  - alert: AlertManagerHealthScore
+    expr: alertmanager_cluster_health_score > 0
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: "An AlertManagerNode is reporting an unhealthy cluster"
+      description: "AM {{ $labels.instance }} is reporting that the cluster has a health score of {{ $value }} (where 0 is healthy.)"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/certificates.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/certificates.yaml
new file mode 100644
index 0000000..10eb3dd
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/certificates.yaml
@@ -0,0 +1,13 @@
+groups:
+- name: certificates
+  interval: 1d
+  rules:
+
+  - alert: CertificateExpiringSoon
+    expr: (certmanager_certificate_expiration_timestamp_seconds - time()) / 60 / 60 / 24 < 7
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Certificate is expiring in < 7 days"
+      description: "The certificate named {{ $labels.name }} is due for expiry in {{ $value | humanize }} days."
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/coredns.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/coredns.yaml
new file mode 100644
index 0000000..9daa660
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/coredns.yaml
@@ -0,0 +1,20 @@
+groups:
+- name: coredns
+  rules:
+
+  - alert: CoreDNSPanics
+    expr: increase(coredns_panics_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: page
+    annotations:
+      summary: "CoreDNS is experiencing panic"
+      description: "Number of CoreDNS panics encountered: {{ $value }}"
+
+  - alert: CoreDNSCacheMisses
+    expr: rate(coredns_cache_misses_total{}[10m]) / rate(coredns_cache_misses_total{}[10m] offset 10m) > 5.00
+    labels:
+      severity: page
+    annotations:
+      summary: "High CoreDNS cache misses in last 10 minutes"
+      description: "This can sometimes be an indication of networking troubles, currently {{ $value | humanizePercentage }} over last 10 minutes."
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/cpu.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/cpu.yaml
new file mode 100644
index 0000000..5e8868e
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/cpu.yaml
@@ -0,0 +1,21 @@
+groups:
+- name: cpu
+  rules:
+
+  - alert: HighCPUThrottling
+    expr: rate(container_cpu_cfs_throttled_seconds_total{pod=~".+", container_name!="POD", image!=""}[5m]) > 1
+    for: 5m
+    labels:
+      severity: page
+    annotations:
+      summary: "Container {{ $labels.container_name }} in {{ $labels.pod }} high throttling "
+      description: "{{ $labels.container_name }} inside {{ $labels.pod }} is at {{ $value }}"
+
+  - alert: HighNodeCPU
+    expr: 100 - (avg by (kubernetes_node) (irate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[5m])) * 100) > 80
+    for: 5m
+    labels:
+      severity: page
+    annotations:
+      summary: "Node {{ $labels.kubernetes_node }} has CPU over 80% for last 5 minute"
+      description: "CPU on {{ $labels.kubernetes_node }} is averaging {{ $value }}"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/jobs.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/jobs.yaml
new file mode 100644
index 0000000..723d267
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/jobs.yaml
@@ -0,0 +1,20 @@
+groups:
+- name: jobs
+  rules:
+  - alert: KubernetesCronjobSuspended
+    expr: kube_cronjob_spec_suspend != 0
+    for: 0m
+    labels:
+      severity: page
+    annotations:
+      summary: "Kubernetes CronJob suspended: {{ $labels.cronjob }}"
+      description: "CronJob {{ $labels.kubernetes_namespace }}/{{ $labels.cronjob }} is suspended"
+
+  - alert: KubernetesJobFailed
+    expr: kube_job_status_failed > 0
+    for: 0m
+    labels:
+      severity: page
+    annotations:
+      summary: "Kubernetes Job failed: {{ $labels.job_name }}"
+      description: "Job {{$labels.kubernetes_namespacenamespace}}/{{$labels.job_name}} failed to complete"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/memory.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/memory.yaml
new file mode 100644
index 0000000..d53da5e
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/memory.yaml
@@ -0,0 +1,12 @@
+groups:
+- name: memory
+  rules:
+
+  - alert: NodeHighMemoryUsage
+    expr: node_memory_Active_bytes / node_memory_MemTotal_bytes > 0.8
+    for: 30s
+    labels:
+      severity: page
+    annotations:
+      summary: "Node {{ $labels.kubernetes_node }} has RAM usage >80% for 5 minutes"
+      description: 'RAM usage is currently {{ $value | humanizePercentage }} on {{ $labels.kubernetes_node }}'
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/nginx.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/nginx.yaml
new file mode 100644
index 0000000..441f7df
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/nginx.yaml
@@ -0,0 +1,30 @@
+groups:
+- name: nginx
+  rules:
+
+  - alert: NGINX4XXRequests
+    expr: sum by(service) (rate(nginx_ingress_controller_requests{status=~"^4..", status!="404", service!="pixels"}[1m])) / sum by(service) (rate(nginx_ingress_controller_requests[1m])) > 0.5
+    for: 1m
+    labels:
+      severity: page
+    annotations:
+      summary: "High rate of 4XX requests for inbound requests"
+      description: "Rate of 4XX errors is {{ $value | humanizePercentage }} on service `{{ $labels.service }}`"
+
+  - alert: NGINX5XXRequests
+    expr: sum(rate(nginx_ingress_controller_requests{status=~"^5.."}[1m])) by (service) / sum(rate(nginx_ingress_controller_requests{}[1m])) by (service) > 0.5
+    for: 1m
+    labels:
+      severity: page
+    annotations:
+      summary: "High rate of 5XX requests for inbound requests"
+      description: "Rate of 5XX errors is {{ $value | humanizePercentage }} on service `{{ $labels.service }}`"
+
+  - alert: NGINXP99Timing
+    expr: histogram_quantile(0.99, sum by(host, service, le) (rate(nginx_ingress_controller_request_duration_seconds_bucket{service!~"(grafana|metabase|prestashop-svc)", host!="pydis-api.default.svc.cluster.local"}[5m]))) > 3 and on(service) increase(nginx_ingress_controller_requests[5m]) > 10
+    for: 5m
+    labels:
+      severity: page
+    annotations:
+      summary: "Request timing P99 has been over 3 seconds for 5 minutes"
+      description: "Requests to service {{ $labels.host }} (to service {{ $labels.service }}) have taken over 3 seconds (P99) to complete."
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/nodes.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/nodes.yaml
new file mode 100644
index 0000000..6bfa6d1
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/nodes.yaml
@@ -0,0 +1,49 @@
+groups:
+- name: nodes
+  rules:
+
+  - alert: KubernetesNodeDiskPressure
+    expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
+    for: 1m
+    labels:
+      severity: page
+    annotations:
+      summary: Node {{ $labels.kubernetes_node }} is experiencing disk pressure
+      description: "{{ $labels.kubernetes_node }} does not have adequate space to work with."
+
+  - alert: KubernetesNodeMemoryPressure
+    expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
+    for: 15s
+    labels:
+      severity: page
+    annotations:
+      summary: Node {{ $labels.kubernetes_node }} is experiencing memory pressure
+      description: "{{ $labels.kubernetes_node }} does not have adequate RAM to work with."
+
+  - alert: KubernetesNodeNetworkUnavailable
+    expr: kube_node_status_condition{condition="NetworkUnavailable",status="true"} == 1
+    for: 15s
+    labels:
+      severity: page
+    annotations:
+      summary: Node {{ $labels.kubernetes_node }} is experiencing network problems
+      description: "{{ $labels.kubernetes_node }} is experiencing trouble with inbound and outbound connections"
+
+
+  - alert: KubernetesNodePIDPressure
+    expr: kube_node_status_condition{condition="PIDPressure",status="true"} == 1
+    for: 15s
+    labels:
+      severity: page
+    annotations:
+      summary: Node {{ $labels.kubernetes_node }} is experiencing PID exhaustion
+      description: "{{ $labels.kubernetes_node }} does not have enough PIDs to work with."
+
+  - alert: KubernetesNodeReady
+    expr: kube_node_status_condition{condition="Ready",status="true"} == 0
+    for: 5m
+    labels:
+      severity: page
+    annotations:
+      summary: Kubernetes node ({{ $labels.kubernetes_node }} ) is marked as unready
+      description: "Node {{ $labels.kubernetes_node }} has been unready for a long time"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/pods.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/pods.yaml
new file mode 100644
index 0000000..9efdffa
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/pods.yaml
@@ -0,0 +1,20 @@
+groups:
+- name: pods
+  rules:
+  - alert: KubernetesPodNotHealthy
+    expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[3m:1m]) > 0
+    for: 3m
+    labels:
+      severity: page
+    annotations:
+      summary: "Kubernetes Pod not healthy: {{ $labels.namespace }}/{{ $labels.pod }}"
+      description: "Pod has been in a non-ready state for longer than 3 minutes."
+
+  - alert: KubernetesPodCrashLooping
+    expr: increase(kube_pod_container_status_restarts_total[5m]) > 3
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Kubernetes pod crash looping: {{ $labels.kubernetes_namespace }}/{{ $labels.pod }}"
+      description: "Pod {{ $labels.pod }} is crash looping"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/postgres.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/postgres.yaml
new file mode 100644
index 0000000..399a84b
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/postgres.yaml
@@ -0,0 +1,29 @@
+groups:
+- name: postgres
+  rules:
+  - alert: PostgresUp
+    expr: pg_up == 0
+    for: 0m
+    labels:
+      severity: page
+    annotations:
+      summary: "PostgreSQL is offline"
+      description: "Postgres Exporter cannot connect to PostgreSQL."
+
+  - alert: PostgresTooManyConnections
+    expr: (sum(pg_stat_activity_count) by (instance)) / on (instance) pg_settings_max_connections * 100 > 80
+    for: 1m
+    labels:
+      severity: page
+    annotations:
+      summary: PostgreSQL connections near max_connections setting
+      description: "PostgreSQL instance is near the maximum connection limit, currently {{ $value }} connections"
+
+  - alert: PostgresDeadlockedTable
+    expr: increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 3
+    for: 1m
+    labels:
+      severity: page
+    annotations:
+      summary: Too many deadlocked tables
+      description: "PostgreSQL has dead-locks, value: {{ $value }}"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/prometheus.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/prometheus.yaml
new file mode 100644
index 0000000..25e555d
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/prometheus.yaml
@@ -0,0 +1,13 @@
+groups:
+- name: prometheus
+  rules:
+
+  # Alert for any instance that is unreachable for >5 minutes.
+  - alert: InstanceDown
+    expr: up == 0
+    for: 5m
+    labels:
+      severity: page
+    annotations:
+      summary: "Instance {{ $labels.instance }} down"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/redis.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/redis.yaml
new file mode 100644
index 0000000..6b946f6
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/redis.yaml
@@ -0,0 +1,20 @@
+groups:
+- name: redis
+  rules:
+  - alert: RedisDown
+    expr: redis_up == 0
+    for: 1m
+    labels:
+      severity: page
+    annotations:
+      summary: "Redis is offline"
+      description: "Redis Exporter cannot connect to Redis."
+
+  - alert: RedisOutOfMemory
+    expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.9
+    for: 0m
+    labels:
+      severity: page
+    annotations:
+      summary: "Redis is approaching it's memory limit"
+      description: "Redis is currently using {{ $value | humanizePercentage }} of configured memory."