spec:
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
      maxSurge: 2

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: bookings-api
  namespace: rutas-norte-pro
spec:
  maxUnavailable: 25%
  selector:
    matchLabels:
      app: bookings-api

# This IGNORES the PDB
kubectl delete pod bookings-api-7c9d4f8b6d-2mk8p -n rutas-norte-pro

# This RESPECTS the PDB
kubectl drain <node> --pod-selector=app=bookings-api

spec:
  minAvailable: 3
  selector:
    matchLabels:
      app: bookings-api

spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      app: bookings-api

During the bank holiday: 30 replicas. Somebody sets minAvailable: 25.
  Evictions allowed: 5. Reasonable.

After the bank holiday: the HPA drops to 4 replicas. The PDB still says minAvailable: 25.
  Available pods: 4. Minimum required: 25.
  We are ALREADY below the minimum.
  Evictions allowed: ZERO. Forever.

Any node hosting a bookings-api replica is PINNED.
The Cluster Autoscaler cannot remove it. Maintenance is blocked.

POST /api/v1/namespaces/rutas-norte-pro/pods/bookings-api-7c9d4f8b6d-2mk8p/eviction

{
  "apiVersion": "policy/v1",
  "kind": "Eviction",
  "metadata": {
    "name": "bookings-api-7c9d4f8b6d-2mk8p",
    "namespace": "rutas-norte-pro"
  }
}

{
  "kind": "Status",
  "status": "Failure",
  "message": "Cannot evict pod as it would violate the pod's disruption budget.",
  "reason": "TooManyRequests",
  "details": {
    "causes": [
      {
        "reason": "DisruptionBudget",
        "message": "The disruption budget bookings-api needs 3 healthy pods and has 3 currently"
      }
    ]
  }
}

kubectl get pdb -n rutas-norte-pro

NAME                   MIN AVAILABLE   MAX UNAVAILABLE   ALLOWED DISRUPTIONS   AGE
bookings-api           N/A             25%               1                     12d
web-store              N/A             1                 1                     12d
bookings-postgres      1               N/A               0                     12d
notifications-worker   N/A             50%               3                     12d

kubectl describe pdb bookings-api -n rutas-norte-pro

Name:             bookings-api
Namespace:        rutas-norte-pro
Max unavailable:  25%
Selector:         app=bookings-api
Status:
    Allowed disruptions:  2
    Current:              8
    Desired:              6
    Total:                8

minikube start -p rutas-norte --nodes=3 --cpus=2 --memory=4096

kubectl create namespace rutas-norte-dev

# A Deployment with 3 replicas
kubectl -n rutas-norte-dev create deployment demo-pdb \
  --image=registry.k8s.io/pause:3.9 --replicas=3

kubectl -n rutas-norte-dev set resources deployment/demo-pdb \
  --requests=cpu=100m,memory=64Mi

# /tmp/pdb-demo.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: demo-pdb
  namespace: rutas-norte-dev
spec:
  # With 3 replicas, requiring 3 available means ZERO evictions allowed.
  minAvailable: 3
  selector:
    matchLabels:
      app: demo-pdb

kubectl apply -f /tmp/pdb-demo.yaml
kubectl get pdb -n rutas-norte-dev

NAME       MIN AVAILABLE   MAX UNAVAILABLE   ALLOWED DISRUPTIONS   AGE
demo-pdb   3               N/A               0                     8s

kubectl get pods -n rutas-norte-dev -o wide

NAME                        READY   STATUS    NODE
demo-pdb-6c9f8d7b5-2mkjp    1/1     Running   rutas-norte
demo-pdb-6c9f8d7b5-7wqzx    1/1     Running   rutas-norte-m02
demo-pdb-6c9f8d7b5-9nvtr    1/1     Running   rutas-norte-m03

kubectl drain rutas-norte-m02 --ignore-daemonsets --delete-emptydir-data

node/rutas-norte-m02 cordoned
evicting pod rutas-norte-dev/demo-pdb-6c9f8d7b5-7wqzx
error when evicting pods/"demo-pdb-6c9f8d7b5-7wqzx" -n "rutas-norte-dev"
(will retry after 5s): Cannot evict pod as it would violate the pod's disruption budget.
evicting pod rutas-norte-dev/demo-pdb-6c9f8d7b5-7wqzx
error when evicting pods/"demo-pdb-6c9f8d7b5-7wqzx" -n "rutas-norte-dev"
(will retry after 5s): Cannot evict pod as it would violate the pod's disruption budget.
evicting pod rutas-norte-dev/demo-pdb-6c9f8d7b5-7wqzx
error when evicting pods/"demo-pdb-6c9f8d7b5-7wqzx" -n "rutas-norte-dev"
(will retry after 5s): Cannot evict pod as it would violate the pod's disruption budget.
...

# Option A: more replicas. With 4 replicas and minAvailable: 3, 1 eviction is allowed.
kubectl -n rutas-norte-dev scale deployment demo-pdb --replicas=4

# Option B: fix the PDB
kubectl -n rutas-norte-dev patch pdb demo-pdb --type merge \
  -p '{"spec":{"minAvailable":null,"maxUnavailable":"25%"}}'

# Option C (LAST RESORT, dangerous): force the drain
kubectl drain rutas-norte-m02 --ignore-daemonsets --delete-emptydir-data \
  --disable-eviction

kubectl uncordon rutas-norte-m02
kubectl delete -f /tmp/pdb-demo.yaml
kubectl delete deployment demo-pdb -n rutas-norte-dev

spec:
  replicas: 3        # in the Deployment
---
spec:
  minAvailable: 3    # in the PDB

spec:
  replicas: 1
---
spec:
  minAvailable: 1        # Equivalent to maxUnavailable: 0

# Every PDB in the cluster with its allowed disruptions
kubectl get pdb --all-namespaces

NAMESPACE          NAME                   MIN AVAILABLE   MAX UNAVAILABLE   ALLOWED DISRUPTIONS
rutas-norte-pro    bookings-api           N/A             25%               2
rutas-norte-pro    web-store              N/A             1                 1
rutas-norte-pro    bookings-postgres      1               N/A               0
rutas-norte-pro    redis-cache            1               N/A               0
rutas-norte-pro    notifications-worker   N/A             50%               4
monitoring         prometheus             1               N/A               0

# PDBs that have gone more than an hour without allowing any eviction
kube_poddisruptionbudget_status_pod_disruptions_allowed == 0

Total replicas: 6
Ready replicas: 2
Not-Ready replicas: 4

PDB: maxUnavailable: 1 -> at least 5 available.
Currently available: 2. We are already BELOW the minimum.
ALLOWED DISRUPTIONS: 0.

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: bookings-api
  namespace: rutas-norte-pro
spec:
  maxUnavailable: 25%
  selector:
    matchLabels:
      app: bookings-api
  # ALWAYS allow evicting pods that are not Ready, without consuming
  # disruption budget.
  unhealthyPodEvictionPolicy: AlwaysAllow

# k8s/environments/pro/pdb-bookings-api.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: bookings-api
  namespace: rutas-norte-pro
  labels:
    app: bookings-api
    app.kubernetes.io/part-of: rutas-norte
spec:
  # A PERCENTAGE, not an absolute number. bookings-api has KEDA/HPA (09-01, 09-04)
  # and swings between 4 and 30 replicas across the day. An absolute
  # minAvailable would be obsolete the moment the HPA moved, and in the
  # night-time trough it would become an IMPOSSIBLE PDB that would block
  # maintenance and node scale-down.
  #
  # With 25%:
  #    4 replicas -> allows evicting 1  (floor(4 x 0.25) = 1)
  #   12 replicas -> allows evicting 3
  #   30 replicas -> allows evicting 7
  # The PROPORTION of service protected is constant: always 75%.
  maxUnavailable: 25%

  selector:
    matchLabels:
      app: bookings-api

  # Pods that fail readiness are not serving traffic: evicting them
  # does not reduce availability and it avoids the circular deadlock of a
  # faulty deployment that prevents its own repair.
  unhealthyPodEvictionPolicy: AlwaysAllow

# k8s/environments/pro/pdb-web-store.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: web-store
  namespace: rutas-norte-pro
  labels:
    app: web-store
    app.kubernetes.io/part-of: rutas-norte
spec:
  # 34%, more permissive than bookings-api. Reason: web-store is nginx serving
  # static assets. It starts in ~1 second and has no state and no cache to warm,
  # so an evicted replica comes back almost instantly on another node.
  # Allowing more simultaneous evictions speeds up maintenance with no real risk.
  #
  # With 3 replicas -> allows evicting 1  (floor(3 x 0.34) = 1)
  # With 15 replicas -> allows evicting 5
  maxUnavailable: 34%

  selector:
    matchLabels:
      app: web-store

  unhealthyPodEvictionPolicy: AlwaysAllow

# k8s/environments/pro/pdb-notifications-worker.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: notifications-worker
  namespace: rutas-norte-pro
  labels:
    app: notifications-worker
    app.kubernetes.io/part-of: rutas-norte
spec:
  # 50%, the most permissive on the platform. Justification:
  #
  # 1. NOBODY IS WAITING. A confirmation email sent 30 seconds later
  #    has no consequence at all for the user.
  # 2. THE QUEUE IS THE BUFFER. If half the workers are evicted, the
  #    work is not lost: it piles up in the queue and is processed later.
  #    The in-flight message is redelivered automatically.
  # 3. KEDA REACTS. If the queue grows because there are fewer workers, KEDA (09-04)
  #    will scale up to compensate.
  #
  # With scale-to-ZERO (minReplicaCount: 0) there is a nuance: when there are 0
  # replicas, the PDB covers no pod and ALLOWED DISRUPTIONS will be 0, but
  # that is irrelevant because there is nothing to evict.
  maxUnavailable: 50%

  selector:
    matchLabels:
      app: notifications-worker

  unhealthyPodEvictionPolicy: AlwaysAllow

# k8s/environments/pro/pdb-redis-cache.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: redis-cache
  namespace: rutas-norte-pro
  labels:
    app: redis-cache
    app.kubernetes.io/part-of: rutas-norte
  annotations:
    # Annotation to exclude this PDB from the "impossible PDB" alert:
    # its ALLOWED DISRUPTIONS is 0 DELIBERATELY.
    rutasnorte.example/pdb-intentional-block: >
      redis-cache is a single-replica StatefulSet. Evicting it empties the cache
      and dumps the whole availability-query load onto
      bookings-postgres. It requires manual intervention and a maintenance window.
spec:
  # minAvailable: 1 with ONE replica = ZERO evictions allowed.
  # It is DELIBERATE. Accepted consequences:
  #   - The Cluster Autoscaler will not remove the node hosting redis-cache.
  #   - kubectl drain on that node will sit there waiting.
  #   - Maintenance on that node requires an explicit human decision.
  #
  # It is an ACCEPTED trade-off: we prefer manual maintenance to
  # an automatic mechanism emptying the cache at the worst possible moment.
  minAvailable: 1

  selector:
    matchLabels:
      app: redis-cache

  # IfHealthyBudget (the default). We do not use AlwaysAllow: if redis-cache is
  # not Ready, it may be loading data or recovering. Evicting it
  # would make things worse.
  unhealthyPodEvictionPolicy: IfHealthyBudget

# k8s/environments/pro/pdb-bookings-postgres.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: bookings-postgres
  namespace: rutas-norte-pro
  labels:
    app: bookings-postgres
    app.kubernetes.io/part-of: rutas-norte
  annotations:
    rutasnorte.example/pdb-intentional-block: >
      bookings-postgres is the platform's primary database with a
      single replica. NO automatic mechanism may move it. Maintenance
      on the node hosting it requires a planned window, with a controlled
      shutdown of the application and a backup beforehand. See 05-06.
      The definitive solution is a PostgreSQL operator: see 06-07.
spec:
  # ZERO evictions. It is the strongest possible protection, and it is intentional.
  #
  # An automatic eviction of bookings-postgres would mean:
  #   - Cutting ALL connections from bookings-api.
  #   - Losing the PostgreSQL page cache (minutes of bad latency).
  #   - Risk of inconsistency if there are transactions in flight.
  #   - The entire platform down during start-up.
  #
  # It holds customers' personal data: any risk is unacceptable.
  minAvailable: 1

  selector:
    matchLabels:
      app: bookings-postgres

  unhealthyPodEvictionPolicy: IfHealthyBudget

node-1 (zone A):  bookings-api x4, web-store x2
node-2 (zone A):  bookings-postgres
node-3 (zone B):  redis-cache, worker x2
node-4 (zone B):  (almost empty)

node-1 goes down  ->  ZERO bookings-api replicas. Platform down.
Zone A goes down  ->  ZERO bookings-api replicas AND the database.

spec:
  template:
    spec:
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: bookings-api
          minDomains: 3
          nodeAffinityPolicy: Honor
          nodeTaintsPolicy: Honor
          matchLabelKeys:
            - pod-template-hash

skew = (pods in the domain with the MOST) - (pods in the domain with the FEWEST)

Without minDomains, with 3 replicas and only 1 zone with available nodes:
  Zone A: 3 pods. Zone B: no nodes. Zone C: no nodes.
  EXISTING domains: 1
  Skew: 3 - 3 = 0  ->  the constraint IS SATISFIED.
  Result: all 3 replicas in the same zone. NOT what you wanted.

With minDomains: 3:
  The scheduler considers there to be 3 domains, two of them with 0 pods.
  Skew: 3 - 0 = 3 > maxSkew 1  ->  NOT satisfied.
  With DoNotSchedule: the pods stay Pending until there are nodes in other zones.

Update of bookings-api from v1.14 to v1.15. 6 replicas.

Intermediate state:
  zone A: 2 pods v1.14 + 1 pod v1.15 = 3 pods
  zone B: 2 pods v1.14 = 2 pods
  zone C: 1 pod v1.14 = 1 pod

Skew computed over ALL of them: 3 - 1 = 2 > maxSkew 1
-> The scheduler CANNOT place more v1.15 pods in zone A.
-> The update stalls or is spread badly.

With matchLabelKeys: ["pod-template-hash"]:
  To place a v1.15 pod, only v1.15 pods count:
    zone A: 1, zone B: 0, zone C: 0
  Skew: 1 - 0 = 1 <= maxSkew 1  ->  it can be placed in B or C.

The new version spreads itself out correctly.

Placing replica 1:
  A: 0, B: 0, C: 0. Any zone works. -> A
  State: A:1, B:0, C:0

Placing replica 2:
  If it goes to A: A:2, B:0, C:0. Skew = 2-0 = 2 > 1. NOT ALLOWED.
  If it goes to B: A:1, B:1, C:0. Skew = 1-0 = 1 <= 1. ALLOWED.
  -> B (or C)
  State: A:1, B:1, C:0

Placing replica 3:
  Only C keeps the skew at 1 or below.  -> C
  State: A:1, B:1, C:1  (skew 0)

Replicas 4, 5, 6: one per zone.
  State: A:2, B:2, C:2  (skew 0)

Placing replica 7:
  Any zone: the skew would become 1. Allowed in all three.
  -> A (arbitrary)
  FINAL state: A:3, B:2, C:2  (skew 1)

Zone A goes down: 4 of 7 replicas remain (57%).
Zone B goes down: 5 of 7 replicas remain (71%).

Without topologySpreadConstraints, in the worst case all 7 could be in one zone:
That zone goes down: 0 of 7 remain (0%). Platform down.

affinity:
  podAntiAffinity:
    # HARD version: it cannot be violated
    requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchLabels:
            app: bookings-api
        topologyKey: kubernetes.io/hostname

Rule: at most 1 bookings-api replica per node.
Nodes: 9.
Maximum placeable replicas: 9.

The HPA asks for 30 replicas.
9 are placed. The other 21 stay Pending FOREVER.

The Cluster Autoscaler sees pending pods and boots nodes... and with each new
node ONE more replica fits. For 30 replicas you would need 30 NODES.
Cost: absurd.

Rule: maxSkew 1 across nodes.
Nodes: 9. Replicas: 30.

Spread: 30 / 9 = 3.33
Result: 3 nodes with 4 replicas, 6 nodes with 3 replicas.
Skew: 4 - 3 = 1. Satisfied.

All 30 replicas are placed. Nothing Pending.

# bookings-api must NOT share a node with the reports CronJob, which uses
# 4 cores in one go and would throttle the API for 40 minutes.
affinity:
  podAntiAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchLabels:
            app: occupancy-reports       # ANOTHER application, not the same one
        topologyKey: kubernetes.io/hostname

spec:
  template:
    spec:
      # 1. Even spread across zones and nodes (availability)
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: bookings-api
          matchLabelKeys: ["pod-template-hash"]
        - maxSkew: 2
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: ScheduleAnyway
          labelSelector:
            matchLabels:
              app: bookings-api
          matchLabelKeys: ["pod-template-hash"]

      # 2. Anti-affinity with ANOTHER application (resource isolation)
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchLabels:
                  app: occupancy-reports
              topologyKey: kubernetes.io/hostname

# k8s/base/deployment-bookings-api.yaml (fragment)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: bookings-api
  namespace: rutas-norte-pro
spec:
  # No replicas: governed by KEDA/HPA (09-01, 09-04)
  selector:
    matchLabels:
      app: bookings-api
  template:
    metadata:
      labels:
        app: bookings-api
        app.kubernetes.io/part-of: rutas-norte
    spec:
      topologySpreadConstraints:
        # LEVEL 1: across AVAILABILITY ZONES. A HARD constraint.
        #
        # This is the big failure domain: if a whole zone goes down, we must
        # guarantee that replicas survive in the other two. It is a
        # guarantee we are NOT willing to negotiate, hence DoNotSchedule:
        # we prefer a Pending pod to an unbalanced spread.
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: bookings-api
          # Always 3 zones, even if one has no nodes right now. Without this,
          # if the Cluster Autoscaler only has nodes in 2 zones, a "balanced"
          # spread across those 2 would satisfy the constraint and we would lose
          # the three-zone guarantee.
          minDomains: 3
          matchLabelKeys: ["pod-template-hash"]

        # LEVEL 2: across NODES. A SOFT constraint.
        #
        # Spreading across nodes is desirable but NOT at the price of leaving
        # pods Pending. During the May bank holiday, with 30 replicas and the
        # Cluster Autoscaler booting nodes, a DoNotSchedule here would block
        # scaling exactly when it is needed most.
        #
        # maxSkew 2 (not 1): with 30 replicas and 9 nodes, requiring maxSkew 1
        # would be unnecessarily rigid. A 4-3-3-4-3-3-4-3-3 is perfectly healthy.
        - maxSkew: 2
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: ScheduleAnyway
          labelSelector:
            matchLabels:
              app: bookings-api
          matchLabelKeys: ["pod-template-hash"]

      containers:
        - name: api
          image: registry.rutasnorte.example/bookings-api:1.14.2
          resources:
            requests: {cpu: 412m, memory: 800Mi}
            limits: {cpu: "1", memory: 1600Mi}

3 zones, maxSkew 1, minDomains 3.

Spread: 4 / 3 = 1.33
Result: A:2, B:1, C:1. Skew = 2-1 = 1. Satisfied.

Zone A goes down: 2 of 4 remain (50%).
Zone B or C goes down: 3 of 4 remain (75%).

With maxUnavailable: 25% in the PDB, 4 replicas allow 1 eviction.
A zone outage (2 replicas) is INVOLUNTARY: the PDB does not apply.
2 replicas survive -> the service carries on, degraded.
KEDA/HPA will detect the doubled load per replica and scale up.

ZONE level (maxSkew 1, DoNotSchedule):
  30 / 3 = exactly 10.
  Result: A:10, B:10, C:10. Skew 0. Perfect.

NODE level (maxSkew 2, ScheduleAnyway), with 9 nodes (3 per zone):
  Within each zone, 10 replicas across 3 nodes: 4-3-3.
  Global skew across nodes: 4 - 3 = 1 <= 2. Satisfied.

  Final spread:
    zone A: node-a1:4, node-a2:3, node-a3:3
    zone B: node-b1:4, node-b2:3, node-b3:3
    zone C: node-c1:4, node-c2:3, node-c3:3

A NODE goes down: 3 or 4 replicas of 30 are lost (10-13%). Imperceptible.
A ZONE goes down: 10 of 30 are lost (33%). The service holds with 20.

      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: web-store
          minDomains: 3
          matchLabelKeys: ["pod-template-hash"]
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          # maxSkew 1 and ScheduleAnyway: with only 3-15 replicas, a spread
          # of 1 per node is achievable and desirable. But we still do not
          # block scaling.
          whenUnsatisfiable: ScheduleAnyway
          labelSelector:
            matchLabels:
              app: web-store
          matchLabelKeys: ["pod-template-hash"]

# k8s/base/statefulset-bookings-postgres.yaml (fragment)
spec:
  template:
    spec:
      # It lives in the data node group (09-03), with its taint and its affinity.
      tolerations:
        - key: role
          operator: Equal
          value: data
          effect: NoSchedule
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: role
                    operator: In
                    values: ["data"]
        # Do NOT share a node with redis-cache: if that node goes down, we do not
        # want to lose the database AND the cache at once, because starting
        # the database with an empty cache is the worst possible scenario.
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchLabels:
                  app: redis-cache
              topologyKey: kubernetes.io/hostname

A 3-member etcd cluster. 2 go down.
Quorum required: 2. Live members: 1. NO QUORUM.

Consequences:
  - etcd goes READ-ONLY. It accepts no writes.
  - The API server cannot persist changes.
  - kubectl apply fails. Objects cannot be created, modified or deleted.
  - The HPA cannot change replicas.
  - The Cluster Autoscaler can do nothing.

BUT, and this is the important part:
  - The pods that were ALREADY running KEEP RUNNING.
  - The kubelets keep running their containers.
  - The Services keep routing (kube-proxy has its local state).
  - THE PLATFORM KEEPS SERVING TRAFFIC.

3 control-plane nodes, one per availability zone:
  zone A: control-1 (etcd, api-server, scheduler, controller-manager)
  zone B: control-2
  zone C: control-3

One zone goes down: 2 of 3 remain. Quorum = 2. IT HOLDS.

# k8s/environments/pro/deployment-ingress-nginx.yaml (fragment)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  replicas: 4              # Never 1. Never 2 in the same failure domain.
  template:
    spec:
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app.kubernetes.io/name: ingress-nginx
          minDomains: 3
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: DoNotSchedule    # Here it IS hard
          labelSelector:
            matchLabels:
              app.kubernetes.io/name: ingress-nginx
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
spec:
  minAvailable: 2          # An absolute number: the Ingress replicas are FIXED
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
  unhealthyPodEvictionPolicy: AlwaysAllow

# Fragment of the CoreDNS Deployment in kube-system
spec:
  replicas: 3
  template:
    spec:
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: ScheduleAnyway
          labelSelector:
            matchLabels:
              k8s-app: kube-dns
      priorityClassName: system-cluster-critical

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: coredns
  namespace: kube-system
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      k8s-app: kube-dns
  unhealthyPodEvictionPolicy: AlwaysAllow

# Example with CloudNativePG
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
  name: bookings-postgres
  namespace: rutas-norte-pro
spec:
  instances: 3                    # 1 primary + 2 replicas
  primaryUpdateStrategy: unsupervised

  storage:
    size: 100Gi
    storageClass: rutasnorte-fast      # From 05-04

  # The operator spreads the instances across zones automatically
  affinity:
    enablePodAntiAffinity: true
    topologyKey: topology.kubernetes.io/zone
    podAntiAffinityType: required

  postgresql:
    parameters:
      max_connections: "200"
      shared_buffers: "1GB"

  monitoring:
    enablePodMonitor: true         # It integrates with Prometheus (07-03)

# Do all the PDBs allow at least one eviction?
kubectl get pdb --all-namespaces

NAMESPACE          NAME                   MIN AVAILABLE   MAX UNAVAILABLE   ALLOWED DISRUPTIONS
rutas-norte-pro    bookings-api           N/A             25%               1
rutas-norte-pro    web-store              N/A             34%               1
rutas-norte-pro    bookings-postgres      1               N/A               0     <- ATTENTION
rutas-norte-pro    redis-cache            1               N/A               0     <- ATTENTION
rutas-norte-pro    notifications-worker   N/A             50%               1
ingress-nginx      ingress-nginx          2               N/A               2
kube-system        coredns                N/A             1                 1

kubectl get pods -n rutas-norte-pro -o wide \
  -l 'app in (bookings-postgres,redis-cache)'

NAME                  READY   STATUS    NODE           ZONE
bookings-postgres-0   2/2     Running   data-node-1    eu-west-1a
redis-cache-0         1/1     Running   data-node-2    eu-west-1b

# What is on the node we are about to drain?
kubectl get pods --all-namespaces -o wide --field-selector spec.nodeName=node-a2

NAMESPACE         NAME                            READY   STATUS    AGE
rutas-norte-pro   bookings-api-7c9d4f8b6d-2mkjp   2/2     Running   4h
rutas-norte-pro   bookings-api-7c9d4f8b6d-9nvtr   2/2     Running   4h
rutas-norte-pro   web-store-6f7d9c4b58-4kjnx      1/1     Running   2d
kube-system       fluentd-x7kqp                   1/1     Running   12d   <- DaemonSet
kube-system       falco-2wxkp                     1/1     Running   12d   <- DaemonSet
kube-system       kube-proxy-mn2vp                1/1     Running   12d   <- DaemonSet

# Are there active alerts? Never do maintenance with open incidents.
kubectl get events --all-namespaces --field-selector type=Warning \
  --sort-by=.lastTimestamp | tail -20

kubectl cordon node-a2

node/node-a2 cordoned

kubectl get nodes

NAME      STATUS                     ROLES    AGE   VERSION
node-a1   Ready                      <none>   12d   v1.30.2
node-a2   Ready,SchedulingDisabled   <none>   12d   v1.30.2
node-a3   Ready                      <none>   12d   v1.30.2

kubectl drain node-a2 \
  --ignore-daemonsets \
  --delete-emptydir-data \
  --grace-period=60 \
  --timeout=300s

node/node-a2 already cordoned
Warning: ignoring DaemonSet-managed Pods: kube-system/fluentd-x7kqp,
         kube-system/falco-2wxkp, kube-system/kube-proxy-mn2vp
evicting pod rutas-norte-pro/bookings-api-7c9d4f8b6d-2mkjp
evicting pod rutas-norte-pro/web-store-6f7d9c4b58-4kjnx
evicting pod rutas-norte-pro/bookings-api-7c9d4f8b6d-9nvtr
error when evicting pods/"bookings-api-7c9d4f8b6d-9nvtr" -n "rutas-norte-pro"
(will retry after 5s): Cannot evict pod as it would violate the pod's disruption budget.
pod/web-store-6f7d9c4b58-4kjnx evicted
pod/bookings-api-7c9d4f8b6d-2mkjp evicted
evicting pod rutas-norte-pro/bookings-api-7c9d4f8b6d-9nvtr
pod/bookings-api-7c9d4f8b6d-9nvtr evicted
node/node-a2 drained

# The node must be empty of application pods
kubectl get pods --all-namespaces -o wide --field-selector spec.nodeName=node-a2

NAMESPACE     NAME                READY   STATUS    AGE
kube-system   fluentd-x7kqp       1/1     Running   12d
kube-system   falco-2wxkp         1/1     Running   12d
kube-system   kube-proxy-mn2vp    1/1     Running   12d

# And the SERVICE must be healthy: this is what really matters
kubectl get pods -n rutas-norte-pro -l app=bookings-api
kubectl get pdb -n rutas-norte-pro

# Example: upgrading the kubelet on a kubeadm cluster (10-02)
ssh node-a2

sudo apt-get update
sudo apt-get install -y kubeadm=1.30.3-1.1
sudo kubeadm upgrade node
sudo apt-get install -y kubelet=1.30.3-1.1 kubectl=1.30.3-1.1
sudo systemctl daemon-reload
sudo systemctl restart kubelet

exit

kubectl get node node-a2

NAME      STATUS                     ROLES    AGE   VERSION
node-a2   Ready,SchedulingDisabled   <none>   12d   v1.30.3

# Check there are no error conditions
kubectl describe node node-a2 | grep -A10 Conditions

kubectl uncordon node-a2

node/node-a2 uncordoned

NAME      STATUS   ROLES    AGE   VERSION
node-a2   Ready    <none>   12d   v1.30.3

# Check the spread before continuing
kubectl get pods -n rutas-norte-pro -l app=bookings-api -o wide \
  | awk '{print $7}' | sort | uniq -c

      2 node-a1
      1 node-a3
      2 node-b1
      1 node-b2

kubectl drain node-a2 --ignore-daemonsets --delete-emptydir-data

node/node-a2 cordoned
evicting pod rutas-norte-pro/bookings-api-7c9d4f8b6d-2mkjp
evicting pod rutas-norte-pro/bookings-api-7c9d4f8b6d-9nvtr
evicting pod rutas-norte-pro/web-store-6f7d9c4b58-4kjnx
pod/bookings-api-7c9d4f8b6d-2mkjp evicted
pod/bookings-api-7c9d4f8b6d-9nvtr evicted
pod/web-store-6f7d9c4b58-4kjnx evicted
node/node-a2 drained

bookings-api had 4 replicas: 2 on node-a2, 1 on node-a1, 1 on node-b1.
After the drain: 2 live replicas, 2 starting (25-30 seconds).

For 30 seconds: HALF of the API's capacity.
Each live replica receives DOUBLE the load.
p95 latency: from 180 ms to 900 ms.
Some requests: timeout.

And if 3 of the 4 replicas had been on that node: 25% of capacity.
The surviving replica saturates and falls over. TOTAL OUTAGE.

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: capacity-filler
  namespace: rutas-norte-pro
spec:
  # 100% unavailability allowed: these pods exist TO be evicted.
  # It documents the intent better than the absence of a PDB.
  maxUnavailable: 100%
  selector:
    matchLabels:
      app: capacity-filler

# In the Deployment: it governs the UPDATE
spec:
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 25%     # Consistent with the PDB
      maxSurge: 25%
---
# In the PDB: it governs EVICTIONS (drain, CA, VPA)
spec:
  maxUnavailable: 25%

minikube start -p rutas-norte --nodes=4 --cpus=2 --memory=4096

# Label the nodes with fictitious zones
kubectl label node rutas-norte     topology.kubernetes.io/zone=zone-a
kubectl label node rutas-norte-m02 topology.kubernetes.io/zone=zone-a
kubectl label node rutas-norte-m03 topology.kubernetes.io/zone=zone-b
kubectl label node rutas-norte-m04 topology.kubernetes.io/zone=zone-b

# /tmp/chaos-demo.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: chaos-api
  namespace: rutas-norte-dev
spec:
  replicas: 6
  selector:
    matchLabels:
      app: chaos-api
  template:
    metadata:
      labels:
        app: chaos-api
    spec:
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: chaos-api
          matchLabelKeys: ["pod-template-hash"]
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: ScheduleAnyway
          labelSelector:
            matchLabels:
              app: chaos-api
          matchLabelKeys: ["pod-template-hash"]
      containers:
        - name: web
          image: nginx:1.27-alpine
          resources:
            requests: {cpu: 50m, memory: 32Mi}
          readinessProbe:
            httpGet: {path: /, port: 80}
            initialDelaySeconds: 2
            periodSeconds: 3
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: chaos-api
  namespace: rutas-norte-dev
spec:
  maxUnavailable: 25%
  selector:
    matchLabels:
      app: chaos-api
  unhealthyPodEvictionPolicy: AlwaysAllow
---
apiVersion: v1
kind: Service
metadata:
  name: chaos-api
  namespace: rutas-norte-dev
spec:
  selector:
    app: chaos-api
  ports:
    - port: 80

kubectl apply -f /tmp/chaos-demo.yaml
kubectl get pods -n rutas-norte-dev -o wide

NAME                         READY   STATUS    NODE              ZONE
chaos-api-7d9c5f8b6b-2mkjp   1/1     Running   rutas-norte       zone-a
chaos-api-7d9c5f8b6b-4nqzx   1/1     Running   rutas-norte-m02   zone-a
chaos-api-7d9c5f8b6b-7wtrv   1/1     Running   rutas-norte-m02   zone-a
chaos-api-7d9c5f8b6b-9hbcx   1/1     Running   rutas-norte-m03   zone-b
chaos-api-7d9c5f8b6b-kp3ln   1/1     Running   rutas-norte-m04   zone-b
chaos-api-7d9c5f8b6b-tv6ws   1/1     Running   rutas-norte-m03   zone-b

# Terminal 1: generate continuous traffic and count errors
kubectl -n rutas-norte-dev run client --rm -it --restart=Never \
  --image=busybox:1.36 -- /bin/sh -c \
  'ok=0; err=0; while true; do
     if wget -q -T2 -O- http://chaos-api > /dev/null 2>&1; then
       ok=$((ok+1)); else err=$((err+1)); fi;
     echo "OK=$ok ERR=$err";
     sleep 0.2;
   done'

# Terminal 2: the drain
kubectl drain rutas-norte-m02 --ignore-daemonsets --delete-emptydir-data

OK=847 ERR=0
OK=848 ERR=0
OK=849 ERR=0
...
OK=1204 ERR=0

# Restore
kubectl uncordon rutas-norte-m02

# Simulate a hardware failure: switch the node off abruptly
minikube -p rutas-norte node stop rutas-norte-m03

kubectl get nodes --watch

NAME              STATUS     ROLES    AGE
rutas-norte-m03   Ready      <none>   1h
rutas-norte-m03   NotReady   <none>   1h     <- after ~40 seconds

kubectl get pods -n rutas-norte-dev -o wide --watch

NAME                         READY   STATUS        NODE
chaos-api-7d9c5f8b6b-9hbcx   1/1     Running       rutas-norte-m03
chaos-api-7d9c5f8b6b-tv6ws   1/1     Running       rutas-norte-m03
...
(~5 minutes pass)
...
chaos-api-7d9c5f8b6b-9hbcx   1/1     Terminating   rutas-norte-m03
chaos-api-7d9c5f8b6b-tv6ws   1/1     Terminating   rutas-norte-m03
chaos-api-7d9c5f8b6b-x2klm   0/1     Pending       <none>
chaos-api-7d9c5f8b6b-p9wnz   0/1     Pending       <none>
chaos-api-7d9c5f8b6b-x2klm   0/1     ContainerCreating   rutas-norte-m04
chaos-api-7d9c5f8b6b-p9wnz   1/1     Running             rutas-norte-m04

OK=1204 ERR=0
OK=1210 ERR=3      <- the errors start
OK=1218 ERR=11
...
OK=1301 ERR=68     <- ~40 seconds of errors
OK=1409 ERR=68     <- they stop: the Service has excluded the dead node
OK=1520 ERR=68

minikube -p rutas-norte node start rutas-norte-m03
kubectl delete -f /tmp/chaos-demo.yaml

# PDBs with no evictions allowed, excluding the intentional ones
kube_poddisruptionbudget_status_pod_disruptions_allowed == 0
  unless on(namespace, poddisruptionbudget)
  kube_poddisruptionbudget_annotations{annotation_rutasnorte_example_pdb_intentional_block!=""}

# Replicas per zone
kubectl get pods -n rutas-norte-pro -l app=bookings-api \
  -o custom-columns=NODE:.spec.nodeName --no-headers | \
  while read n; do kubectl get node "$n" -o jsonpath='{.metadata.labels.topology\.kubernetes\.io/zone}{"\n"}'; done | \
  sort | uniq -c

      3 eu-west-1a
      3 eu-west-1b
      2 eu-west-1c

kubectl drain node-b1 --ignore-daemonsets --delete-emptydir-data

node/node-b1 already cordoned
Warning: ignoring DaemonSet-managed Pods: kube-system/fluentd-2wxkp, kube-system/falco-x7kqp
evicting pod monitoring/prometheus-server-0
evicting pod rutas-norte-pro/bookings-api-7c9d4f8b6d-mn2vp
evicting pod rutas-norte-pro/notifications-worker-5f8d9c-4kjnx
pod/notifications-worker-5f8d9c-4kjnx evicted
error when evicting pods/"prometheus-server-0" -n "monitoring"
(will retry after 5s): Cannot evict pod as it would violate the pod's disruption budget.
error when evicting pods/"bookings-api-7c9d4f8b6d-mn2vp" -n "rutas-norte-pro"
(will retry after 5s): Cannot evict pod as it would violate the pod's disruption budget.

NAMESPACE          NAME                   MIN AVAILABLE   MAX UNAVAILABLE   ALLOWED DISRUPTIONS
rutas-norte-pro    bookings-api           8               N/A               0
rutas-norte-pro    web-store              N/A             34%               1
rutas-norte-pro    notifications-worker   N/A             50%               1
monitoring         prometheus             1               N/A               0

NAMESPACE          NAME                            READY   STATUS    NODE
rutas-norte-pro    bookings-api-7c9d4f8b6d-2mkjp   2/2     Running   node-a1
rutas-norte-pro    bookings-api-7c9d4f8b6d-9nvtr   2/2     Running   node-a2
rutas-norte-pro    bookings-api-7c9d4f8b6d-mn2vp   2/2     Running   node-b1
rutas-norte-pro    bookings-api-7c9d4f8b6d-tv6ws   2/2     Running   node-b2
rutas-norte-pro    bookings-api-7c9d4f8b6d-x2klm   1/2     Running   node-a1
monitoring         prometheus-server-0             1/1     Running   node-b1

topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: DoNotSchedule
    minDomains: 3
    labelSelector:
      matchLabels:
        app: bookings-api
  - maxSkew: 2
    topologyKey: kubernetes.io/hostname
    whenUnsatisfiable: ScheduleAnyway
    labelSelector:
      matchLabels:
        app: bookings-api

# PDB
spec:
  maxUnavailable: 25%

Total bookings-api replicas: 5
  - 4 with READY 2/2 (healthy)
  - 1 with READY 1/2 (the pod x2klm: one container is not ready)

AVAILABLE (Ready) replicas: 4
The PDB requires: minAvailable: 8

4 < 8  ->  we are ALREADY below the minimum.
ALLOWED DISRUPTIONS: 0. And it will stay 0 whatever happens.

kubectl describe pod bookings-api-7c9d4f8b6d-x2klm -n rutas-norte-pro
kubectl logs bookings-api-7c9d4f8b6d-x2klm -n rutas-norte-pro -c metrics-exporter

kubectl patch pdb bookings-api -n rutas-norte-pro --type merge \
  -p '{"spec":{"minAvailable":null,"maxUnavailable":"25%"}}'

Desired replicas: 5
maxUnavailable: 25% -> floor(5 x 0.25) = 1
Minimum available: 5 - 1 = 4
Currently available: 4
ALLOWED DISRUPTIONS: 0

...Still 0, because of the pod x2klm that is not Ready.

# Way A: if the pod is broken beyond repair, delete it so it gets recreated
kubectl delete pod bookings-api-7c9d4f8b6d-x2klm -n rutas-norte-pro
# (kubectl delete does NOT respect PDBs: here that works in our favour)

# Way B: add unhealthyPodEvictionPolicy so that unhealthy pods
# do not consume budget
kubectl patch pdb bookings-api -n rutas-norte-pro --type merge \
  -p '{"spec":{"unhealthyPodEvictionPolicy":"AlwaysAllow"}}'

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: bookings-api
  namespace: rutas-norte-pro
  annotations:
    rutasnorte.example/note: >
      A PERCENTAGE is mandatory: bookings-api has KEDA/HPA and swings between
      4 and 30 replicas. An absolute minAvailable becomes impossible in the
      trough and blocks all maintenance. See 09-05.
spec:
  maxUnavailable: 25%
  selector:
    matchLabels:
      app: bookings-api
  unhealthyPodEvictionPolicy: AlwaysAllow

prometheus-server-0: 1 replica (it is a StatefulSet).
PDB: minAvailable: 1.
Evicting it would leave 0 available.
ALLOWED DISRUPTIONS: 0. Permanent and by design.

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: prometheus
  namespace: monitoring
spec:
  # A single replica: we allow the eviction. We lose a few minutes of
  # metrics during the maintenance, which is acceptable: Prometheus
  # is not on the critical path of the sale.
  maxUnavailable: 1
  selector:
    matchLabels:
      app: prometheus

# 1. Fix the bookings-api PDB
kubectl patch pdb bookings-api -n rutas-norte-pro --type merge \
  -p '{"spec":{"minAvailable":null,"maxUnavailable":"25%","unhealthyPodEvictionPolicy":"AlwaysAllow"}}'

# 2. Fix the prometheus PDB
kubectl patch pdb prometheus -n monitoring --type merge \
  -p '{"spec":{"minAvailable":null,"maxUnavailable":1}}'

# 3. Verify
kubectl get pdb --all-namespaces

NAMESPACE          NAME                   MAX UNAVAILABLE   ALLOWED DISRUPTIONS
rutas-norte-pro    bookings-api           25%               1
monitoring         prometheus             1                 1

# 4. Investigate the pod x2klm (not urgent, but do not forget it)
kubectl describe pod bookings-api-7c9d4f8b6d-x2klm -n rutas-norte-pro

kubectl get pdb --all-namespaces \
  -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,ALLOWED:.status.disruptionsAllowed' \
  | awk 'NR==1 || $3=="0"'

kube_poddisruptionbudget_status_pod_disruptions_allowed == 0

11 / 3 = 3.67

Most balanced spread possible:
  eu-west-1a: 4
  eu-west-1b: 4
  eu-west-1c: 3

Skew = 4 - 3 = 1 <= maxSkew 1. SATISFIED.

Zone a (4 replicas, 3 nodes): 2-1-1
Zone b (4 replicas, 3 nodes): 2-1-1
Zone c (3 replicas, 3 nodes): 1-1-1

Global skew across nodes: max 2 - min 1 = 1 <= maxSkew 2. SATISFIED.

Worst case: a node with 2 replicas goes down.
Lost: 2 of 11.
Remaining: 9 of 11 = 81.8% of capacity.

Worst case: a zone with 4 replicas goes down.
Lost: 4 of 11.
Remaining: 7 of 11 = 63.6% of capacity.

maxUnavailable: 25% over 11 replicas.
floor(11 x 0.25) = floor(2.75) = 2

ALLOWED DISRUPTIONS: 2

26 / 3 = 8.67

Spread:
  eu-west-1a: 9
  eu-west-1b: 9
  eu-west-1c: 8

Skew = 9 - 8 = 1 <= maxSkew 1. SATISFIED.

Zone a (9 replicas, 3 nodes): 3-3-3
Zone b (9 replicas, 3 nodes): 3-3-3
Zone c (8 replicas, 3 nodes): 3-3-2

Global skew across nodes: 3 - 2 = 1 <= maxSkew 2. SATISFIED.

Worst case: a node with 3 replicas goes down.
Lost: 3 of 26.
Remaining: 23 of 26 = 88.5% of capacity.

Worst case: a zone with 9 replicas goes down.
Lost: 9 of 26.
Remaining: 17 of 26 = 65.4% of capacity.

floor(26 x 0.25) = floor(6.5) = 6

ALLOWED DISRUPTIONS: 6

A node holds 3 bookings-api replicas.
The PDB allows 6 simultaneous evictions.

3 <= 6  ->  YES, all 3 can be evicted at once.

The drain will NOT be serialised by the bookings-api PDB.
It will be almost instantaneous for this component.

With 11 replicas, a node holds 2 replicas and the PDB allows 2 evictions.
2 <= 2 -> they also fit in one go, though right at the limit.

If the node held 3 replicas (possible with maxSkew 2), the PDB would serialise:
evict 2, wait for them to be recreated, evict the third.

minDomains: 3 means the scheduler considers 3 domains,
even if some of them are empty.

Placing replica 1: zone a. State: a:1, b:0, c:0.
  Skew = 1 - 0 = 1 <= maxSkew 1. Satisfied.

Placing replica 2:
  If it goes to zone a: a:2, b:0, c:0. Skew = 2 - 0 = 2 > 1. NOT ALLOWED.
  If it goes to zone b: a:1, b:1, c:0. Skew = 1 - 0 = 1 <= 1. ALLOWED.

Result: 1 replica in zone a, 1 in zone b, 0 in c.
Zone c is left empty, and that IS CORRECT: with 2 replicas you cannot
cover 3 zones.

Scenario: zone c loses all its nodes (a complete outage).
Available nodes: only in zones a and b.

minDomains: 3 makes the scheduler KEEP COUNTING 3 domains.

State after the outage: a:1, b:1, c:0 (zone c has no nodes).
Skew = 1 - 0 = 1 <= maxSkew 1. SATISFIED. Nothing to do.

But if the HPA wanted to scale to 4 replicas:
  Placing replica 3:
    If it goes to zone a: a:2, b:1, c:0. Skew = 2 - 0 = 2 > 1. NOT ALLOWED.
    If it goes to zone b: a:1, b:2, c:0. Skew = 2 - 0 = 2 > 1. NOT ALLOWED.
    If it goes to zone c: THERE ARE NO NODES.

  -> Replica 3 stays PENDING FOREVER.

# EMERGENCY PROCEDURE: zone outage
# Relax the topology constraint to allow scaling in the surviving zones.
kubectl patch deployment bookings-api -n rutas-norte-pro --type json -p '[
  {"op": "replace", "path": "/spec/template/spec/topologySpreadConstraints/0/whenUnsatisfiable",
   "value": "ScheduleAnyway"}
]'
# REVERT when the zone comes back.

The bank allows 50 simultaneous connections from the Rutas Norte IP.

If each replica keeps a pool of N connections:
  Maximum replicas = 50 / N

With N = 5 connections per replica:  10 maximum replicas
With N = 3 connections per replica:  16 maximum replicas
With N = 2 connections per replica:  25 maximum replicas

How many transactions per second does one replica sustain?
  Each transaction takes 2-8 s, averaging ~5 s.
  With a pool of N concurrent connections:
    transactions/s = N / 5

  With N = 5: 1 transaction/s per replica
  With N = 3: 0.6 transactions/s per replica

For 300 transactions/s at the peak:
  With N = 5: 300 replicas. IMPOSSIBLE (30x over the bank's limit).
  With N = 3: 500 replicas. Worse.

The system's THEORETICAL MAXIMUM capacity:
  50 concurrent connections / 5 seconds per transaction = 10 transactions/s

PEAK TARGET: 300 transactions/s

THE SYSTEM CANNOT PROCESS MORE THAN 10 TRANSACTIONS PER SECOND.
The limit is the BANK, not Kubernetes.

1. NEGOTIATE a higher limit with the bank. 50 connections is ridiculous for
   300 transactions/s. It is the most important action and it is not technical.

2. In the meantime: DECOUPLE with a queue.
   - bookings-api queues the charge request and answers the user
     "your payment is being processed".
   - payments-gateway consumes from the queue at the rate the bank allows.
   - The user receives confirmation by push notification or email.

   This turns a hard constraint into an acceptable latency, and it fits
   perfectly with KEDA (09-04): the queue is the natural scaling signal.

3. A SHARED POOL of connections to the bank, instead of a pool per replica.

# k8s/environments/pro/deployment-payments-gateway.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: payments-gateway
  namespace: rutas-norte-pro
  labels:
    app: payments-gateway
    app.kubernetes.io/part-of: rutas-norte
spec:
  # NO replicas field: it is governed by the HPA/KEDA.
  selector:
    matchLabels:
      app: payments-gateway

  strategy:
    type: RollingUpdate
    rollingUpdate:
      # Consistent with the PDB (see below). Never lose more than 1 replica.
      maxUnavailable: 1
      # maxSurge 1, no more: every new replica opens connections to the bank and
      # we have a hard limit of 50. A high maxSurge could exhaust it
      # during the deployment.
      maxSurge: 1

  template:
    metadata:
      labels:
        app: payments-gateway
        app.kubernetes.io/part-of: rutas-norte
    spec:
      # A 90-SECOND GRACE PERIOD.
      #
      # A transaction takes up to 8 seconds and CANNOT be interrupted:
      # cutting it could leave a charge with no associated booking, which is the
      # worst possible failure in a payment gateway.
      #
      # 90 s = 8 s (longest transaction) + a wide margin for:
      #   - draining the internal queue of in-flight transactions
      #   - writing the compliance log for each of them
      #   - cleanly closing the TLS connections to the bank
      #
      # The process MUST handle SIGTERM: stop accepting new requests,
      # finish the ones in flight, write the logs, and exit.
      terminationGracePeriodSeconds: 90

      topologySpreadConstraints:
        # ZONES: a HARD constraint. The requirement to survive a zone
        # outage is explicit and non-negotiable.
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
          minDomains: 3
          labelSelector:
            matchLabels:
              app: payments-gateway
          matchLabelKeys: ["pod-template-hash"]

        # NODES: a HARD constraint too, and here we CAN afford it.
        # With a maximum of 9 replicas and 9 nodes, requiring 1 per node is
        # achievable. And two gateway replicas on the same node are
        # two replicas lost together: unacceptable on the critical
        # path of a payment.
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: payments-gateway
          matchLabelKeys: ["pod-template-hash"]

      containers:
        - name: gateway
          image: registry.rutasnorte.example/payments-gateway:2.4.1

          # ORDERLY SHUTDOWN: the preStop hook gives the Service time to
          # remove this pod from its endpoints BEFORE it starts shutting down.
          # Without it, there is a 1-2 second window in which the pod receives
          # new requests while it is already shutting down.
          lifecycle:
            preStop:
              exec:
                command: ["/bin/sh", "-c", "sleep 10"]

          resources:
            requests: {cpu: 200m, memory: 256Mi}
            limits: {cpu: 500m, memory: 512Mi}

          env:
            # A SMALL, EXPLICIT POOL: 5 connections per replica.
            # With maxReplicas 9: 9 x 5 = 45 < 50. It leaves 5 connections spare
            # for the deployment (maxSurge 1) and for retries.
            # THIS NUMBER IS CRITICAL: raising it exhausts the bank's limit.
            - name: BANK_POOL_MAX
              value: "5"

          readinessProbe:
            httpGet: {path: /ready, port: 8080}
            initialDelaySeconds: 20        # Start-up takes 20 s (mutual TLS)
            periodSeconds: 5
            failureThreshold: 2
          livenessProbe:
            httpGet: {path: /health, port: 8080}
            initialDelaySeconds: 30
            periodSeconds: 10
            failureThreshold: 3
          # STARTUP PROBE: gives up to 60 s for start-up without the
          # livenessProbe killing the pod midway through the TLS negotiation.
          startupProbe:
            httpGet: {path: /health, port: 8080}
            periodSeconds: 5
            failureThreshold: 12

# k8s/environments/pro/pdb-payments-gateway.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: payments-gateway
  namespace: rutas-norte-pro
  labels:
    app: payments-gateway
    app.kubernetes.io/part-of: rutas-norte
spec:
  # maxUnavailable: 1 AS AN ABSOLUTE NUMBER, not a percentage.
  #
  # It is the exception to the general rule of section 3, and it is deliberate:
  #
  # 1. The replica range is NARROW (3 to 9), unlike bookings-api (4 to 30).
  #    A 25% percentage would give: 3 replicas -> 0 evictions (BLOCKED), and
  #    9 replicas -> 2 evictions. The 3-replica case would be an impossible PDB.
  #
  # 2. Every replica that goes down can take up to 5 in-flight transactions with it,
  #    each one with a customer's real money. We want evictions to be
  #    SERIALISED always, whatever the replica count.
  #
  # 3. With a terminationGracePeriodSeconds of 90 s, each eviction takes up to
  #    a minute and a half. Serialising them makes maintenance slow but safe,
  #    which is exactly the trade-off we want in a payment gateway.
  maxUnavailable: 1

  selector:
    matchLabels:
      app: payments-gateway

  # AlwaysAllow: a pod that fails readiness is not processing payments
  # (the Service is no longer sending it traffic), so evicting it puts
  # no transaction at risk.
  unhealthyPodEvictionPolicy: AlwaysAllow

# k8s/environments/pro/scaledobject-payments-gateway.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: payments-gateway
  namespace: rutas-norte-pro
spec:
  scaleTargetRef:
    name: payments-gateway

  pollingInterval: 15
  cooldownPeriod: 600

  # A floor of 3 replicas: ONE PER ZONE. It is the minimum for surviving a
  # zone outage with 2 operational replicas. NEVER scale to zero:
  # the 20 s start-up (mutual TLS negotiation with the bank) is unacceptable
  # with a user sitting on the payment screen.
  minReplicaCount: 3

  # 9 REPLICAS MAXIMUM. THIS NUMBER DOES NOT COME FROM THE TRAFFIC, IT COMES FROM THE BANK:
  #   9 replicas x 5 connections = 45 connections < the limit of 50.
  #   5 connections spare for the deployment (maxSurge 1) and for retries.
  #
  # CRITICAL WARNING: raising this number WITHOUT renegotiating the limit with the
  # bank does NOT add capacity. It only makes the replicas compete for
  # the same 50 connections and start receiving connection-refused
  # errors. The bottleneck is the BANK (see 09-01).
  maxReplicaCount: 9

  fallback:
    failureThreshold: 3
    replicas: 6

  advanced:
    horizontalPodAutoscalerConfig:
      behavior:
        scaleUp:
          # 30 s of stabilisation (not 0): every new replica takes 20 s to
          # be ready and opens 5 connections to the bank. We do not want to create
          # replicas wildly and exhaust the connection limit.
          stabilizationWindowSeconds: 30
          selectPolicy: Max
          policies:
            - type: Pods
              value: 2
              periodSeconds: 60
        scaleDown:
          # 15 minutes. Every replica that shuts down takes up to 90 s to finish
          # its transactions. Coming down slowly is essential.
          stabilizationWindowSeconds: 900
          selectPolicy: Min
          policies:
            - type: Pods
              value: 1
              periodSeconds: 300

  triggers:
    # Scaling on ACTIVE CONNECTIONS TO THE BANK, not on CPU nor on requests.
    # It is the metric that reflects the real scarce resource.
    - type: prometheus
      metadata:
        serverAddress: http://prometheus-operated.monitoring.svc.cluster.local:9090
        metricName: gateway_bank_connections_active
        query: |
          sum(payments_gateway_bank_connections_active{namespace="rutas-norte-pro"})
        # Target: 3.5 active connections per replica (out of a pool of 5).
        # At 70% pool occupancy, we scale.
        threshold: "3.5"
        activationThreshold: "1"
      authenticationRef:
        kind: ClusterTriggerAuthentication
        name: prometheus-read

    # Safety net: transactions queued waiting for a free connection.
    - type: prometheus
      metadata:
        serverAddress: http://prometheus-operated.monitoring.svc.cluster.local:9090
        metricName: gateway_transactions_waiting
        query: |
          sum(payments_gateway_transactions_waiting{namespace="rutas-norte-pro"})
        threshold: "3"
        activationThreshold: "1"
      authenticationRef:
        kind: ClusterTriggerAuthentication
        name: prometheus-read