1. Read a message from the queue          ~2 ms of CPU
2. Look up the booking data               ~5 ms of CPU + 8 ms waiting on PostgreSQL
3. Render the email template              ~12 ms of CPU
4. Connect to the SMTP server and send    ~3 ms of CPU + 450 ms WAITING
5. Acknowledge the message on the queue    ~1 ms of CPU

Total: ~23 ms of CPU and ~458 ms of waiting.
Ratio CPU/total time: 23 / 481 = 4.8%

ratio = 60 / 210 = 0.286
desiredReplicas = ceil(2 x 0.286) = 1

The HPA wants to go DOWN to 1 replica.

Queue: 40,000 messages.
One worker processes ~2 messages/second (limited by SMTP).
Goal: drain the queue in 20 minutes = 1,200 seconds.

Messages per second needed: 40,000 / 1,200 = 33.3
Workers needed: 33.3 / 2 = 17 workers.

Conditions:
  Type           Status  Reason                 Message
  ----           ------  ------                 -------
  ScalingActive  False   FailedGetExternalMetric  unable to get external metric
                         rutas-norte-pro/email_queue_length/nil:
                         no custom metrics API (external.metrics.k8s.io/v1beta1)
                         registered

- type: Pods
  pods:
    metric:
      name: requests_per_second
    target:
      type: AverageValue
      averageValue: "85"

- type: Object
  object:
    describedObject:
      apiVersion: networking.k8s.io/v1
      kind: Ingress
      name: rutas-norte-public
    metric:
      name: requests_per_second
    target:
      type: Value
      value: "5000"

- type: External
  external:
    metric:
      name: email_queue_length
      selector:
        matchLabels:
          queue: confirmation-notifications
    target:
      type: AverageValue
      averageValue: "2500"

kubectl get apiservices | grep metrics

NAME                                    SERVICE                              AVAILABLE   AGE
v1beta1.metrics.k8s.io                  kube-system/metrics-server           True        41d
v1beta1.external.metrics.k8s.io         keda/keda-operator-metrics-apiserver  True        12d

# Fragment of the prometheus-adapter configuration
rules:
  - seriesQuery: 'rutasnorte_requests_total{namespace!="",pod!=""}'
    resources:
      overrides:
        namespace: {resource: "namespace"}
        pod: {resource: "pod"}
    name:
      matches: "^rutasnorte_requests_total$"
      as: "requests_per_second"
    metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[2m])) by (<<.GroupBy>>)'

kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/namespaces/rutas-norte-pro/pods/*/requests_per_second" \
  | python3 -m json.tool

# With Helm (we will cover it in depth in 10-03)
helm repo add kedacore https://kedacore.github.io/charts
helm repo update
helm install keda kedacore/keda --namespace keda --create-namespace

# Verification
kubectl get pods -n keda

NAME                                              READY   STATUS    RESTARTS   AGE
keda-admission-webhooks-6d8f7c9b4d-x2klm          1/1     Running   0          2m
keda-operator-7d9c5f8b6b-4mnpq                    1/1     Running   0          2m
keda-operator-metrics-apiserver-5f7b8c9d6a-9wrtz  1/1     Running   0          2m

kubectl get crd | grep keda

clustertriggerauthentications.keda.sh    2026-04-20T11:32:14Z
scaledjobs.keda.sh                       2026-04-20T11:32:14Z
scaledobjects.keda.sh                    2026-04-20T11:32:14Z
triggerauthentications.keda.sh           2026-04-20T11:32:14Z

kubectl get hpa -n rutas-norte-pro

NAME                                  REFERENCE                          TARGETS        MINPODS  MAXPODS  REPLICAS
keda-hpa-notifications-worker         Deployment/notifications-worker    1847/2500 (avg)  1        25       8

kubectl get hpa keda-hpa-notifications-worker -n rutas-norte-pro -o yaml

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: keda-hpa-notifications-worker
  namespace: rutas-norte-pro
  labels:
    app.kubernetes.io/managed-by: keda-operator
    scaledobject.keda.sh/name: notifications-worker
  ownerReferences:
    - apiVersion: keda.sh/v1alpha1
      kind: ScaledObject
      name: notifications-worker
      controller: true
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: notifications-worker
  minReplicas: 1
  maxReplicas: 25
  metrics:
    - type: External
      external:
        metric:
          name: s0-rabbitmq-confirmation-notifications
          selector:
            matchLabels:
              scaledobject.keda.sh/name: notifications-worker
        target:
          type: AverageValue
          averageValue: "2500"

apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: notifications-worker
  namespace: rutas-norte-pro
spec:
  # WHICH workload it applies to.
  scaleTargetRef:
    apiVersion: apps/v1        # Optional, apps/v1 by default
    kind: Deployment           # Optional, Deployment by default
    name: notifications-worker
    envSourceContainerName: worker   # Which container to read env vars from

  # HOW OFTEN KEDA queries the external source.
  pollingInterval: 15

  # HOW LONG it waits with no activity before dropping to ZERO.
  cooldownPeriod: 300

  # Replicas when there is NO activity. 0 = scale-to-zero.
  minReplicaCount: 0
  maxReplicaCount: 25

  # Replicas in an "idle" state OTHER than zero.
  idleReplicaCount: 0

  # What to do if the metrics source DOES NOT RESPOND.
  fallback:
    failureThreshold: 3
    replicas: 6

  # Advanced configuration.
  advanced:
    restoreToOriginalReplicaCount: false
    horizontalPodAutoscalerConfig:
      name: hpa-notifications-worker
      behavior:
        scaleUp: {}
        scaleDown: {}

  # WHAT it watches. It is a list: there can be several.
  triggers:
    - type: rabbitmq
      metadata:
        queueName: confirmation-notifications
        mode: QueueLength
        value: "2500"
      authenticationRef:
        name: rabbitmq-credentials

minReplicaCount: 3       # When there is activity, at least 3
idleReplicaCount: 1      # When there is NO activity, drop to 1

fallback:
  failureThreshold: 3    # After 3 consecutive failed queries...
  replicas: 6            # ...pin 6 replicas and hold them

advanced:
  horizontalPodAutoscalerConfig:
    name: hpa-notifications-worker       # Custom name for the generated HPA
    behavior:
      scaleUp:
        stabilizationWindowSeconds: 0
        selectPolicy: Max
        policies:
          - type: Percent
            value: 100
            periodSeconds: 30
          - type: Pods
            value: 5
            periodSeconds: 30
      scaleDown:
        stabilizationWindowSeconds: 300
        selectPolicy: Min
        policies:
          - type: Percent
            value: 25
            periodSeconds: 60

triggers:
  - type: rabbitmq        # Primary signal: the queue
    metadata: {...}
  - type: cpu             # Safety net: if CPU spikes, scale anyway
    metricType: Utilization
    metadata:
      value: "80"

- type: prometheus
  metadata:
    serverAddress: http://prometheus-operated.monitoring.svc.cluster.local:9090
    query: |
      sum(rate(rutasnorte_requests_total{namespace="rutas-norte-pro"}[2m]))
    threshold: "85"
    # What value to use if the query returns nothing (an empty series)
    ignoreNullValues: "true"
    unsafeSsl: "false"

- type: rabbitmq
  metadata:
    protocol: amqp
    queueName: confirmation-notifications
    mode: QueueLength        # QueueLength or MessageRate
    value: "2500"
    # activationValue: below this, it is considered INACTIVE (for the zero)
    activationValue: "10"
  authenticationRef:
    name: rabbitmq-credentials

- type: cron
  metadata:
    timezone: Europe/Madrid
    start: "30 9 * * *"       # At 9:30
    end: "0 14 * * *"         # Until 14:00
    desiredReplicas: "25"

- type: postgresql
  metadata:
    query: "SELECT COUNT(*) FROM pending_tasks WHERE status = 'pending'"
    targetQueryValue: "50"
    activationTargetQueryValue: "1"
  authenticationRef:
    name: postgres-credentials

metadata:
  value: "2500"              # 1 replica per 2,500 messages
  activationValue: "10"      # With fewer than 10 messages, it counts as inactive

# 1. The Secret with the credentials (module 3 and 08-05)
apiVersion: v1
kind: Secret
metadata:
  name: rabbitmq-credentials
  namespace: rutas-norte-pro
type: Opaque
stringData:
  # Full connection string with user and password.
  host: "amqp://keda_reader:[email protected]:5672/"
---
# 2. The TriggerAuthentication that references it
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
  name: rabbitmq-credentials
  namespace: rutas-norte-pro
spec:
  secretTargetRef:
    - parameter: host          # Name of the parameter the scaler expects
      name: rabbitmq-credentials
      key: host                # Key inside the Secret

triggers:
  - type: rabbitmq
    metadata:
      queueName: confirmation-notifications
      mode: QueueLength
      value: "2500"
    authenticationRef:
      name: rabbitmq-credentials

apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
  name: aws-sqs-identity
  namespace: rutas-norte-pro
spec:
  podIdentity:
    provider: aws
    roleArn: arn:aws:iam::000000000000:role/rutas-norte-keda-sqs

apiVersion: keda.sh/v1alpha1
kind: ClusterTriggerAuthentication
metadata:
  name: prometheus-read           # No namespace
spec:
  secretTargetRef:
    - parameter: bearerToken
      name: prometheus-token
      key: token

apiVersion: keda.sh/v1alpha1
kind: ScaledJob
metadata:
  name: invoice-processor
  namespace: rutas-norte-pro
spec:
  jobTargetRef:
    parallelism: 1
    completions: 1
    backoffLimit: 3
    template:
      spec:
        restartPolicy: Never
        containers:
          - name: processor
            image: registry.rutasnorte.example/invoice-processor:2.1.0
            resources:
              requests: {cpu: 500m, memory: 512Mi}
              limits: {cpu: "2", memory: 2Gi}

  pollingInterval: 30
  maxReplicaCount: 20
  successfulJobsHistoryLimit: 5
  failedJobsHistoryLimit: 10

  # How to work out how many Jobs to create from the metric value.
  scalingStrategy:
    strategy: "default"

  triggers:
    - type: rabbitmq
      metadata:
        queueName: pending-invoices
        mode: QueueLength
        value: "1"          # ONE Job per message
      authenticationRef:
        name: rabbitmq-credentials

# k8s/environments/pro/scaledobject-notifications-worker.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: notifications-worker
  namespace: rutas-norte-pro
  labels:
    app: notifications-worker
    app.kubernetes.io/part-of: rutas-norte
    environment: pro
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: notifications-worker

  # Query RabbitMQ every 15 s. Fast enough that reactivation from zero is
  # almost immediate, and slow enough not to hammer the RabbitMQ management
  # API.
  pollingInterval: 15

  # 10 minutes with no activity before switching off entirely. It is generous
  # on purpose: bookings arrive in bursts, and switching off and on every two
  # minutes would cost more (in cold starts) than keeping one pod.
  cooldownPeriod: 600

  # SCALE-TO-ZERO. Out of season there is no reason to keep workers running
  # 24 hours a day waiting for one email every few minutes.
  minReplicaCount: 0

  # Ceiling: 25 workers x 2 emails/s = 50 emails/s = 3,000 emails/minute.
  # The bank holiday's 40,000 emails drain in ~14 minutes. Enough.
  # Also: 25 x 120m = 3 cores, which fits the capacity plan of 09-03.
  maxReplicaCount: 25

  # If RabbitMQ stops responding, we do not freeze: we pin 6 replicas, which
  # is a normal day's throughput with margin. Better a known safe number than
  # an unknown frozen one.
  fallback:
    failureThreshold: 3
    replicas: 6

  advanced:
    # On deleting the ScaledObject, return the Deployment to its original replicas.
    restoreToOriginalReplicaCount: true
    horizontalPodAutoscalerConfig:
      name: hpa-notifications-worker
      behavior:
        scaleUp:
          # No wait: if there is a queue, there is work pending NOW.
          stabilizationWindowSeconds: 0
          selectPolicy: Max
          policies:
            - type: Percent
              value: 100
              periodSeconds: 30
            - type: Pods
              value: 5
              periodSeconds: 30
        scaleDown:
          # 5 minutes of calm. Less conservative than bookings-api because
          # starting a worker is cheap (it has no cache to warm) and because
          # there is no user waiting for a response.
          stabilizationWindowSeconds: 300
          selectPolicy: Min
          policies:
            - type: Percent
              value: 30
              periodSeconds: 60

  triggers:
    # PRIMARY TRIGGER: the queue length.
    - type: rabbitmq
      metadata:
        protocol: amqp
        queueName: confirmation-notifications
        mode: QueueLength
        # 1 worker per 2,500 messages in the queue.
        #
        # Calculation: we want to drain 40,000 messages in ~15 minutes.
        #   40,000 / 900 s = 44.4 messages/s needed
        #   44.4 / 2 (per worker) = 22.2 workers
        #   40,000 / 22.2 = 1,800 messages per worker
        # We round to 2,500 so as not to hit the ceiling with a tight margin:
        #   40,000 / 2,500 = 16 workers -> drains in ~21 minutes. Acceptable.
        value: "2500"
        # Below 10 messages, it counts as inactive and may drop to zero.
        # We do not use 0 because there are always 1-2 residual messages in flight.
        activationValue: "10"
      authenticationRef:
        name: rabbitmq-credentials

    # SECONDARY TRIGGER: a CPU safety net.
    # If for some unforeseen reason the workers saturate on CPU (a pathological
    # email template, say) we scale anyway, even with a short queue. The
    # trigger asking for MORE replicas wins.
    - type: cpu
      metricType: Utilization
      metadata:
        value: "80"

kubectl apply -f k8s/environments/pro/scaledobject-notifications-worker.yaml
kubectl get scaledobject -n rutas-norte-pro

NAME                   SCALETARGETKIND      SCALETARGETNAME        MIN  MAX  TRIGGERS   AUTHENTICATION        READY  ACTIVE  FALLBACK  AGE
notifications-worker   apps/v1.Deployment   notifications-worker   0    25   rabbitmq   rabbitmq-credentials  True   False   False     34s

kubectl get deployment notifications-worker -n rutas-norte-pro

NAME                   READY   UP-TO-DATE   AVAILABLE   AGE
notifications-worker   0/0     0            0           89d

kubectl get deployment notifications-worker -n rutas-norte-pro --watch

# Simulate 30,000 confirmation emails (the sale opening)
kubectl exec -n rutas-norte-pro rabbitmq-0 -- \
  rabbitmqadmin publish routing_key=confirmation-notifications \
  payload='{"booking":"RN-2026-054321","recipient":"[email protected]"}' \
  --count=30000

NAME                   READY   UP-TO-DATE   AVAILABLE   AGE
notifications-worker   0/0     0            0           89d
notifications-worker   0/1     0            0           89d     <- KEDA activates: 0 -> 1
notifications-worker   1/1     1            1           89d
notifications-worker   1/6     1            1           89d     <- The HPA takes over
notifications-worker   6/6     6            6           89d
notifications-worker   6/12    6            6           89d
notifications-worker   12/12   12           12          89d
notifications-worker   12/12   12           12          89d     <- Stable: 30000/2500 = 12

kubectl get hpa hpa-notifications-worker -n rutas-norte-pro

NAME                       REFERENCE                         TARGETS            MINPODS  MAXPODS  REPLICAS
hpa-notifications-worker   Deployment/notifications-worker   2483/2500 (avg)    1        25       12

notifications-worker   12/12   12   12   89d
notifications-worker   12/8    12   12   89d     <- HPA scales down (after stabilisation)
notifications-worker   8/3     8    8    89d
notifications-worker   3/1     3    3    89d
notifications-worker   1/1     1    1    89d     <- It stays at 1 (the HPA minimum)
                                                    ...10 minutes of cooldownPeriod pass...
notifications-worker   0/0     0    0    89d     <- KEDA switches off: 1 -> 0

# HELP rutasnorte_requests_total Total HTTP requests served
# TYPE rutasnorte_requests_total counter
rutasnorte_requests_total{method="GET",route="/routes",code="200"} 1847293
rutasnorte_requests_total{method="POST",route="/bookings",code="201"} 92841

# HELP rutasnorte_request_duration_seconds Request duration
# TYPE rutasnorte_request_duration_seconds histogram
rutasnorte_request_duration_seconds_bucket{route="/bookings",le="0.1"} 78201
rutasnorte_request_duration_seconds_bucket{route="/bookings",le="0.5"} 91043
...

# HELP rutasnorte_bookings_confirmed_total Confirmed bookings
# TYPE rutasnorte_bookings_confirmed_total counter
rutasnorte_bookings_confirmed_total 92841

# k8s/environments/pro/scaledobject-bookings-api.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: bookings-api
  namespace: rutas-norte-pro
  labels:
    app: bookings-api
    app.kubernetes.io/part-of: rutas-norte
    environment: pro
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: bookings-api

  pollingInterval: 15
  cooldownPeriod: 300

  # We do NOT scale to zero. bookings-api is the API that carries the sale: a
  # cold start in front of a user who wants to buy is unacceptable.
  # 4 replicas is the agreed availability floor (09-01, and 09-05).
  minReplicaCount: 4
  maxReplicaCount: 30

  fallback:
    failureThreshold: 3
    # If Prometheus goes down, we pin 12 replicas: triple the minimum. It is a
    # conservative number that comfortably handles a normal day and gives us
    # time to fix Prometheus with no risk of an outage.
    replicas: 12

  advanced:
    restoreToOriginalReplicaCount: true
    horizontalPodAutoscalerConfig:
      name: hpa-bookings-api
      behavior:
        # The SAME asymmetric behavior as 09-01: up in zero seconds,
        # down after ten minutes of sustained calm.
        scaleUp:
          stabilizationWindowSeconds: 0
          selectPolicy: Max
          policies:
            - type: Percent
              value: 100
              periodSeconds: 30
            - type: Pods
              value: 6
              periodSeconds: 30
        scaleDown:
          stabilizationWindowSeconds: 600
          selectPolicy: Min
          policies:
            - type: Percent
              value: 20
              periodSeconds: 120
            - type: Pods
              value: 3
              periodSeconds: 120

  triggers:
    # PRIMARY TRIGGER: requests per second, measured in Prometheus.
    - type: prometheus
      metadata:
        serverAddress: http://prometheus-operated.monitoring.svc.cluster.local:9090
        # Name the metric will appear under in the generated HPA.
        metricName: bookings_api_requests_per_second
        query: |
          sum(
            rate(
              rutasnorte_requests_total{
                namespace="rutas-norte-pro",
                app="bookings-api"
              }[2m]
            )
          )
        # 85 requests per second and replica.
        #
        # Calculation (from the load test of 09-06): one bookings-api replica
        # with 412m of CPU sustains ~120 rps before the p95 latency exceeds
        # 300 ms. We leave a 30% margin to absorb the start-up time of the new
        # replicas: 120 x 0.7 = 84 -> 85 rps.
        threshold: "85"
        activationThreshold: "5"
        ignoreNullValues: "true"
      authenticationRef:
        kind: ClusterTriggerAuthentication
        name: prometheus-read

    # SAFETY NET 1: CPU. If for whatever reason the requests are much more
    # expensive than usual (a pathological query, an abusive client), CPU will
    # detect it even if the request count is normal.
    - type: cpu
      metricType: Utilization
      metadata:
        value: "70"

sum(
  rate(
    rutasnorte_requests_total{
      namespace="rutas-norte-pro",
      app="bookings-api"
    }[2m]
  )
)

kubectl get hpa hpa-bookings-api -n rutas-norte-pro

NAME               REFERENCE                 TARGETS                   MINPODS  MAXPODS  REPLICAS
hpa-bookings-api   Deployment/bookings-api   1247/85 (avg), 34%/70%    4        30       15

Total value of the query: 1,247 rps (all replicas together)
Current replicas: 15
Value per replica: 1,247 / 15 = 83.1 rps
Target: 85 rps

ratio = 83.1 / 85 = 0.978  ->  inside the tolerance band. Stable.

# 1. Delete the manual HPA
kubectl delete hpa bookings-api -n rutas-norte-pro

# 2. Create the ScaledObject
kubectl apply -f k8s/environments/pro/scaledobject-bookings-api.yaml

# 3. Verify that KEDA has created its own
kubectl get hpa -n rutas-norte-pro

Error from server: admission webhook "vscaledobject.kb.io" denied the request:
the workload 'bookings-api' of type 'apps/v1.Deployment' is already managed by
the hpa 'bookings-api'

09:59:50  4 bookings-api replicas. Normal traffic: 180 rps.
10:00:00  The sale opens. Traffic jumps to 2,400 rps in 40 seconds.
10:00:15  Prometheus scrapes. The rate[2m] still reflects the average of the
          last 2 minutes, which includes 1m50s of calm: ~400 rps.
10:00:15  KEDA/HPA: 400/4 = 100 rps per replica. Ratio 1.18. Asks for 5 replicas.
          <- FAR BELOW what is needed. The rate[2m] SMOOTHS the spike.
10:00:45  The rate now reflects reality better: ~1,500 rps. Asks for 18 replicas.
10:01:15  The first new pods are Ready.
10:02:30  The 28 replicas needed are reached.

DEGRADATION: two and a half minutes at the most valuable moment of the year.

# k8s/environments/pro/scaledobject-bookings-api.yaml (added fragment)
  triggers:
    - type: prometheus
      # ... (the primary trigger above, unchanged)

    - type: cpu
      # ... (the safety net, unchanged)

    # PRE-WARMING TRIGGER: the May bank holiday.
    #
    # The sale opens at 10:00 on 25 April. By 9:30 we already want
    # 25 replicas BOOTED AND WARM: PostgreSQL connection pool
    # established, route cache populated, Node.js JIT warmed up.
    #
    # Remember the HPA rule: THE TRIGGER ASKING FOR THE MOST REPLICAS WINS.
    # Between 9:30 and 14:00, the floor is 25 replicas. If real traffic
    # asks for more, the Prometheus trigger takes it up to 30.
    - type: cron
      metadata:
        timezone: Europe/Madrid
        start: "30 9 25 4 *"        # 25 April at 9:30
        end: "0 14 25 4 *"          # 25 April at 14:00
        desiredReplicas: "25"

    # Daily pre-warming of the busiest sales window.
    # Monday to Sunday, from 9:30 to 13:00, a floor of 10 replicas.
    - type: cron
      metadata:
        timezone: Europe/Madrid
        start: "30 9 * * *"
        end: "0 13 * * *"
        desiredReplicas: "10"

    # Weekend reinforcement: Friday afternoon and Saturday morning concentrate
    # the short-break bookings.
    - type: cron
      metadata:
        timezone: Europe/Madrid
        start: "0 16 * * 5"         # Friday at 16:00
        end: "0 22 * * 6"           # Saturday at 22:00
        desiredReplicas: "15"

# k8s/environments/pro/scaledobject-capacity-filler.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: capacity-filler
  namespace: rutas-norte-pro
spec:
  scaleTargetRef:
    name: capacity-filler
  minReplicaCount: 2
  maxReplicaCount: 10
  cooldownPeriod: 300
  triggers:
    # Expand the cushion ONE HOUR BEFORE the bookings-api replicas.
    # That way the Cluster Autoscaler has time to boot the nodes, and at
    # 9:30, when the bookings-api cron asks for 25 replicas, there is room.
    - type: cron
      metadata:
        timezone: Europe/Madrid
        start: "30 8 25 4 *"        # 8:30, one hour earlier
        end: "0 15 25 4 *"
        desiredReplicas: "10"

08:30  The cron expands the filler from 2 to 10 pods.
       8 filler pods go to Pending.
08:31  The Cluster Autoscaler detects the pending pods, asks for 3 nodes.
08:35  The nodes are Ready. The filler is placed.
       CAPACITY READY AND PAID FOR, but occupied by pods that do nothing.
09:30  The bookings-api cron asks for 25 replicas.
       The scheduler EVICTS the filler pods (priority -10).
09:30  The 25 replicas are placed in SECONDS. No waiting for nodes.
09:31  The filler pods go to Pending -> the CA boots more nodes ->
       the cushion regenerates for whatever comes next.
09:32  25 replicas Ready and warm.
10:00  OPENING. Zero degradation.

t=0     A message arrives on the queue.
t=0-15  KEDA detects it on its next cycle (pollingInterval).
t=15    KEDA scales the Deployment from 0 to 1.
t=16    The scheduler places the pod. If there is NO room -> +3 min (09-03).
t=17    The kubelet pulls the image. If cached, ~2 s; if not, 20-60 s.
t=20    The container starts. Node.js: ~15 s until it is ready.
t=35    Connections to PostgreSQL and RabbitMQ established.
t=36    The first message is processed.

TOTAL DELAY: between 35 seconds and 4 minutes.

# k8s/environments/dev/scaledobject-nightly-shutdown.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: bookings-api-schedule
  namespace: rutas-norte-dev
  labels:
    app: bookings-api
    environment: dev
spec:
  scaleTargetRef:
    name: bookings-api
  # Outside working hours: ZERO. The environment switches itself off.
  minReplicaCount: 0
  maxReplicaCount: 3
  cooldownPeriod: 300
  triggers:
    # Monday to Friday, from 7:45 to 19:30: 1 replica running.
    # 7:45 so that it is warm when the team arrives at 8:00.
    - type: cron
      metadata:
        timezone: Europe/Madrid
        start: "45 7 * * 1-5"
        end: "30 19 * * 1-5"
        desiredReplicas: "1"

spec:
  # When there IS activity: at least 4 replicas.
  minReplicaCount: 4
  # When there is NO activity: 1 replica running.
  idleReplicaCount: 1

kubectl get scaledobject -n rutas-norte-pro

NAME                   SCALETARGETKIND      SCALETARGETNAME        MIN  MAX  READY  ACTIVE  FALLBACK  AGE
notifications-worker   apps/v1.Deployment   notifications-worker   0    25   False  False   False     4m

kubectl describe scaledobject notifications-worker -n rutas-norte-pro

Status:
  Conditions:
    Type:     Ready
    Status:   False
    Reason:   ScalerFailed
    Message:  error creating scaler for trigger 0: error parsing rabbitmq metadata:
              no host setting given
Events:
  Type     Reason         Age   From           Message
  ----     ------         ----  ----           -------
  Warning  ScalerFailed   4m    keda-operator  error creating scaler for trigger 0

kubectl describe hpa keda-hpa-notifications-worker -n rutas-norte-pro

Conditions:
  Type           Status  Reason                   Message
  ----           ------  ------                   -------
  AbleToScale    True    SucceededGetScale        the HPA controller was able to get the target's current scale
  ScalingActive  False   FailedGetExternalMetric  unable to get external metric
                         rutas-norte-pro/s0-rabbitmq-confirmation-notifications/&LabelSelector{...}:
                         unable to fetch metrics from external metrics API:
                         rpc error: code = Unknown desc = error inspecting rabbitMQ:
                         Object Not Found

kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/rutas-norte-pro/s0-rabbitmq-confirmation-notifications?labelSelector=scaledobject.keda.sh%2Fname%3Dnotifications-worker" \
  | python3 -m json.tool

{
    "kind": "ExternalMetricValueList",
    "apiVersion": "external.metrics.k8s.io/v1beta1",
    "items": [
        {
            "metricName": "s0-rabbitmq-confirmation-notifications",
            "metricLabels": null,
            "timestamp": "2026-05-01T10:14:32Z",
            "value": "29800"
        }
    ]
}

kubectl logs -n keda deployment/keda-operator --tail=100 | grep -i "notifications-worker"

ERROR  scale_handler  error getting scale decision  {"scaledObject.Namespace": "rutas-norte-pro",
       "scaledObject.Name": "notifications-worker",
       "error": "dial tcp 10.96.42.17:5672: connect: connection refused"}

kubectl logs -n keda deployment/keda-operator-metrics-apiserver --tail=50

# Can KEDA reach RabbitMQ?
kubectl run connectivity-test --rm -it --restart=Never \
  -n keda --image=busybox:1.36 -- \
  wget -qO- --timeout=5 http://rabbitmq.rutas-norte-pro.svc.cluster.local:15672/api/overview

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-keda-to-rabbitmq
  namespace: rutas-norte-pro
spec:
  podSelector:
    matchLabels:
      app: rabbitmq
  policyTypes: [Ingress]
  ingress:
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: keda
      ports:
        - protocol: TCP
          port: 5672
        - protocol: TCP
          port: 15672

kubectl port-forward -n monitoring svc/prometheus-operated 9090:9090

# Scaler errors: if this rises, KEDA cannot query the source
sum(rate(keda_scaler_errors_total[5m])) by (scaledObject, scaler) > 0

# The value of the metric KEDA is reporting
keda_scaler_metrics_value{scaledObject="notifications-worker"}

# ScaledObjects in fallback: a sign that a source is down
keda_scaled_object_errors_total > 0

kubectl get scaledobject -n rutas-norte-pro

NAME                   SCALETARGETKIND      MIN  MAX  READY  ACTIVE  FALLBACK  AGE
notifications-worker   apps/v1.Deployment   0    25   True   True    True      3h

kubectl describe hpa keda-hpa-notifications-worker -n rutas-norte-pro

Conditions:
  Type            Status  Reason              Message
  ----            ------  ------              -------
  AbleToScale     True    ReadyForNewScale    recommended size matches current size
  ScalingActive   True    ValidMetricFound    the HPA was able to successfully calculate a replica count
  ScalingLimited  False   DesiredWithinRange  the desired count is within the acceptable range
Metrics:
  ( current / target )
  "s0-rabbitmq-confirmation-notifications" (target average value):  6 / 2500
Events: <none>

kubectl logs -n keda deployment/keda-operator --tail=20 | grep worker

ERROR  scale_handler  error getting metric for scaler  {"scaledObject.Name": "notifications-worker",
       "scaler": "rabbitmqScaler",
       "error": "Get \"https://rabbitmq.rutas-norte-pro.svc.cluster.local:15671/api/queues/%2F/confirmation-notifications\":
       dial tcp 10.96.42.17:15671: i/o timeout"}

Each invoice: 8 seconds, processed one at a time.
Invoices per second per worker = 1 / 8 = 0.125 invoices/s

12,000 invoices in 4 hours = 12,000 / 14,400 s = 0.833 invoices/s of input.

If the system processed exactly at the input rate, the queue would not grow
but nor would it drain, and one invoice would wait... it depends on the
accumulated backlog. We need to process FASTER than the input.

Approach via the 30-minute goal:
  In 30 minutes (1,800 s) 0.833 x 1,800 = 1,500 invoices arrive.
  For none to wait more than 30 min, we must process 1,500 invoices
  in less than 1,800 s: capacity >= 1,500 / 1,800 = 0.833 invoices/s.

With a 100% margin (to absorb bursts, since input is not uniform):
  Target capacity: 1.67 invoices/s

Workers = 1.67 / 0.125 = 13.3  ->  14 workers

First hour: 6,000 invoices = 1.67 invoices/s of input.
With 14 workers: 14 x 0.125 = 1.75 invoices/s of capacity.
Capacity > input: the queue does not grow in a sustained way. Correct.

Estimated maximum backlog: ~500 invoices.
Time to drain those 500: 500 / 1.75 = 286 s = 4.8 minutes.
WELL below the 30 minutes. Plenty of room.

14 workers x 250m = 3.5 cores
Free quota: 8 cores.

They fit. And there is room to raise maxReplicaCount to 25:
  25 x 250m = 6.25 cores < 8. That fits too.

Memory: 14 x 512Mi = 7 GiB. No problem.

At the peak, how many messages are in the queue?

The estimated maximum backlog is ~500 invoices (computed above). But if
scaling is slow to start, the initial backlog is larger: in the first
2 minutes with just 1 replica, 1.67 x 120 = 200 invoices arrive
and 0.125 x 120 = 15 are processed. Backlog: ~185.

We want ~500 messages in the queue to give 14 workers:
  threshold = 500 / 14 = 35.7  ->  round to 35

Check with other queue values:
  100 messages -> ceil(100/35) = 3 workers
  300 messages -> ceil(300/35) = 9 workers
  500 messages -> ceil(500/35) = 15 workers  (goal reached)
  700 messages -> ceil(700/35) = 20 workers  (ceiling)

Task duration: 8 seconds.
Pod start-up time: ~15 seconds.

8 < 15: the task takes LESS than the start-up.
-> ScaledObject. A ScaledJob would spend more time starting than working.

# k8s/environments/pro/scaledobject-invoices-worker.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: invoices-worker
  namespace: rutas-norte-pro
  labels:
    app: invoices-worker
    app.kubernetes.io/part-of: rutas-norte
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: invoices-worker

  pollingInterval: 20

  # 15 minutes with no activity before switching off. Generous because the
  # invoices arrive in bursts after the sale closes, and switching off between
  # bursts would cost more in cold starts than keeping one pod.
  cooldownPeriod: 900

  # SCALE-TO-ZERO. Nobody waits for an invoice in real time: the goal is
  # 30 minutes, and the cold start is 35 seconds. Plenty of margin.
  # Out of season only a few invoices a day are generated.
  minReplicaCount: 0

  # 20 workers = 5 cores, within the 8 free in the quota.
  # 20 x 0.125 = 2.5 invoices/s = 9,000 invoices/hour. Well above
  # the 3,000/hour of the bank-holiday peak.
  maxReplicaCount: 20

  fallback:
    failureThreshold: 3
    # If the queue store does not respond, 5 workers: enough for the
    # usual throughput while it is investigated.
    replicas: 5

  advanced:
    restoreToOriginalReplicaCount: true
    horizontalPodAutoscalerConfig:
      name: hpa-invoices-worker
      behavior:
        scaleUp:
          # No wait: if there is a queue, there is work pending.
          stabilizationWindowSeconds: 0
          selectPolicy: Max
          policies:
            - type: Percent
              value: 100
              periodSeconds: 30
            - type: Pods
              value: 5
              periodSeconds: 30
        scaleDown:
          # 10 minutes: the invoices arrive in bursts after each sale
          # closes, and we do not want to destroy workers between bursts.
          stabilizationWindowSeconds: 600
          selectPolicy: Min
          policies:
            - type: Percent
              value: 30
              periodSeconds: 120

  triggers:
    - type: rabbitmq
      metadata:
        protocol: amqp
        queueName: pending-invoices
        mode: QueueLength
        # 1 worker per 35 invoices in the queue.
        #
        # Calculation: a worker processes 1 invoice every 8 s = 0.125 invoices/s.
        # Input at the bank-holiday peak is 0.833 invoices/s (12,000 in 4 h).
        # With a 100% margin for bursts: target capacity 1.67 invoices/s.
        # Workers needed: 1.67 / 0.125 = 14.
        # Expected queue backlog at the peak: ~500 messages.
        # threshold = 500 / 14 = 35.
        #
        # Business goal: no invoice waits more than 30 minutes.
        # With 14 workers, draining 500 invoices takes ~5 minutes.
        value: "35"
        # Below 3 invoices, it counts as inactive. It allows
        # scale-to-zero without 1-2 residual messages preventing it.
        activationValue: "3"
      authenticationRef:
        name: rabbitmq-credentials

    # Safety net: if the object store slows down and the invoices take
    # 30 s instead of 8, the queue will grow and the primary trigger will
    # detect it. But if the problem is CPU (a pathological PDF), this
    # trigger covers it.
    - type: cpu
      metricType: Utilization
      metadata:
        value: "80"

Get "https://rabbitmq.rutas-norte-pro.svc.cluster.local:15671/api/queues/..."
dial tcp 10.96.42.17:15671: i/o timeout

KEDA wants the HPA to compute 6 replicas.
The HPA computes: ceil(currentReplicas x value / threshold)

But since the HPA uses AverageValue, KEDA's fallback mechanism works by
reporting a value such that the result is the fallback replicas.

In the output we see "6 / 2500", which is the reported fallback value.
The HPA with 1 replica computes ceil(1 x 6/2500) = ceil(0.0024) = 1.

# 1. Does the Service exist and respond?
kubectl get svc rabbitmq -n rutas-norte-pro
kubectl get endpoints rabbitmq -n rutas-norte-pro

# 2. Is there a new NetworkPolicy?
kubectl get networkpolicy -n rutas-norte-pro
kubectl describe networkpolicy -n rutas-norte-pro

# 3. Test connectivity FROM the KEDA namespace
kubectl run test --rm -it --restart=Never -n keda --image=busybox:1.36 -- \
  wget -qO- --timeout=5 http://rabbitmq.rutas-norte-pro.svc.cluster.local:15672/api/overview

# k8s/environments/pro/networkpolicy-keda-rabbitmq.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-keda-to-rabbitmq
  namespace: rutas-norte-pro
spec:
  podSelector:
    matchLabels:
      app: rabbitmq
  policyTypes: [Ingress]
  ingress:
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: keda
      ports:
        - protocol: TCP
          port: 15672      # HTTP management API
        - protocol: TCP
          port: 5672       # AMQP

# If RabbitMQ does not have TLS configured on the management port, the
# TriggerAuthentication must point at http://...:15672, not https://...:15671
stringData:
  host: "amqp://keda_reader:[email protected]:5672/"

spec:
  # During high season, we do not scale to zero. The fallback then works
  # reliably and the cold start does not penalise us at the worst moment.
  minReplicaCount: 2
  idleReplicaCount: 1     # Outside activity, 1 replica running
  fallback:
    failureThreshold: 3
    replicas: 8

# 1. THE MOST IMPORTANT ONE: KEDA in fallback mode.
#    If this fires, scaling is NOT working even though everything looks normal.
keda_scaled_object_errors_total > 0

# 2. Scaler errors: KEDA cannot query the source.
sum(rate(keda_scaler_errors_total[5m])) by (scaledObject, scaler) > 0

# 3. A business alert: the queue grows without the workers growing.
#    This is the one that would have caught the problem in 2 minutes.
(
  rabbitmq_queue_messages{queue="confirmation-notifications"} > 5000
)
and
(
  kube_deployment_status_replicas{deployment="notifications-worker"} < 5
)

# 4. Age of the oldest message in the queue: the real business indicator.
rabbitmq_queue_head_message_timestamp_seconds > 600

pricing-engine         -> Deployment + ScaledObject (HTTP API)
price-recalculation    -> CronJob (nightly process)

# k8s/environments/pro/scaledobject-pricing-engine.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: pricing-engine
  namespace: rutas-norte-pro
  labels:
    app: pricing-engine
    app.kubernetes.io/part-of: rutas-norte
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: pricing-engine

  pollingInterval: 15
  cooldownPeriod: 300

  # We do NOT scale to zero. It sits on the critical path of every route
  # search: its latency feeds directly into bookings-api's. A 35 s cold start
  # with users waiting is unacceptable, and it would also exhaust the outbound
  # connection pool of bookings-api: a cascading degradation.
  #
  # A floor of 3 replicas: 3 x 180 = 540 rps, well above the minimum of 50.
  # The floor is set by AVAILABILITY (09-05), not by capacity.
  minReplicaCount: 3

  # Ceiling: 2,500 rps / 180 rps per replica = 13.9 -> 14.
  # With a 40% margin: 20 replicas.
  maxReplicaCount: 20

  fallback:
    failureThreshold: 3
    # If Prometheus goes down, 8 replicas = 1,440 rps of capacity. It covers
    # normal traffic comfortably while it is fixed.
    replicas: 8

  advanced:
    restoreToOriginalReplicaCount: true
    horizontalPodAutoscalerConfig:
      name: hpa-pricing-engine
      behavior:
        scaleUp:
          # No wait. It is an internal service with critical latency: any
          # delay propagates to the end user's route search.
          stabilizationWindowSeconds: 0
          selectPolicy: Max
          policies:
            - type: Percent
              value: 100
              periodSeconds: 30
            - type: Pods
              value: 5
              periodSeconds: 30
        scaleDown:
          # 10 minutes. Search traffic is very irregular (bursts of
          # users comparing routes) and we do not want to destroy capacity
          # between bursts.
          stabilizationWindowSeconds: 600
          selectPolicy: Min
          policies:
            - type: Percent
              value: 20
              periodSeconds: 120

  triggers:
    # PRIMARY TRIGGER: requests per second.
    - type: prometheus
      metadata:
        serverAddress: http://prometheus-operated.monitoring.svc.cluster.local:9090
        metricName: pricing_engine_requests_per_second
        query: |
          sum(
            rate(
              pricing_engine_requests_total{
                namespace="rutas-norte-pro",
                app="pricing-engine"
              }[2m]
            )
          )
        # 125 rps per replica.
        #
        # Calculation: one replica sustains 180 rps before the p95 latency
        # exceeds 80 ms. We apply a 30% margin to cover the start-up time
        # of the new replicas: 180 x 0.7 = 126 -> 125.
        #
        # With 2,500 rps: ceil(2500/125) = 20 replicas. Exactly maxReplicaCount.
        threshold: "125"
        activationThreshold: "10"
        ignoreNullValues: "true"
      authenticationRef:
        kind: ClusterTriggerAuthentication
        name: prometheus-read

    # SECONDARY TRIGGER: LATENCY, which is what the business cares about.
    #
    # If the p95 exceeds 60 ms (75% of the 80 ms target), we scale even
    # when requests per second are within normal range. This covers the
    # case of abnormally EXPENSIVE requests: routes with many combinations,
    # or a degraded dependency.
    #
    # IMPORTANT NOTE: scaling on latency is a double-edged sword. If the
    # latency rises for a cause that is NOT solved with more replicas (for
    # example, a saturated PostgreSQL), scaling makes things worse (09-01, the
    # real bottleneck). That is why the threshold is conservative and
    # maxReplicaCount is bounded.
    - type: prometheus
      metadata:
        serverAddress: http://prometheus-operated.monitoring.svc.cluster.local:9090
        metricName: pricing_engine_p95_latency
        query: |
          histogram_quantile(0.95,
            sum(rate(pricing_engine_duration_seconds_bucket{
              namespace="rutas-norte-pro"
            }[2m])) by (le)
          ) * 1000
        threshold: "60"
        activationThreshold: "1"
        ignoreNullValues: "true"
      authenticationRef:
        kind: ClusterTriggerAuthentication
        name: prometheus-read

    # SAFETY NET: CPU.
    - type: cpu
      metricType: Utilization
      metadata:
        value: "70"

# k8s/environments/pro/cronjob-price-recalculation.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: price-recalculation
  namespace: rutas-norte-pro
  labels:
    app: price-recalculation
    app.kubernetes.io/part-of: rutas-norte
spec:
  schedule: "0 3 * * *"
  timeZone: "Europe/Madrid"      # CronJob supports timeZone from 1.27
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 5
  jobTemplate:
    spec:
      backoffLimit: 2
      # If it has not finished in 2 hours, cut it off. Stops a hung process
      # from pinning a node (09-03) and overlapping with the morning traffic.
      activeDeadlineSeconds: 7200
      ttlSecondsAfterFinished: 86400
      template:
        metadata:
          labels:
            app: price-recalculation
          annotations:
            # Do NOT evict during the run: we would lose 45 minutes of
            # work. The Cluster Autoscaler honours this annotation (09-03).
            cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
        spec:
          restartPolicy: OnFailure
          containers:
            - name: recalculation
              image: registry.rutasnorte.example/pricing-engine:4.2.0
              command: ["node", "recalculate-matrix.js"]
              resources:
                requests:
                  cpu: "4"
                  memory: 4Gi
                limits:
                  cpu: "4"
                  memory: 4Gi
                  # requests == limits -> Guaranteed QoS. It runs at 3:00
                  # with the cluster empty: we take capacity from nobody and
                  # we guarantee it is not evicted under memory pressure.

# k8s/environments/pro/vpa-price-recalculation.yaml
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: price-recalculation
  namespace: rutas-norte-pro
spec:
  targetRef:
    apiVersion: batch/v1
    kind: CronJob
    name: price-recalculation
  updatePolicy:
    # Initial: every new run is born with the learned resources. We never
    # evict a Job halfway through.
    updateMode: "Initial"
  resourcePolicy:
    containerPolicies:
      - containerName: recalculation
        minAllowed:
          cpu: "2"
          memory: 2Gi
        maxAllowed:
          cpu: "8"
          memory: 8Gi
        controlledValues: RequestsAndLimits

# High latency WITH the replicas at maximum: the problem is NOT solved by scaling
(
  histogram_quantile(0.95, sum(rate(pricing_engine_duration_seconds_bucket[5m])) by (le)) * 1000 > 80
)
and
(
  kube_deployment_status_replicas{deployment="pricing-engine"} >= 20
)