---
# yamllint disable rule:line-length
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  creationTimestamp: null
  labels:
    prometheus: example-prometheus
    role: alert-rules
  name: prometheus-example-rules
spec:
  groups:
    - name: tsb-example-rules
      rules:
        - alert: TSBNoMetrics
          labels:
            service: 'tsb'
            severity: 'critical'
          # TSB talks to its persistent backend even without constant external load.
          expr: |
            sum(rate(persistence_operation[15m]))
            == bool 0
          for: 30m
          annotations:
            summary: 'TSB metrics are absent'
            description: "TSB metrics collection is not working.\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#tsb-absent-metrics'

        - alert: TSBApiNoTraffic
          labels:
            service: 'tsb'
            severity: 'warning'
          expr: |
            sum(rate(grpc_server_handled_total{component="tsb",grpc_method!="SendAuditLog"}[1h]))
            == bool 0
          for: 30m
          annotations:
            summary: 'TSB API has no traffic'
            description:
              "TSB API has served no requests in the last hour, this may be abnormal.\n"

        - alert: TSBAPIAvailabilityLow
          labels:
            service: 'tsb'
            severity: 'critical'
            # divide the number of OK requests by the number of all requests and fire the alert if the ratio is <0.99 AND >0
          expr: |
            (
              sum(rate(grpc_server_handled_total{component="tsb", grpc_code="OK", grpc_type="unary", grpc_method!="SendAuditLog"}[1m])) by (grpc_method)
              /
              sum(rate(grpc_server_handled_total{component="tsb", grpc_type="unary", grpc_method!="SendAuditLog"}[1m])) by (grpc_method)
              < 0.99
            )
            AND
            (
              sum(rate(grpc_server_handled_total{component="tsb", grpc_code="OK", grpc_type="unary", grpc_method!="SendAuditLog"}[1m])) by (grpc_method)
              /
              sum(rate(grpc_server_handled_total{component="tsb", grpc_type="unary", grpc_method!="SendAuditLog"}[1m])) by (grpc_method)
              > 0
            )
          for: 10m
          annotations:
            summary: 'TSB gRPC API availability critically low'
            description:
              "TSB gRPC API availability is < 99% for the last 10 minutes for {{ $labels.grpc_method}} method. Current availability {{ $value }}\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#tsb-api-availability'

        - alert: FrontEnvoyAvailabilityLow
          labels:
            service: 'tsb'
            severity: 'critical'
          # Redirects are a regular part of OIDC flow, do not count 3xx as errors.
          # 403s may be also caused by expired authN tokens or incorrect
          # permissions, ignore them as well
          expr: |
            sum by(envoy_cluster_name, cluster) (rate(envoy_cluster_internal_upstream_rq{envoy_response_code=~"2.*|3.*|401|403"}[1m]))
            /
            sum by(envoy_cluster_name, namespace, cluster) (rate(envoy_cluster_internal_upstream_rq[1m]))
            < 0.99
          for: 5m
          annotations:
            summary: 'TSB Front Envoy availability critically low'
            description:
              "TSB Front Envoy availability < 99% for the last 5 minutes for {{ $labels.envoy_cluster_name }}. Current availability {{ $value }}\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#front-envoy-availability'

        - alert: FrontEnvoyP99LatencyHigh
          labels:
            service: 'tsb'
            severity: 'critical'
          # exclude telemetry latency, telemetry bulk writes naturally can have
          # higher latencies and it is not an issue.
          expr: |
            histogram_quantile(0.99, sum(rate(envoy_cluster_internal_upstream_rq_time_bucket{component="front-envoy", envoy_cluster_name!="elasticsearch"}[1m])) by (le, envoy_cluster_name))
            >
            3000
          for: 10m
          annotations:
            summary: 'TSB Front Envoy P99 latency is very high'
            description:
              "TSB Front Envoy P99 latency > 3s for the last 10 minutes for {{ $labels.envoy_cluster_name }} cluster. Current p99 latency is {{ $value }}\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#tsb-front-envoy-p99-latency'

        - alert: PostgresAvailabilityLow
          labels:
            service: 'tsb'
            severity: 'critical'
          # Explicitly exclude "resource not found" errors from availability calculations.
          # Some level of "not found" errors is OK and is an implementation detail of how NGAC works.
          # `OR on() vector(0)` means basically "cast NaN into 0"
          # (it doesn't make sense mathematically, but it's a good behaviour approximate explanation).
          # NaN may arise if `error!="", error!="resource not found"` returns
          # no metrics (all errors are the "missing resource" ones), which is
          # represented as NaN in Prometheus.
          # Any arithmetical operation with a NaN results in NaN, so we "cast"
          # it into 0 which is what we want semantically for the purposes of
          # alert evaluation.
          # Also, `1 - error_rate(and "cast" error_rate into 0 if it's NaN)` reads better that `success_rate OR 1 if success_rate=NaN`
          expr: |
            1 -
            (
              sum(rate(persistence_operation{error!="", error!="resource not found"}[1m]))
              /
              sum(rate(persistence_operation[1m])) OR on() vector(0)
            )
            < 0.99
          for: 5m
          annotations:
            summary: 'TSB Persistent Operations failing'
            description:
              "TSB Postgres persistent backend operations have a success rate lower than 99%. Current success rate {{ $value }}\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#persistent-backend-availability'

        - alert: PostgresTransactionsAvailabilityLow
          labels:
            service: 'tsb'
            severity: 'critical'
          expr: |
            ( sum(rate(persistence_transaction{error=""}[1m])) + sum(rate(persistence_transaction{error="old or duplicated event"}[1m])) )
            /
            sum(rate(persistence_transaction[1m]))
            < 0.99
          for: 5m
          annotations:
            summary: 'TSB Persistent Transactions failing'
            description:
              "TSB Postgres persistent backend transactions have a success rate lower than 99%. Current success rate {{ $value }}\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#persistent-backend-availability'

        - alert: PostgresMissingResourceHigh
          labels:
            service: 'tsb'
            severity: 'warning'
            # Some rate of 404-like responses is expected, a large rate MAY indicate an issue
            # However, it is not a CRITICAL alert.
          expr: |
            (
              (sum (rate(persistence_operation{error="resource not found",kind!="iam_revoked_token"}[1m]))) or on() vector(0)
              /
              (sum (rate(persistence_operation[1m])))
            )
            > 0.75
          for: 5m
          annotations:
            summary: 'TSB Persistent Operations High Rate of Resouce Missing'
            description: 'TSB Postgres persistent backend operations have a very high rate of queries ending in `resource missing`'
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#persistent-backend-availability'

        - alert: MPCPermanentStreamOperationErrorRateHigh
          labels:
            service: 'mpc'
            severity: 'critical'
          expr: |
            sum(rate(permanent_stream_operation{error!=""}[1m]))  by (name)
            /
            sum(rate(permanent_stream_operation[1m])) by (name)
            > 0.10
          for: 15m
          annotations:
            summary: 'MPC Config Updates Error Rate High'
            description:
              "MPC config updates have an error rate greater than 10% in the last 15 minutes for the operation {{ $labels.name }} \n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#mpc-component-health'

        - alert: MPCStreamConnectionsErrorRateHigh
          labels:
            service: 'mpc'
            severity: 'critical'
          expr: |
            sum(rate(permanent_stream_connection_attempts{error!=""}[1m])) by (name)
            /
            sum(rate(permanent_stream_connection_attempts[1m])) by (name)
            > 0.10
          for: 15m
          annotations:
            summary: 'MPC Config Updates Stream Connections Error Rate High'
            description:
              "MPC config updates stream connection attempts have an error rate greater than 10% in the last 15 minutes\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#mpc-component-health'

        - alert: MPCConfigUpdatesApplyErrorRateHigh
          labels:
            service: 'mpc'
            severity: 'critical'
          expr: |
            sum(rate(mpc_xcp_config_push_count{error!=""}[1m]))
            /
            sum(rate(mpc_xcp_config_push_count[1m]))
            > 0.10
          for: 15m
          annotations:
            summary: 'MPC Config Updates Apply Error Rate High'
            description:
              "MPC config updates push attempts have an error rate greater than 10% in the last 15 minutes\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#mpc-component-health'

        - alert: MPCXCPResourceConversionErrorRateHigh
          labels:
            service: 'mpc'
            severity: 'critical'
          expr: |
            sum(rate(mpc_xcp_conversion_count{error!=""}[1m]))
            /
            sum(rate(mpc_xcp_conversion_count[1m]))
            > 0.10
          for: 15m
          annotations:
            summary: 'MPC -> XCP Resource Conversion Error Rate High'
            description:
              "MPC -> XCP resource conversion attempts have an error rate greater than 10% in the last 15 minutes\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#mpc-component-health'

        - alert: MPCClusterUpdatesApplyErrorRateHigh
          labels:
            service: 'mpc'
            severity: 'critical'
          expr: |
            sum(rate(mpc_xcp_cluster_push_count{error!=""}[1m]))
            /
            sum(rate(mpc_xcp_cluster_push_count[1m]))
            > 0.10
          for: 15m
          annotations:
            summary: 'MPC Cluster Updates Apply Error Rate High'
            description:
              "MPC cluster updates push attempts have an error rate greater than 10% in the last 15 minutes\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#mpc-component-health'

        - alert: XCPCentralLastSyncTooLong
          labels:
            service: 'xcp'
            severity: 'warning'
          expr: |
            time() - min(xcp_central_last_config_propagation_event_timestamp_ms{edge!=""} / 1000) BY (edge, status)
            > 300
          for: 15m
          annotations:
            summary: 'Control Plane {{ $labels.edge }} last sync more than 5 minutes ago.'
            description:
              "Control Plane {{ $labels.edge }} last sync more than 5 minutes ago.\n"
            runbook: 'https://docs.tetrate.io/service-bridge/operations/telemetry/alerting_guidelines#last-management-plane-sync'
