Configuring Service Communication Proxy Alert using SCPAlertrules.yaml file

Note:

Default NameSpace is scpsvc for Service Communication Proxy. You can update the NameSpace as per the deployment.

Following is a sample yaml file.

apiVersion: v1
data:
  alertsscp: |
    groups:
    - name: SCPAlerts
      rules:
         #Alerts for SCP Ingress Traffic Rate, it uses namespace of scp deployed
      - alert: SCPIngressTrafficRateAboveMinorThreshold
        annotations:
          description: 'Ingress Traffic Rate at Locality: "{{$labels.ocscp_locality}}"  is above minor threshold (i.e. 800 mps)'
          summary: 'Current Ingress Traffic Rate is {{ $value | printf "%.2f" }} mps which is above 80 Percent of Max MPS(1000)'
         # Provide app and kubernetes_namespace of scp deployed
        expr: sum(rate(ocscp_metric_total_http_rx_req{app="scp-worker",kubernetes_namespace="scpsvc"}[2m])) by (kubernetes_namespace,ocscp_locality) >= 800 < 900
        labels:
          severity: Minor
      - alert: SCPIngressTrafficRateAboveMajorThreshold
        annotations:
          description: 'Ingress Traffic Rate at Locality: {{$labels.ocscp_locality}} and is above major threshold (i.e. 900 mps)'
          summary: 'Current Ingress Traffic Rate is {{ $value | printf "%.2f" }} mps which is above 90 Percent of Max MPS(1000)'
         # Provide app and kubernetes_namespace of scp deployed
        expr: sum(rate(ocscp_metric_total_http_rx_req{app="scp-worker",kubernetes_namespace="scpsvc"}[2m])) by (kubernetes_namespace,ocscp_locality)  >= 900 < 950
        labels:
          severity: Major
      - alert: SCPIngressTrafficRateAboveCriticalThreshold
        annotations:
          description: 'Ingress Traffic Rate at Locality: {{$labels.ocscp_locality}} and is above critical threshold (i.e. 950 mps)'
          summary: 'Current Ingress Traffic Rate is {{ $value | printf "%.2f" }} mps which is above 95 Percent of Max MPS(1000)'
         # Provide app and kubernetes_namespace of scp deployed
        expr: sum(rate(ocscp_metric_total_http_rx_req{app="scp-worker",kubernetes_namespace="scpsvc"}[2m])) by (kubernetes_namespace,ocscp_locality) >= 950
        labels:
          severity: Critical
      - alert: SCPRoutingFailedForServiceAlert
        annotations:
          description: 'Routing failed for service'
          summary: 'Routing failed for service: NFService Type = "{{$labels.ocscp_nf_service_type}}", NFType = "{{$labels.ocscp_nf_type}}", Locality = "{{$labels.ocscp_locality}}" and value = "{{ $value }}" '
         # Provide app and kubernetes_namespace of scp deployed
        expr: ocscp_metric_total_routing_send_fail{app="scp-worker",kubernetes_namespace="scpsvc"}
        labels:
          severity: Minor
      - alert: SCPSoothsayerPodMemoryUsage
         # Provide kubernetes_namespace of scp deployed and pod name substring as its regex match of pod name
        expr: sum(container_memory_usage_bytes{image!="",namespace="scpsvc",pod_name=~".+soothsayer.+"})  by (pod_name, namespace) > 8589934592
        for: 2m
        labels:
          severity: Warning
        annotations:
          summary: "NameSpace: {{$labels.namespace}}, Pod: {{$labels.pod_name}}: Soothsayer Pod High Memory usage detected"
          description: "Namespace: {{$labels.namespace}},Pod: {{$labels.pod_name}}: Memory usage is above 8 GB (current value is: {{ $value }})"
      - alert: SCPWorkerPodMemoryUsage
         # Provide kubernetes_namespace of scp deployed and pod name substring as its regex match of pod name
        expr: sum(container_memory_usage_bytes{image!="",namespace="scpsvc",pod_name=~".+worker.+"})  by (pod_name, namespace) > 4294967296
        for: 2m
        labels:
          severity: Warning
        annotations:
          summary: "NameSpace: {{$labels.namespace}}, Pod: {{$labels.pod_name}}: Worker Pod High Memory usage detected"
          description: "Namespace: {{$labels.namespace}},Pod: {{$labels.pod_name}}: Memory usage is above 4 GB (current value is: {{ $value }})"
      - alert: SCPPilotPodMemoryUsage
         # Provide kubernetes_namespace of scp deployed and pod name substring as its regex match of pod name
        expr: sum(container_memory_usage_bytes{image!="",namespace="scpsvc",pod_name=~".+pilot.+"})  by (pod_name, namespace)  > 6442450944
        for: 2m
        labels:
          severity: Warning
        annotations:
          summary: "NameSpace: {{$labels.namespace}}, Pod: {{$labels.pod_name}}: Pilot Pod High Memory usage detected"
          description: "Namespace: {{$labels.namespace}},Pod: {{$labels.pod_name}}: Memory usage is above 6 GB (current value is: {{ $value }})"