Setup Prometheus Alerts - allanrogerr/public GitHub Wiki

Define prometheus

cat << EOF > /tmp/prometheus.yml
global:
  scrape_interval: 15s
scrape_configs:
- job_name: minio-job
  bearer_token: <bearer_token>
  metrics_path: /minio/v2/metrics/cluster
  scheme: https
  static_configs:
  - targets: [play.min.io:9000]
alerting:
  alertmanagers:
  - static_configs:
    - targets: [<alertmanager-target>:9093]
rule_files:
  - /etc/prometheus/alert_rules.yml
EOF

Define alert

cat << EOF > /tmp/alert_rules.yml
groups:
- name: ExampleAlert
  rules:
  - alert: ClusterUsageOver19TiB
    expr: avg_over_time(minio_cluster_usage_total_bytes{job="minio-job"}[5m])/ 1024/ 1024/ 1024/ 1024 > 19
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "Cluster size in MinIO deployment is too high"
      description: "Monitor size of cluster"
EOF

Sample alert manager

cat << EOF > /tmp/config_alertmanager.yml
# The root route on which each incoming alert enters.
route:
  # The root route must not have any matchers as it is the entry point for
  # all alerts. It needs to have a receiver configured so alerts that do not
  # match any of the sub-routes are sent to someone.
  receiver: 'team-X-webhooks'
  group_wait: 10s
  group_interval: 1m
  routes:
  - receiver: team-X-webhooks
    repeat_interval: 1m
    continue: true
receivers:
- name: 'team-X-webhooks'
  webhook_configs:
  - url: <example-webhook-url>
EOF

Start alert manager

docker stop alertmanager
docker rm alertmanager
docker run -itd \
    --name alertmanager \
    -p 9093:9093 \
    -v /tmp/config_alertmanager.yml:/etc/alertmanager/config.yml \
    quay.io/prometheus/alertmanager --config.file=/etc/alertmanager/config.yml &> /dev/null

Start prometheus

docker stop prometheus
docker rm prometheus
docker run -itd \
    --name prometheus \
    -p 9090:9090 \
    -v /tmp/prometheus.yml:/etc/prometheus/prometheus.yml \
    -v /tmp/alert_rules.yml:/etc/prometheus/alert_rules.yml \
    prom/prometheus &> /dev/null
⚠️ **GitHub.com Fallback** ⚠️