Refactor alert rules and alertmanager configuration

codeperfectplus · codeperfectplus · commit 6dca004bee3a · 2024-09-28T07:43:17.000+05:30
diff --git a/prometheus_config/alert_rules.yml b/prometheus_config/alert_rules.yml
@@ -25,7 +25,7 @@ groups:
         annotations:
           description: Battery percentage is below 20%.
           summary: Instance {{ $labels.instance }} low battery. Current battery percentage is {{ $value }}%. Charge the device.
-        expr: avg by (instance) (battery_percentage) < 80
+        expr: avg by (instance) (battery_percentage) < 40
         for: 2m
         labels:
           severity: warning
diff --git a/prometheus_config/alertmanager.yml b/prometheus_config/alertmanager.yml
@@ -12,7 +12,7 @@ route:
     - receiver: 'systemguard-webhook'
 
 receivers:
-  
+
   - name: 'systemguard-webhook'
     webhook_configs:
       - send_resolved: true
diff --git a/src/scripts/initialization/init_alertmanager.sh b/src/scripts/initialization/init_alertmanager.sh
@@ -16,32 +16,17 @@ route:
   group_wait: 30s
   group_interval: 5m
   repeat_interval: 3h
-  receiver: 'slack-notifications'  # Default receiver
+  receiver: 'systemguard-webhook'
 
   routes:
-    - receiver: 'slack-notifications'
-      continue: true  # This allows the alert to also be sent to the next receiver
     - receiver: 'systemguard-webhook'
 
 receivers:
-  - name: 'slack-notifications'
-    slack_configs:
-      - send_resolved: true
-        api_url: '$SLACK_WEBHOOK_URL'
-        channel: '#general'
-        username: 'Alertmanager'
-        icon_emoji: ':warning:'
-        text: |
-          *Alert:* {{ .CommonLabels.alertname }}
-          *Instance:* {{ .CommonLabels.instance }}
-          *Severity:* {{ .CommonLabels.severity }}
-          *Description:* {{ .CommonAnnotations.description }}
-          *Summary:* {{ .CommonAnnotations.summary }}
 
   - name: 'systemguard-webhook'
     webhook_configs:
       - send_resolved: true
-        url: 'http://$IP_ADDRESS:5001/alerts'
+        url: 'http://$IP_ADDRESS:5050/alerts'
         max_alerts: 0  # Send all alerts in one webhook request
 
 EOL
diff --git a/src/scripts/start_prometheus.sh b/src/scripts/start_prometheus.sh
@@ -22,10 +22,14 @@ NETWORK_NAME="flask-prometheus-net"
 CONTAINER_NAME="prometheus"
 PROMETHEUS_IMAGE="prom/prometheus:v2.55.0-rc.0"
 PROMETHEUS_PORT="9090"
+# SCRIPT_DIR and PWD are different
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
 PROMETHEUS_CONFIG_DIR="$(pwd)/prometheus_config"
 ALERT_RULES_FILE="$PROMETHEUS_CONFIG_DIR/alert_rules.yml"
 PROMETHEUS_CONFIG_FILE="$PROMETHEUS_CONFIG_DIR/prometheus.yml"
 PROMETHEUS_DATA_DIR="/home/$USER_NAME/.database/prometheus"
+INIT_ALERT_MANAGER_SCRIPT="$SCRIPT_DIR/initialization/init_alertmanager.sh"
+ALERT_MANAGER_SCRIPT="$SCRIPT_DIR/start_alertmanager.sh"
 FLASK_APP_IP=$(hostname -I | cut -d' ' -f1)
 FLASK_APP_PORT="5050"
 SCRAPING_INTERVAL="10s"
@@ -35,6 +39,14 @@ job_name="localhost"
 prometheus_username="prometheus_admin"
 prometheus_password="prometheus_password"
 
+# call INIT_ALERT_MANAGER_SCRIPT
+echo "Initializing Alert Manager $INIT_ALERT_MANAGER_SCRIPT"
+bash $INIT_ALERT_MANAGER_SCRIPT
+
+# starting alert manager container
+echo "Starting Alert Manager $ALERT_MANAGER_SCRIPT"
+bash $ALERT_MANAGER_SCRIPT
+
 # Logging function for better readability
 log() {
     echo "[INFO] $1"
@@ -112,6 +124,7 @@ run_output=$(docker run -d \
   -v "$PROMETHEUS_DATA_DIR:/prometheus" \
   "$PROMETHEUS_IMAGE" 2>&1)
 
+
 # Check if Prometheus started successfully
 if [ $? -eq 0 ]; then
     log "Prometheus container $CONTAINER_NAME started successfully on port $PROMETHEUS_PORT."
