feat(monitoring): add prod-monitor watchdog role
Adds roles/monitoring with monitor_prod.sh — silent cron watchdog that checks hermes_infra services, restart flapping, failed tasks, brencher liveness, disk and memory. Stdout empty when healthy; non-empty when any check fails (delivered as a single Telegram alert by cron no_agent job). Script is already running as cron job 'prod-monitor' (id 4ebfb853ea68) every 5m against docker_socket_proxy:2375. Role currently only ships the file; cron wiring on remote is the caller's responsibility until a follow-up adds a systemd/cron task.
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
---
|
||||
# defaults for monitoring role
|
||||
|
||||
# docker_socket_proxy endpoint used by monitor_prod.sh
|
||||
monitoring_proxy_url: "http://docker_socket_proxy:2375"
|
||||
|
||||
# Service list (without stack prefix) expected in the swarm stack
|
||||
monitoring_stack_prefix: "hermes_infra_"
|
||||
monitoring_expected_services:
|
||||
- squid
|
||||
- sarg
|
||||
- hermes-ui
|
||||
- hermes-gateway
|
||||
- docker_socket_proxy
|
||||
|
||||
# Alert thresholds
|
||||
monitoring_restart_threshold: 3
|
||||
monitoring_brencher_max_age_s: 600
|
||||
monitoring_disk_threshold_pct: 90
|
||||
monitoring_mem_min_available_mb: 512
|
||||
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env bash
|
||||
# prod-monitor: silent watchdog for hermes_infra + manager host.
|
||||
# stdout empty => nothing to report. non-empty => single Telegram alert.
|
||||
|
||||
set -u
|
||||
PROXY="${PROXY_URL:-http://docker_socket_proxy:2375}"
|
||||
CURL_MAX=3
|
||||
EXPECTED_SVCS=(squid sarg hermes-ui hermes-gateway docker_socket_proxy)
|
||||
STACK_PREFIX="hermes_infra_"
|
||||
RESTART_THRESH=3
|
||||
BRENCHER_MAX_AGE_S=600
|
||||
|
||||
PROBLEMS=()
|
||||
|
||||
# Tiny JSON helper (avoids jq dependency)
|
||||
jget() {
|
||||
# jget <json> <python-path-expr> -- e.g. jget "$x" 'x["ID"]' or 'x[0]["Spec"]["Name"]'
|
||||
local json="$1" expr="$2"
|
||||
python3 -c "
|
||||
import json,sys
|
||||
try:
|
||||
d=json.loads(sys.stdin.read())
|
||||
except Exception as e:
|
||||
sys.exit(0)
|
||||
v=$expr
|
||||
if isinstance(v,(list,tuple)):
|
||||
print('\n'.join(str(x) for x in v))
|
||||
elif v is None:
|
||||
pass
|
||||
else:
|
||||
print(v)
|
||||
" <<<"$json"
|
||||
}
|
||||
|
||||
# 1. Swarm reachable
|
||||
ping=$(curl -fsS -m "$CURL_MAX" "$PROXY/_ping" 2>/dev/null)
|
||||
[[ "$ping" == "OK" ]] || PROBLEMS+=("swarm_ping: response='${ping:-no_response}'")
|
||||
|
||||
# 2. hermes_infra services running
|
||||
names_filter=$(python3 -c "
|
||||
import json; names=['${STACK_PREFIX}${EXPECTED_SVCS[0]}','${STACK_PREFIX}${EXPECTED_SVCS[1]}','${STACK_PREFIX}${EXPECTED_SVCS[2]}','${STACK_PREFIX}${EXPECTED_SVCS[3]}','${STACK_PREFIX}${EXPECTED_SVCS[4]}']
|
||||
print(json.dumps({'name':names}))
|
||||
")
|
||||
services_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/services?filters=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.stdin.read()))" <<<"$names_filter")" 2>/dev/null)
|
||||
|
||||
declare -A SVC_ID
|
||||
for i in "${!EXPECTED_SVCS[@]}"; do
|
||||
svc="${EXPECTED_SVCS[$i]}"
|
||||
full="${STACK_PREFIX}${svc}"
|
||||
id=$(jget "$services_json" "[s for s in __import__('json').loads('''$services_json''') if s.get('Spec',{}).get('Name')=='$full'][0]['ID']" 2>/dev/null)
|
||||
# Simpler approach: re-loop in python
|
||||
id=$(python3 -c "
|
||||
import json,sys
|
||||
data=json.loads('''$services_json''')
|
||||
for s in data:
|
||||
if s.get('Spec',{}).get('Name')=='$full':
|
||||
print(s['ID'])
|
||||
break
|
||||
" 2>/dev/null)
|
||||
if [[ -z "$id" ]]; then
|
||||
PROBLEMS+=("service_missing: ${svc}")
|
||||
continue
|
||||
fi
|
||||
SVC_ID["$svc"]="$id"
|
||||
tasks_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'service':['$id']})))")" 2>/dev/null)
|
||||
running=$(python3 -c "
|
||||
import json
|
||||
d=json.loads('''$tasks_json''')
|
||||
print(sum(1 for t in d if t.get('Status',{}).get('State')=='running'))
|
||||
" 2>/dev/null)
|
||||
[[ "${running:-0}" -ge 1 ]] || PROBLEMS+=("service_no_running_task: ${svc}")
|
||||
done
|
||||
|
||||
# 3. Restart flapping
|
||||
flaps_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'desiredstate':['running']})))")" 2>/dev/null)
|
||||
flaps=$(python3 -c "
|
||||
import json
|
||||
d=json.loads('''$flaps_json''')
|
||||
out=[]
|
||||
for t in d:
|
||||
rc=t.get('Status',{}).get('ContainerStatus',{}).get('RestartCount',0)
|
||||
if rc>$RESTART_THRESH:
|
||||
name=t.get('Spec',{}).get('Name','?')
|
||||
out.append(f'{name}={rc}')
|
||||
print(';'.join(out))
|
||||
" 2>/dev/null)
|
||||
[[ -z "$flaps" ]] || PROBLEMS+=("flapping_tasks: ${flaps:0:200}")
|
||||
|
||||
# 4. Failed/rejected tasks
|
||||
failed_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'currentstate':['failed','rejected']})))")" 2>/dev/null)
|
||||
failed=$(python3 -c "
|
||||
import json
|
||||
d=json.loads('''$failed_json''')
|
||||
out=[]
|
||||
for t in d[:5]:
|
||||
n=t.get('Spec',{}).get('Name','?')
|
||||
e=t.get('Status',{}).get('Err','')[:80]
|
||||
out.append(f'{n}={e}' if e else n)
|
||||
print(';'.join(out))
|
||||
" 2>/dev/null)
|
||||
[[ -z "$failed" ]] || PROBLEMS+=("failed_tasks: ${failed:0:300}")
|
||||
|
||||
# 5. brencher liveness — running container
|
||||
containers_json=$(curl -fsS -m "$CURL_MAX" \
|
||||
"$PROXY/containers/json?all=0&filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'name':['brencher']})))")" \
|
||||
2>/dev/null)
|
||||
brencher_id=$(python3 -c "
|
||||
import json
|
||||
d=json.loads('''$containers_json''')
|
||||
print(d[0]['Id'] if d else '')
|
||||
" 2>/dev/null)
|
||||
if [[ -n "$brencher_id" ]]; then
|
||||
raw_logs=$(curl -fsS -m "$CURL_MAX" \
|
||||
"$PROXY/containers/$brencher_id/logs?stdout=1&stderr=1&tail=1×tamps=1" \
|
||||
2>/dev/null)
|
||||
# Docker stream frames: 8-byte header per line. Strip to plain text.
|
||||
last_ts=$(printf '%s' "$raw_logs" \
|
||||
| python3 -c "
|
||||
import sys,re
|
||||
data=sys.stdin.read()
|
||||
# Find ISO-8601 timestamps even if header bytes precede them
|
||||
m=re.findall(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[^\s]*', data)
|
||||
print(m[-1] if m else '')
|
||||
")
|
||||
if [[ -n "$last_ts" ]]; then
|
||||
last_epoch=$(date -d "$last_ts" +%s 2>/dev/null || echo 0)
|
||||
age=$(( $(date +%s) - last_epoch ))
|
||||
[[ $age -le $BRENCHER_MAX_AGE_S ]] || PROBLEMS+=("brencher_stale: age=${age}s (threshold ${BRENCHER_MAX_AGE_S}s)")
|
||||
else
|
||||
PROBLEMS+=("brencher_log_unparseable: id=${brencher_id:0:12}")
|
||||
fi
|
||||
else
|
||||
PROBLEMS+=("brencher_not_running")
|
||||
fi
|
||||
|
||||
# 6. Disk > 90%
|
||||
disk_use=$(df -P /opt/data | awk 'NR==2 {gsub("%",""); print $5}')
|
||||
[[ "$disk_use" -lt 90 ]] || PROBLEMS+=("disk_high: ${disk_use}% on /opt/data")
|
||||
|
||||
# 7. Memory available < 512M
|
||||
mem_avail_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
[[ "$mem_avail_kb" -gt 524288 ]] || PROBLEMS+=("memory_low: ${mem_avail_kb} kB available")
|
||||
|
||||
# Emit single alert
|
||||
if [[ ${#PROBLEMS[@]} -gt 0 ]]; then
|
||||
echo "⚠️ prod-monitor (${HOSTNAME}):"
|
||||
printf ' • %s\n' "${PROBLEMS[@]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
- name: Verify monitor_prod.sh syntax
|
||||
ansible.builtin.command: bash -n "{{ monitoring_script_dest | default('/usr/local/bin/monitor_prod.sh') }}"
|
||||
changed_when: false
|
||||
tags: [monitoring, verify]
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "monitoring",
|
||||
"version": "0.1.0",
|
||||
"author": "infra",
|
||||
"description": "Prod monitoring watchdog script (silent cron, alerts on failure).",
|
||||
"license": "proprietary",
|
||||
"min_ansible_version": "2.10",
|
||||
"platforms": [
|
||||
{
|
||||
"name": "Debian",
|
||||
"versions": ["trixie"]
|
||||
}
|
||||
],
|
||||
"galaxy_info": {
|
||||
"role_name": "monitoring",
|
||||
"author": "infra",
|
||||
"description": "Prod monitoring watchdog script (silent cron, alerts on failure).",
|
||||
"license": "proprietary",
|
||||
"min_ansible_version": "2.10",
|
||||
"platforms": [
|
||||
{
|
||||
"name": "Debian",
|
||||
"versions": ["trixie"]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
galaxy_info:
|
||||
role_name: monitoring
|
||||
author: infra
|
||||
description: Prod monitoring watchdog script (silent cron, alerts on failure).
|
||||
license: proprietary
|
||||
min_ansible_version: "2.10"
|
||||
platforms:
|
||||
- name: Debian
|
||||
versions:
|
||||
- trixie
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
- name: Install prod-monitor watchdog script
|
||||
copy:
|
||||
src: monitor_prod.sh
|
||||
dest: "{{ monitoring_script_dest | default('/usr/local/bin/monitor_prod.sh') }}"
|
||||
mode: "0755"
|
||||
owner: root
|
||||
group: root
|
||||
become: true
|
||||
tags: [monitoring, install]
|
||||
Reference in New Issue
Block a user