feat(monitoring): add prod-monitor watchdog role #36

Open
hermes wants to merge 1 commits from hermes/ansible:feat/monitoring-prod-watchdog into master
6 changed files with 260 additions and 0 deletions
+20
View File
@@ -0,0 +1,20 @@
---
# defaults for monitoring role
# docker_socket_proxy endpoint used by monitor_prod.sh
monitoring_proxy_url: "http://docker_socket_proxy:2375"
# Service list (without stack prefix) expected in the swarm stack
monitoring_stack_prefix: "hermes_infra_"
monitoring_expected_services:
- squid
- sarg
- hermes-ui
- hermes-gateway
- docker_socket_proxy
# Alert thresholds
monitoring_restart_threshold: 3
monitoring_brencher_max_age_s: 600
monitoring_disk_threshold_pct: 90
monitoring_mem_min_available_mb: 512
+187
View File
@@ -0,0 +1,187 @@
#!/usr/bin/env bash
# prod-monitor: silent watchdog for hermes_infra + manager host.
# stdout empty => nothing to report. non-empty => single Telegram alert.
set -u
PROXY="${PROXY_URL:-http://docker_socket_proxy:2375}"
CURL_MAX=3
EXPECTED_SVCS=(squid sarg hermes-ui hermes-gateway docker_socket_proxy)
STACK_PREFIX="hermes_infra_"
RESTART_THRESH=3
BRENCHER_MAX_AGE_S=600
BRENCHER_LOG_TAIL=100
BRENCHER_ERROR_THRESH=5
# Patterns that count as "log thrash" when exceeding threshold
BRENCHER_ERR_PATTERNS='ERROR[[:space:]]+-|\] ERROR:|\bTraceback\b|\bfatal:|\bFATAL\b|\bException\b|\bBaseException\b|exit code\([0-9]+\)'
PROBLEMS=()
# Tiny JSON helper (avoids jq dependency)
jget() {
# jget <json> <python-path-expr> -- e.g. jget "$x" 'x["ID"]' or 'x[0]["Spec"]["Name"]'
local json="$1" expr="$2"
python3 -c "
import json,sys
try:
d=json.loads(sys.stdin.read())
except Exception as e:
sys.exit(0)
v=$expr
if isinstance(v,(list,tuple)):
print('\n'.join(str(x) for x in v))
elif v is None:
pass
else:
print(v)
" <<<"$json"
}
# 1. Swarm reachable
ping=$(curl -fsS -m "$CURL_MAX" "$PROXY/_ping" 2>/dev/null)
[[ "$ping" == "OK" ]] || PROBLEMS+=("swarm_ping: response='${ping:-no_response}'")
# 2. hermes_infra services running
names_filter=$(python3 -c "
import json; names=['${STACK_PREFIX}${EXPECTED_SVCS[0]}','${STACK_PREFIX}${EXPECTED_SVCS[1]}','${STACK_PREFIX}${EXPECTED_SVCS[2]}','${STACK_PREFIX}${EXPECTED_SVCS[3]}','${STACK_PREFIX}${EXPECTED_SVCS[4]}']
print(json.dumps({'name':names}))
")
services_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/services?filters=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.stdin.read()))" <<<"$names_filter")" 2>/dev/null)
declare -A SVC_ID
for i in "${!EXPECTED_SVCS[@]}"; do
svc="${EXPECTED_SVCS[$i]}"
full="${STACK_PREFIX}${svc}"
id=$(jget "$services_json" "[s for s in __import__('json').loads('''$services_json''') if s.get('Spec',{}).get('Name')=='$full'][0]['ID']" 2>/dev/null)
# Simpler approach: re-loop in python
id=$(python3 -c "
import json,sys
data=json.loads('''$services_json''')
for s in data:
if s.get('Spec',{}).get('Name')=='$full':
print(s['ID'])
break
" 2>/dev/null)
if [[ -z "$id" ]]; then
PROBLEMS+=("service_missing: ${svc}")
continue
fi
SVC_ID["$svc"]="$id"
tasks_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'service':['$id']})))")" 2>/dev/null)
running=$(python3 -c "
import json
d=json.loads('''$tasks_json''')
print(sum(1 for t in d if t.get('Status',{}).get('State')=='running'))
" 2>/dev/null)
[[ "${running:-0}" -ge 1 ]] || PROBLEMS+=("service_no_running_task: ${svc}")
done
# 3. Restart flapping
flaps_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'desiredstate':['running']})))")" 2>/dev/null)
flaps=$(python3 -c "
import json
d=json.loads('''$flaps_json''')
out=[]
for t in d:
rc=t.get('Status',{}).get('ContainerStatus',{}).get('RestartCount',0)
if rc>$RESTART_THRESH:
name=t.get('Spec',{}).get('Name','?')
out.append(f'{name}={rc}')
print(';'.join(out))
" 2>/dev/null)
[[ -z "$flaps" ]] || PROBLEMS+=("flapping_tasks: ${flaps:0:200}")
# 4. Failed/rejected tasks
failed_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'currentstate':['failed','rejected']})))")" 2>/dev/null)
failed=$(python3 -c "
import json
d=json.loads('''$failed_json''')
out=[]
for t in d[:5]:
n=t.get('Spec',{}).get('Name','?')
e=t.get('Status',{}).get('Err','')[:80]
out.append(f'{n}={e}' if e else n)
print(';'.join(out))
" 2>/dev/null)
[[ -z "$failed" ]] || PROBLEMS+=("failed_tasks: ${failed:0:300}")
# 5. brencher liveness — running container
containers_json=$(curl -fsS -m "$CURL_MAX" \
"$PROXY/containers/json?all=0&filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'name':['brencher']})))")" \
2>/dev/null)
brencher_id=$(python3 -c "
import json
d=json.loads('''$containers_json''')
print(d[0]['Id'] if d else '')
" 2>/dev/null)
if [[ -n "$brencher_id" ]]; then
# Replace all non-printable/non-space bytes with '?' so the stream is pure ASCII
# and downstream tools (grep, python3, cron UTF-8 decoder) handle it cleanly.
raw_logs=$(curl -fsS -m "$CURL_MAX" \
"$PROXY/containers/$brencher_id/logs?stdout=1&stderr=1&tail=${BRENCHER_LOG_TAIL}&timestamps=1" \
2>/dev/null \
| LC_ALL=C tr -c '[:print:][:space:]' '?')
# Docker stream frames: 8-byte header per line. Strip to plain text.
parsed=$(printf '%s' "$raw_logs" \
| python3 -c "
import sys,re
data=sys.stdin.buffer.read().decode('utf-8', errors='replace')
m=re.findall(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[^\s]*', data)
print(m[-1] if m else '')
")
last_ts="$parsed"
if [[ -n "$last_ts" ]]; then
last_epoch=$(date -d "$last_ts" +%s 2>/dev/null || echo 0)
age=$(( $(date +%s) - last_epoch ))
[[ $age -le $BRENCHER_MAX_AGE_S ]] || PROBLEMS+=("brencher_stale: age=${age}s (threshold ${BRENCHER_MAX_AGE_S}s)")
else
PROBLEMS+=("brencher_log_unparseable: id=${brencher_id:0:12}")
fi
# Count error patterns in the same tail window.
err_count=$(printf '%s' "$raw_logs" | grep -cE "$BRENCHER_ERR_PATTERNS")
if [[ "${err_count:-0}" -ge "$BRENCHER_ERROR_THRESH" ]]; then
sample=$(printf '%s' "$raw_logs" \
| grep -E "$BRENCHER_ERR_PATTERNS" \
| sed -E 's/^.{8}//' \
| grep -vE '^\s*$' \
| head -3 \
| tr '\n' ';' \
| head -c 200)
PROBLEMS+=("brencher_log_thrash: ${err_count} error-pattern lines in last ${BRENCHER_LOG_TAIL} lines; sample: ${sample}")
fi
else
PROBLEMS+=("brencher_not_running")
fi
# 6. Disk > 90%
disk_use=$(df -P /opt/data | awk 'NR==2 {gsub("%",""); print $5}')
[[ "$disk_use" -lt 90 ]] || PROBLEMS+=("disk_high: ${disk_use}% on /opt/data")
# 7. Memory available < 512M
mem_avail_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
[[ "$mem_avail_kb" -gt 524288 ]] || PROBLEMS+=("memory_low: ${mem_avail_kb} kB available")
# Emit single alert
# Final sanitization: drop control bytes (NUL, frame headers, anything
# outside printable+space Unicode). Done in Python because GNU tr treats
# UTF-8 as bytes and mangles multi-byte chars (emoji, •, кириллица).
# Suppress bash's "ignored null byte" / grep "binary file" noise — watchdog
# contract is stdout-only.
if [[ ${#PROBLEMS[@]} -gt 0 ]]; then
{
echo "⚠️ prod-monitor (${HOSTNAME}):"
printf ' • %s\n' "${PROBLEMS[@]}"
} 2>/dev/null \
| python3 -c "
import sys
data = sys.stdin.buffer.read()
try:
s = data.decode('utf-8')
except UnicodeDecodeError:
s = data.decode('utf-8', errors='replace')
out = ''.join(ch if (ch.isprintable() or ch.isspace()) else '?' for ch in s)
sys.stdout.write(out)
"
fi
exit 0
+5
View File
@@ -0,0 +1,5 @@
---
- name: Verify monitor_prod.sh syntax
ansible.builtin.command: bash -n "{{ monitoring_script_dest | default('/usr/local/bin/monitor_prod.sh') }}"
changed_when: false
tags: [monitoring, verify]
+27
View File
@@ -0,0 +1,27 @@
{
"name": "monitoring",
"version": "0.1.0",
"author": "infra",
"description": "Prod monitoring watchdog script (silent cron, alerts on failure).",
"license": "proprietary",
"min_ansible_version": "2.10",
"platforms": [
{
"name": "Debian",
"versions": ["trixie"]
}
],
"galaxy_info": {
"role_name": "monitoring",
"author": "infra",
"description": "Prod monitoring watchdog script (silent cron, alerts on failure).",
"license": "proprietary",
"min_ansible_version": "2.10",
"platforms": [
{
"name": "Debian",
"versions": ["trixie"]
}
]
}
}
+11
View File
@@ -0,0 +1,11 @@
---
galaxy_info:
role_name: monitoring
author: infra
description: Prod monitoring watchdog script (silent cron, alerts on failure).
license: proprietary
min_ansible_version: "2.10"
platforms:
- name: Debian
versions:
- trixie
+10
View File
@@ -0,0 +1,10 @@
---
- name: Install prod-monitor watchdog script
copy:
src: monitor_prod.sh
dest: "{{ monitoring_script_dest | default('/usr/local/bin/monitor_prod.sh') }}"
mode: "0755"
owner: root
group: root
become: true
tags: [monitoring, install]