|
|
|
@@ -0,0 +1,187 @@
|
|
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
# prod-monitor: silent watchdog for hermes_infra + manager host.
|
|
|
|
|
# stdout empty => nothing to report. non-empty => single Telegram alert.
|
|
|
|
|
|
|
|
|
|
set -u
|
|
|
|
|
PROXY="${PROXY_URL:-http://docker_socket_proxy:2375}"
|
|
|
|
|
CURL_MAX=3
|
|
|
|
|
EXPECTED_SVCS=(squid sarg hermes-ui hermes-gateway docker_socket_proxy)
|
|
|
|
|
STACK_PREFIX="hermes_infra_"
|
|
|
|
|
RESTART_THRESH=3
|
|
|
|
|
BRENCHER_MAX_AGE_S=600
|
|
|
|
|
BRENCHER_LOG_TAIL=100
|
|
|
|
|
BRENCHER_ERROR_THRESH=5
|
|
|
|
|
|
|
|
|
|
# Patterns that count as "log thrash" when exceeding threshold
|
|
|
|
|
BRENCHER_ERR_PATTERNS='ERROR[[:space:]]+-|\] ERROR:|\bTraceback\b|\bfatal:|\bFATAL\b|\bException\b|\bBaseException\b|exit code\([0-9]+\)'
|
|
|
|
|
|
|
|
|
|
PROBLEMS=()
|
|
|
|
|
|
|
|
|
|
# Tiny JSON helper (avoids jq dependency)
|
|
|
|
|
jget() {
|
|
|
|
|
# jget <json> <python-path-expr> -- e.g. jget "$x" 'x["ID"]' or 'x[0]["Spec"]["Name"]'
|
|
|
|
|
local json="$1" expr="$2"
|
|
|
|
|
python3 -c "
|
|
|
|
|
import json,sys
|
|
|
|
|
try:
|
|
|
|
|
d=json.loads(sys.stdin.read())
|
|
|
|
|
except Exception as e:
|
|
|
|
|
sys.exit(0)
|
|
|
|
|
v=$expr
|
|
|
|
|
if isinstance(v,(list,tuple)):
|
|
|
|
|
print('\n'.join(str(x) for x in v))
|
|
|
|
|
elif v is None:
|
|
|
|
|
pass
|
|
|
|
|
else:
|
|
|
|
|
print(v)
|
|
|
|
|
" <<<"$json"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# 1. Swarm reachable
|
|
|
|
|
ping=$(curl -fsS -m "$CURL_MAX" "$PROXY/_ping" 2>/dev/null)
|
|
|
|
|
[[ "$ping" == "OK" ]] || PROBLEMS+=("swarm_ping: response='${ping:-no_response}'")
|
|
|
|
|
|
|
|
|
|
# 2. hermes_infra services running
|
|
|
|
|
names_filter=$(python3 -c "
|
|
|
|
|
import json; names=['${STACK_PREFIX}${EXPECTED_SVCS[0]}','${STACK_PREFIX}${EXPECTED_SVCS[1]}','${STACK_PREFIX}${EXPECTED_SVCS[2]}','${STACK_PREFIX}${EXPECTED_SVCS[3]}','${STACK_PREFIX}${EXPECTED_SVCS[4]}']
|
|
|
|
|
print(json.dumps({'name':names}))
|
|
|
|
|
")
|
|
|
|
|
services_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/services?filters=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.stdin.read()))" <<<"$names_filter")" 2>/dev/null)
|
|
|
|
|
|
|
|
|
|
declare -A SVC_ID
|
|
|
|
|
for i in "${!EXPECTED_SVCS[@]}"; do
|
|
|
|
|
svc="${EXPECTED_SVCS[$i]}"
|
|
|
|
|
full="${STACK_PREFIX}${svc}"
|
|
|
|
|
id=$(jget "$services_json" "[s for s in __import__('json').loads('''$services_json''') if s.get('Spec',{}).get('Name')=='$full'][0]['ID']" 2>/dev/null)
|
|
|
|
|
# Simpler approach: re-loop in python
|
|
|
|
|
id=$(python3 -c "
|
|
|
|
|
import json,sys
|
|
|
|
|
data=json.loads('''$services_json''')
|
|
|
|
|
for s in data:
|
|
|
|
|
if s.get('Spec',{}).get('Name')=='$full':
|
|
|
|
|
print(s['ID'])
|
|
|
|
|
break
|
|
|
|
|
" 2>/dev/null)
|
|
|
|
|
if [[ -z "$id" ]]; then
|
|
|
|
|
PROBLEMS+=("service_missing: ${svc}")
|
|
|
|
|
continue
|
|
|
|
|
fi
|
|
|
|
|
SVC_ID["$svc"]="$id"
|
|
|
|
|
tasks_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'service':['$id']})))")" 2>/dev/null)
|
|
|
|
|
running=$(python3 -c "
|
|
|
|
|
import json
|
|
|
|
|
d=json.loads('''$tasks_json''')
|
|
|
|
|
print(sum(1 for t in d if t.get('Status',{}).get('State')=='running'))
|
|
|
|
|
" 2>/dev/null)
|
|
|
|
|
[[ "${running:-0}" -ge 1 ]] || PROBLEMS+=("service_no_running_task: ${svc}")
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
# 3. Restart flapping
|
|
|
|
|
flaps_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'desiredstate':['running']})))")" 2>/dev/null)
|
|
|
|
|
flaps=$(python3 -c "
|
|
|
|
|
import json
|
|
|
|
|
d=json.loads('''$flaps_json''')
|
|
|
|
|
out=[]
|
|
|
|
|
for t in d:
|
|
|
|
|
rc=t.get('Status',{}).get('ContainerStatus',{}).get('RestartCount',0)
|
|
|
|
|
if rc>$RESTART_THRESH:
|
|
|
|
|
name=t.get('Spec',{}).get('Name','?')
|
|
|
|
|
out.append(f'{name}={rc}')
|
|
|
|
|
print(';'.join(out))
|
|
|
|
|
" 2>/dev/null)
|
|
|
|
|
[[ -z "$flaps" ]] || PROBLEMS+=("flapping_tasks: ${flaps:0:200}")
|
|
|
|
|
|
|
|
|
|
# 4. Failed/rejected tasks
|
|
|
|
|
failed_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'currentstate':['failed','rejected']})))")" 2>/dev/null)
|
|
|
|
|
failed=$(python3 -c "
|
|
|
|
|
import json
|
|
|
|
|
d=json.loads('''$failed_json''')
|
|
|
|
|
out=[]
|
|
|
|
|
for t in d[:5]:
|
|
|
|
|
n=t.get('Spec',{}).get('Name','?')
|
|
|
|
|
e=t.get('Status',{}).get('Err','')[:80]
|
|
|
|
|
out.append(f'{n}={e}' if e else n)
|
|
|
|
|
print(';'.join(out))
|
|
|
|
|
" 2>/dev/null)
|
|
|
|
|
[[ -z "$failed" ]] || PROBLEMS+=("failed_tasks: ${failed:0:300}")
|
|
|
|
|
|
|
|
|
|
# 5. brencher liveness — running container
|
|
|
|
|
containers_json=$(curl -fsS -m "$CURL_MAX" \
|
|
|
|
|
"$PROXY/containers/json?all=0&filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'name':['brencher']})))")" \
|
|
|
|
|
2>/dev/null)
|
|
|
|
|
brencher_id=$(python3 -c "
|
|
|
|
|
import json
|
|
|
|
|
d=json.loads('''$containers_json''')
|
|
|
|
|
print(d[0]['Id'] if d else '')
|
|
|
|
|
" 2>/dev/null)
|
|
|
|
|
if [[ -n "$brencher_id" ]]; then
|
|
|
|
|
# Replace all non-printable/non-space bytes with '?' so the stream is pure ASCII
|
|
|
|
|
# and downstream tools (grep, python3, cron UTF-8 decoder) handle it cleanly.
|
|
|
|
|
raw_logs=$(curl -fsS -m "$CURL_MAX" \
|
|
|
|
|
"$PROXY/containers/$brencher_id/logs?stdout=1&stderr=1&tail=${BRENCHER_LOG_TAIL}×tamps=1" \
|
|
|
|
|
2>/dev/null \
|
|
|
|
|
| LC_ALL=C tr -c '[:print:][:space:]' '?')
|
|
|
|
|
# Docker stream frames: 8-byte header per line. Strip to plain text.
|
|
|
|
|
parsed=$(printf '%s' "$raw_logs" \
|
|
|
|
|
| python3 -c "
|
|
|
|
|
import sys,re
|
|
|
|
|
data=sys.stdin.buffer.read().decode('utf-8', errors='replace')
|
|
|
|
|
m=re.findall(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[^\s]*', data)
|
|
|
|
|
print(m[-1] if m else '')
|
|
|
|
|
")
|
|
|
|
|
last_ts="$parsed"
|
|
|
|
|
if [[ -n "$last_ts" ]]; then
|
|
|
|
|
last_epoch=$(date -d "$last_ts" +%s 2>/dev/null || echo 0)
|
|
|
|
|
age=$(( $(date +%s) - last_epoch ))
|
|
|
|
|
[[ $age -le $BRENCHER_MAX_AGE_S ]] || PROBLEMS+=("brencher_stale: age=${age}s (threshold ${BRENCHER_MAX_AGE_S}s)")
|
|
|
|
|
else
|
|
|
|
|
PROBLEMS+=("brencher_log_unparseable: id=${brencher_id:0:12}")
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Count error patterns in the same tail window.
|
|
|
|
|
err_count=$(printf '%s' "$raw_logs" | grep -cE "$BRENCHER_ERR_PATTERNS")
|
|
|
|
|
if [[ "${err_count:-0}" -ge "$BRENCHER_ERROR_THRESH" ]]; then
|
|
|
|
|
sample=$(printf '%s' "$raw_logs" \
|
|
|
|
|
| grep -E "$BRENCHER_ERR_PATTERNS" \
|
|
|
|
|
| sed -E 's/^.{8}//' \
|
|
|
|
|
| grep -vE '^\s*$' \
|
|
|
|
|
| head -3 \
|
|
|
|
|
| tr '\n' ';' \
|
|
|
|
|
| head -c 200)
|
|
|
|
|
PROBLEMS+=("brencher_log_thrash: ${err_count} error-pattern lines in last ${BRENCHER_LOG_TAIL} lines; sample: ${sample}")
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
PROBLEMS+=("brencher_not_running")
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# 6. Disk > 90%
|
|
|
|
|
disk_use=$(df -P /opt/data | awk 'NR==2 {gsub("%",""); print $5}')
|
|
|
|
|
[[ "$disk_use" -lt 90 ]] || PROBLEMS+=("disk_high: ${disk_use}% on /opt/data")
|
|
|
|
|
|
|
|
|
|
# 7. Memory available < 512M
|
|
|
|
|
mem_avail_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
|
|
|
|
[[ "$mem_avail_kb" -gt 524288 ]] || PROBLEMS+=("memory_low: ${mem_avail_kb} kB available")
|
|
|
|
|
|
|
|
|
|
# Emit single alert
|
|
|
|
|
# Final sanitization: drop control bytes (NUL, frame headers, anything
|
|
|
|
|
# outside printable+space Unicode). Done in Python because GNU tr treats
|
|
|
|
|
# UTF-8 as bytes and mangles multi-byte chars (emoji, •, кириллица).
|
|
|
|
|
# Suppress bash's "ignored null byte" / grep "binary file" noise — watchdog
|
|
|
|
|
# contract is stdout-only.
|
|
|
|
|
if [[ ${#PROBLEMS[@]} -gt 0 ]]; then
|
|
|
|
|
{
|
|
|
|
|
echo "⚠️ prod-monitor (${HOSTNAME}):"
|
|
|
|
|
printf ' • %s\n' "${PROBLEMS[@]}"
|
|
|
|
|
} 2>/dev/null \
|
|
|
|
|
| python3 -c "
|
|
|
|
|
import sys
|
|
|
|
|
data = sys.stdin.buffer.read()
|
|
|
|
|
try:
|
|
|
|
|
s = data.decode('utf-8')
|
|
|
|
|
except UnicodeDecodeError:
|
|
|
|
|
s = data.decode('utf-8', errors='replace')
|
|
|
|
|
out = ''.join(ch if (ch.isprintable() or ch.isspace()) else '?' for ch in s)
|
|
|
|
|
sys.stdout.write(out)
|
|
|
|
|
"
|
|
|
|
|
fi
|
|
|
|
|
exit 0
|