diff --git a/roles/monitoring/defaults/main.yml b/roles/monitoring/defaults/main.yml new file mode 100644 index 0000000..e3dda6f --- /dev/null +++ b/roles/monitoring/defaults/main.yml @@ -0,0 +1,20 @@ +--- +# defaults for monitoring role + +# docker_socket_proxy endpoint used by monitor_prod.sh +monitoring_proxy_url: "http://docker_socket_proxy:2375" + +# Service list (without stack prefix) expected in the swarm stack +monitoring_stack_prefix: "hermes_infra_" +monitoring_expected_services: + - squid + - sarg + - hermes-ui + - hermes-gateway + - docker_socket_proxy + +# Alert thresholds +monitoring_restart_threshold: 3 +monitoring_brencher_max_age_s: 600 +monitoring_disk_threshold_pct: 90 +monitoring_mem_min_available_mb: 512 diff --git a/roles/monitoring/files/monitor_prod.sh b/roles/monitoring/files/monitor_prod.sh new file mode 100644 index 0000000..ae6e611 --- /dev/null +++ b/roles/monitoring/files/monitor_prod.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# prod-monitor: silent watchdog for hermes_infra + manager host. +# stdout empty => nothing to report. non-empty => single Telegram alert. + +set -u +PROXY="${PROXY_URL:-http://docker_socket_proxy:2375}" +CURL_MAX=3 +EXPECTED_SVCS=(squid sarg hermes-ui hermes-gateway docker_socket_proxy) +STACK_PREFIX="hermes_infra_" +RESTART_THRESH=3 +BRENCHER_MAX_AGE_S=600 +BRENCHER_LOG_TAIL=100 +BRENCHER_ERROR_THRESH=5 + +# Patterns that count as "log thrash" when exceeding threshold +BRENCHER_ERR_PATTERNS='ERROR[[:space:]]+-|\] ERROR:|\bTraceback\b|\bfatal:|\bFATAL\b|\bException\b|\bBaseException\b|exit code\([0-9]+\)' + +PROBLEMS=() + +# Tiny JSON helper (avoids jq dependency) +jget() { + # jget -- e.g. jget "$x" 'x["ID"]' or 'x[0]["Spec"]["Name"]' + local json="$1" expr="$2" + python3 -c " +import json,sys +try: + d=json.loads(sys.stdin.read()) +except Exception as e: + sys.exit(0) +v=$expr +if isinstance(v,(list,tuple)): + print('\n'.join(str(x) for x in v)) +elif v is None: + pass +else: + print(v) +" <<<"$json" +} + +# 1. Swarm reachable +ping=$(curl -fsS -m "$CURL_MAX" "$PROXY/_ping" 2>/dev/null) +[[ "$ping" == "OK" ]] || PROBLEMS+=("swarm_ping: response='${ping:-no_response}'") + +# 2. hermes_infra services running +names_filter=$(python3 -c " +import json; names=['${STACK_PREFIX}${EXPECTED_SVCS[0]}','${STACK_PREFIX}${EXPECTED_SVCS[1]}','${STACK_PREFIX}${EXPECTED_SVCS[2]}','${STACK_PREFIX}${EXPECTED_SVCS[3]}','${STACK_PREFIX}${EXPECTED_SVCS[4]}'] +print(json.dumps({'name':names})) +") +services_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/services?filters=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.stdin.read()))" <<<"$names_filter")" 2>/dev/null) + +declare -A SVC_ID +for i in "${!EXPECTED_SVCS[@]}"; do + svc="${EXPECTED_SVCS[$i]}" + full="${STACK_PREFIX}${svc}" + id=$(jget "$services_json" "[s for s in __import__('json').loads('''$services_json''') if s.get('Spec',{}).get('Name')=='$full'][0]['ID']" 2>/dev/null) + # Simpler approach: re-loop in python + id=$(python3 -c " +import json,sys +data=json.loads('''$services_json''') +for s in data: + if s.get('Spec',{}).get('Name')=='$full': + print(s['ID']) + break +" 2>/dev/null) + if [[ -z "$id" ]]; then + PROBLEMS+=("service_missing: ${svc}") + continue + fi + SVC_ID["$svc"]="$id" + tasks_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'service':['$id']})))")" 2>/dev/null) + running=$(python3 -c " +import json +d=json.loads('''$tasks_json''') +print(sum(1 for t in d if t.get('Status',{}).get('State')=='running')) +" 2>/dev/null) + [[ "${running:-0}" -ge 1 ]] || PROBLEMS+=("service_no_running_task: ${svc}") +done + +# 3. Restart flapping +flaps_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'desiredstate':['running']})))")" 2>/dev/null) +flaps=$(python3 -c " +import json +d=json.loads('''$flaps_json''') +out=[] +for t in d: + rc=t.get('Status',{}).get('ContainerStatus',{}).get('RestartCount',0) + if rc>$RESTART_THRESH: + name=t.get('Spec',{}).get('Name','?') + out.append(f'{name}={rc}') +print(';'.join(out)) +" 2>/dev/null) +[[ -z "$flaps" ]] || PROBLEMS+=("flapping_tasks: ${flaps:0:200}") + +# 4. Failed/rejected tasks +failed_json=$(curl -fsS -m "$CURL_MAX" "$PROXY/tasks?filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'currentstate':['failed','rejected']})))")" 2>/dev/null) +failed=$(python3 -c " +import json +d=json.loads('''$failed_json''') +out=[] +for t in d[:5]: + n=t.get('Spec',{}).get('Name','?') + e=t.get('Status',{}).get('Err','')[:80] + out.append(f'{n}={e}' if e else n) +print(';'.join(out)) +" 2>/dev/null) +[[ -z "$failed" ]] || PROBLEMS+=("failed_tasks: ${failed:0:300}") + +# 5. brencher liveness — running container +containers_json=$(curl -fsS -m "$CURL_MAX" \ + "$PROXY/containers/json?all=0&filters=$(python3 -c "import json,urllib.parse; print(urllib.parse.quote(json.dumps({'name':['brencher']})))")" \ + 2>/dev/null) +brencher_id=$(python3 -c " +import json +d=json.loads('''$containers_json''') +print(d[0]['Id'] if d else '') +" 2>/dev/null) +if [[ -n "$brencher_id" ]]; then + # Replace all non-printable/non-space bytes with '?' so the stream is pure ASCII + # and downstream tools (grep, python3, cron UTF-8 decoder) handle it cleanly. + raw_logs=$(curl -fsS -m "$CURL_MAX" \ + "$PROXY/containers/$brencher_id/logs?stdout=1&stderr=1&tail=${BRENCHER_LOG_TAIL}×tamps=1" \ + 2>/dev/null \ + | LC_ALL=C tr -c '[:print:][:space:]' '?') + # Docker stream frames: 8-byte header per line. Strip to plain text. + parsed=$(printf '%s' "$raw_logs" \ + | python3 -c " +import sys,re +data=sys.stdin.buffer.read().decode('utf-8', errors='replace') +m=re.findall(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[^\s]*', data) +print(m[-1] if m else '') +") + last_ts="$parsed" + if [[ -n "$last_ts" ]]; then + last_epoch=$(date -d "$last_ts" +%s 2>/dev/null || echo 0) + age=$(( $(date +%s) - last_epoch )) + [[ $age -le $BRENCHER_MAX_AGE_S ]] || PROBLEMS+=("brencher_stale: age=${age}s (threshold ${BRENCHER_MAX_AGE_S}s)") + else + PROBLEMS+=("brencher_log_unparseable: id=${brencher_id:0:12}") + fi + + # Count error patterns in the same tail window. + err_count=$(printf '%s' "$raw_logs" | grep -cE "$BRENCHER_ERR_PATTERNS") + if [[ "${err_count:-0}" -ge "$BRENCHER_ERROR_THRESH" ]]; then + sample=$(printf '%s' "$raw_logs" \ + | grep -E "$BRENCHER_ERR_PATTERNS" \ + | sed -E 's/^.{8}//' \ + | grep -vE '^\s*$' \ + | head -3 \ + | tr '\n' ';' \ + | head -c 200) + PROBLEMS+=("brencher_log_thrash: ${err_count} error-pattern lines in last ${BRENCHER_LOG_TAIL} lines; sample: ${sample}") + fi +else + PROBLEMS+=("brencher_not_running") +fi + +# 6. Disk > 90% +disk_use=$(df -P /opt/data | awk 'NR==2 {gsub("%",""); print $5}') +[[ "$disk_use" -lt 90 ]] || PROBLEMS+=("disk_high: ${disk_use}% on /opt/data") + +# 7. Memory available < 512M +mem_avail_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) +[[ "$mem_avail_kb" -gt 524288 ]] || PROBLEMS+=("memory_low: ${mem_avail_kb} kB available") + +# Emit single alert +# Final sanitization: drop control bytes (NUL, frame headers, anything +# outside printable+space Unicode). Done in Python because GNU tr treats +# UTF-8 as bytes and mangles multi-byte chars (emoji, •, кириллица). +# Suppress bash's "ignored null byte" / grep "binary file" noise — watchdog +# contract is stdout-only. +if [[ ${#PROBLEMS[@]} -gt 0 ]]; then + { + echo "⚠️ prod-monitor (${HOSTNAME}):" + printf ' • %s\n' "${PROBLEMS[@]}" + } 2>/dev/null \ + | python3 -c " +import sys +data = sys.stdin.buffer.read() +try: + s = data.decode('utf-8') +except UnicodeDecodeError: + s = data.decode('utf-8', errors='replace') +out = ''.join(ch if (ch.isprintable() or ch.isspace()) else '?' for ch in s) +sys.stdout.write(out) +" +fi +exit 0 diff --git a/roles/monitoring/handlers/main.yml b/roles/monitoring/handlers/main.yml new file mode 100644 index 0000000..0777de0 --- /dev/null +++ b/roles/monitoring/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Verify monitor_prod.sh syntax + ansible.builtin.command: bash -n "{{ monitoring_script_dest | default('/usr/local/bin/monitor_prod.sh') }}" + changed_when: false + tags: [monitoring, verify] diff --git a/roles/monitoring/meta/main.json b/roles/monitoring/meta/main.json new file mode 100644 index 0000000..a35258b --- /dev/null +++ b/roles/monitoring/meta/main.json @@ -0,0 +1,27 @@ +{ + "name": "monitoring", + "version": "0.1.0", + "author": "infra", + "description": "Prod monitoring watchdog script (silent cron, alerts on failure).", + "license": "proprietary", + "min_ansible_version": "2.10", + "platforms": [ + { + "name": "Debian", + "versions": ["trixie"] + } + ], + "galaxy_info": { + "role_name": "monitoring", + "author": "infra", + "description": "Prod monitoring watchdog script (silent cron, alerts on failure).", + "license": "proprietary", + "min_ansible_version": "2.10", + "platforms": [ + { + "name": "Debian", + "versions": ["trixie"] + } + ] + } +} diff --git a/roles/monitoring/meta/main.yml b/roles/monitoring/meta/main.yml new file mode 100644 index 0000000..018b437 --- /dev/null +++ b/roles/monitoring/meta/main.yml @@ -0,0 +1,11 @@ +--- +galaxy_info: + role_name: monitoring + author: infra + description: Prod monitoring watchdog script (silent cron, alerts on failure). + license: proprietary + min_ansible_version: "2.10" + platforms: + - name: Debian + versions: + - trixie diff --git a/roles/monitoring/tasks/main.yml b/roles/monitoring/tasks/main.yml new file mode 100644 index 0000000..6596f2e --- /dev/null +++ b/roles/monitoring/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Install prod-monitor watchdog script + copy: + src: monitor_prod.sh + dest: "{{ monitoring_script_dest | default('/usr/local/bin/monitor_prod.sh') }}" + mode: "0755" + owner: root + group: root + become: true + tags: [monitoring, install]