"""ServerMonitor main loop. Runs periodic health checks and sends Telegram alerts on failures, reminders while services stay down, and recovery messages when they come back. """ import argparse import json import logging import os import signal import sys import threading import time from urllib.parse import urlparse, urlunparse from checks import http_check, netdata_alarms, netdata_metrics, ping_check, tcp_check from notify import AlertEngine, StateStore, escape_html, send_telegram logger = logging.getLogger("servermonitor.monitor") shutdown_event = threading.Event() def load_config(path): """Load the JSON configuration file.""" with open(path, "r", encoding="utf-8") as f: return json.load(f) def _replace_host_in_url(url, new_host): """Replace the netloc host in a URL without changing the port or path.""" parsed = urlparse(url) # Preserve user:pass and port; replace only the hostname. netloc = new_host if parsed.port is not None: netloc = f"{new_host}:{parsed.port}" if parsed.username is not None: prefix = f"{parsed.username}" if parsed.password is not None: prefix += f":{parsed.password}" netloc = f"{prefix}@{netloc}" return urlunparse(parsed._replace(netloc=netloc)) def apply_env_overrides(config): """Override config values from environment variables.""" new_host = None if "HOST" in os.environ: new_host = os.environ["HOST"] config["host"] = new_host if "CHECK_INTERVAL_SEC" in os.environ: try: config["check_interval_sec"] = int(os.environ["CHECK_INTERVAL_SEC"]) except ValueError: logger.warning("Invalid CHECK_INTERVAL_SEC, using config value") if "TELEGRAM_BOT_TOKEN" in os.environ: config["telegram"]["bot_token"] = os.environ["TELEGRAM_BOT_TOKEN"] if "TELEGRAM_CHAT_ID" in os.environ: config["telegram"]["chat_id"] = os.environ["TELEGRAM_CHAT_ID"] if "STATE_FILE" in os.environ: config["state_file"] = os.environ["STATE_FILE"] if new_host: for check in config.get("http_checks", []): check["url"] = _replace_host_in_url(check.get("url", ""), new_host) for check in config.get("tcp_checks", []): check["host"] = new_host netdata_cfg = config.get("netdata", {}) if netdata_cfg.get("base_url"): netdata_cfg["base_url"] = _replace_host_in_url(netdata_cfg["base_url"], new_host) def setup_logging(): """Configure logging to stdout.""" logging.basicConfig( stream=sys.stdout, format="%(asctime)s %(levelname)s %(message)s", level=logging.INFO, ) def shutdown_handler(signum, frame): """Handle SIGTERM/SIGINT gracefully.""" logger.info("shutting down") shutdown_event.set() def run_checks(config): """Run every configured check and return a list of results. Each result is a tuple ``(name, ok, message)``. """ host = config["host"] timeout = config.get("timeout_sec", 5) results = [] if config.get("ping", {}).get("enabled", True): ok, msg = ping_check(host, timeout) results.append(("ping", ok, msg)) for check in config.get("http_checks", []): ok, msg = http_check( check["name"], check["url"], check.get("expected", [200]), timeout, ) results.append((check["name"], ok, msg)) for check in config.get("tcp_checks", []): ok, msg = tcp_check( check["name"], check.get("host", host), check["port"], timeout, ) results.append((check["name"], ok, msg)) netdata_cfg = config.get("netdata", {}) base_url = netdata_cfg.get("base_url", f"http://{host}:19999") if netdata_cfg.get("check_alarms", True): ok, msg = netdata_alarms(base_url, timeout) results.append(("netdata_alarms", ok, msg)) if netdata_cfg.get("check_metrics", True): metrics = netdata_metrics(base_url, timeout) thresholds = config.get("thresholds", {}) cpu = metrics.get("cpu_percent") cpu_threshold = thresholds.get("cpu_percent", 90) if cpu is not None and cpu > cpu_threshold: results.append(( "CPU", False, f"CPU {cpu:.1f}% (threshold {cpu_threshold}%)", )) elif cpu is not None: results.append(("CPU", True, f"CPU {cpu:.1f}%")) ram_used = metrics.get("ram_used_mb") ram_total = metrics.get("ram_total_mb") ram_threshold = thresholds.get("ram_avail_mb", 2048) if ram_used is not None and ram_total is not None: ram_avail = ram_total - ram_used if ram_avail < ram_threshold: results.append(( "RAM", False, f"RAM available {ram_avail:.0f}MB (threshold {ram_threshold}MB)", )) else: results.append(("RAM", True, f"RAM available {ram_avail:.0f}MB")) disk = metrics.get("disk_used_percent") disk_threshold = thresholds.get("disk_percent", 90) if disk is not None and disk > disk_threshold: results.append(( "Disk", False, f"Disk {disk:.1f}% used (threshold {disk_threshold}%)", )) elif disk is not None: results.append(("Disk", True, f"Disk {disk:.1f}% used")) return results def format_message(host, name, ok, message, is_recovery, alert_count): """Format a Telegram HTML alert message.""" safe_name = escape_html(name) safe_message = escape_html(message) safe_host = escape_html(host) if is_recovery: header = f"✅ Recovered · {safe_host}" body = f"🟢 {safe_name}: {safe_message}" return f"{header}\n\n{body}" if ok: # Should not normally alert on OK, but handle it gracefully. header = f"✅ Recovered · {safe_host}" body = f"🟢 {safe_name}: {safe_message}" return f"{header}\n\n{body}" if name == "netdata_alarms": header = f"⚠️ Server alert · {safe_host}" body = f"🔴 Netdata alarms: {safe_message}" return f"{header}\n\n{body}" if alert_count > 1: header = f"⏰ Still down · {safe_host}" body = f"🔴 {safe_name}: {safe_message}" return f"{header}\n\n{body}" header = f"⚠️ Server alert · {safe_host}" body = f"🔴 {safe_name}: {safe_message}" return f"{header}\n\n{body}" def main_loop(config, state_store, alert_engine, once=False): """Run the monitoring loop until shutdown or a single cycle if ``once``.""" reminder_interval = config.get("reminder_interval_sec", 1800) host = config["host"] bot_token = config["telegram"].get("bot_token", "") chat_id = config["telegram"].get("chat_id", "") timeout = config.get("timeout_sec", 5) baseline = not os.path.exists(state_store.path) while not shutdown_event.is_set(): now = time.time() results = run_checks(config) for name, ok, message in results: logger.info("%s %s (%s)", name, "OK" if ok else "FAIL", message) current_state = "OK" if ok else "CRIT" alert_engine.set_state(name, current_state) should_send, is_recovery = alert_engine.should_alert( name, now, state_store, reminder_interval ) if baseline: # First run with no state file: establish baseline, record state, # do not alert. alert_count stays 0 so the next real failure is # treated as the first alert. state_store.set(name, current_state, 0.0, 0) continue if should_send: state_entry = state_store.get(name) alert_count = state_entry["alert_count"] last_alert_ts = state_entry["last_alert_ts"] text = format_message( host, name, ok, message, is_recovery, alert_count, ) logger.warning("Alert: %s", text.replace("\n", " ")) send_telegram(bot_token, chat_id, text, timeout) baseline = False state_store.save() if once: break try: shutdown_event.wait(config.get("check_interval_sec", 60)) except Exception: # noqa: BLE001 break def main(): parser = argparse.ArgumentParser(description="ServerMonitor") parser.add_argument("--once", action="store_true", help="Run one cycle and exit") args = parser.parse_args() setup_logging() script_dir = os.path.dirname(os.path.abspath(__file__)) config_path = os.environ.get("CONFIG_FILE", os.path.join(script_dir, "config.json")) try: config = load_config(config_path) except Exception as exc: # noqa: BLE001 logger.error("Failed to load config from %s: %s", config_path, exc) sys.exit(1) apply_env_overrides(config) state_store = StateStore(config.get("state_file", "/data/state.json")) state_store.load() alert_engine = AlertEngine() signal.signal(signal.SIGTERM, shutdown_handler) signal.signal(signal.SIGINT, shutdown_handler) try: main_loop(config, state_store, alert_engine, once=args.once) except Exception as exc: # noqa: BLE001 logger.exception("Unexpected error in main loop: %s", exc) finally: state_store.save() logger.info("shutting down") if __name__ == "__main__": main()