ServerMonitor/monitor.py

"""ServerMonitor main loop.

Runs periodic health checks and sends Telegram alerts on failures, reminders
while services stay down, and recovery messages when they come back.
"""

import argparse
import json
import logging
import os
import signal
import sys
import threading
import time
from urllib.parse import urlparse, urlunparse

from checks import http_check, netdata_alarms, netdata_metrics, ping_check, tcp_check
from notify import AlertEngine, StateStore, escape_html, send_telegram

logger = logging.getLogger("servermonitor.monitor")

shutdown_event = threading.Event()


def load_config(path):
    """Load the JSON configuration file."""
    with open(path, "r", encoding="utf-8") as f:
        return json.load(f)


def _replace_host_in_url(url, new_host):
    """Replace the netloc host in a URL without changing the port or path."""
    parsed = urlparse(url)
    # Preserve user:pass and port; replace only the hostname.
    netloc = new_host
    if parsed.port is not None:
        netloc = f"{new_host}:{parsed.port}"
    if parsed.username is not None:
        prefix = f"{parsed.username}"
        if parsed.password is not None:
            prefix += f":{parsed.password}"
        netloc = f"{prefix}@{netloc}"
    return urlunparse(parsed._replace(netloc=netloc))


def apply_env_overrides(config):
    """Override config values from environment variables."""
    new_host = None
    if "HOST" in os.environ:
        new_host = os.environ["HOST"]
        config["host"] = new_host

    if "CHECK_INTERVAL_SEC" in os.environ:
        try:
            config["check_interval_sec"] = int(os.environ["CHECK_INTERVAL_SEC"])
        except ValueError:
            logger.warning("Invalid CHECK_INTERVAL_SEC, using config value")
    if "TELEGRAM_BOT_TOKEN" in os.environ:
        config["telegram"]["bot_token"] = os.environ["TELEGRAM_BOT_TOKEN"]
    if "TELEGRAM_CHAT_ID" in os.environ:
        config["telegram"]["chat_id"] = os.environ["TELEGRAM_CHAT_ID"]
    if "STATE_FILE" in os.environ:
        config["state_file"] = os.environ["STATE_FILE"]

    if new_host:
        for check in config.get("http_checks", []):
            check["url"] = _replace_host_in_url(check.get("url", ""), new_host)
        for check in config.get("tcp_checks", []):
            check["host"] = new_host
        netdata_cfg = config.get("netdata", {})
        if netdata_cfg.get("base_url"):
            netdata_cfg["base_url"] = _replace_host_in_url(netdata_cfg["base_url"], new_host)


def setup_logging():
    """Configure logging to stdout."""
    logging.basicConfig(
        stream=sys.stdout,
        format="%(asctime)s %(levelname)s %(message)s",
        level=logging.INFO,
    )


def shutdown_handler(signum, frame):
    """Handle SIGTERM/SIGINT gracefully."""
    logger.info("shutting down")
    shutdown_event.set()


def run_checks(config):
    """Run every configured check and return a list of results.

    Each result is a tuple ``(name, ok, message)``.
    """
    host = config["host"]
    timeout = config.get("timeout_sec", 5)
    results = []

    if config.get("ping", {}).get("enabled", True):
        ok, msg = ping_check(host, timeout)
        results.append(("ping", ok, msg))

    for check in config.get("http_checks", []):
        ok, msg = http_check(
            check["name"],
            check["url"],
            check.get("expected", [200]),
            timeout,
        )
        results.append((check["name"], ok, msg))

    for check in config.get("tcp_checks", []):
        ok, msg = tcp_check(
            check["name"],
            check.get("host", host),
            check["port"],
            timeout,
        )
        results.append((check["name"], ok, msg))

    netdata_cfg = config.get("netdata", {})
    base_url = netdata_cfg.get("base_url", f"http://{host}:19999")

    if netdata_cfg.get("check_alarms", True):
        ok, msg = netdata_alarms(base_url, timeout)
        results.append(("netdata_alarms", ok, msg))

    if netdata_cfg.get("check_metrics", True):
        metrics = netdata_metrics(base_url, timeout)
        thresholds = config.get("thresholds", {})

        cpu = metrics.get("cpu_percent")
        cpu_threshold = thresholds.get("cpu_percent", 90)
        if cpu is not None and cpu > cpu_threshold:
            results.append((
                "CPU",
                False,
                f"CPU {cpu:.1f}% (threshold {cpu_threshold}%)",
            ))
        elif cpu is not None:
            results.append(("CPU", True, f"CPU {cpu:.1f}%"))

        ram_used = metrics.get("ram_used_mb")
        ram_total = metrics.get("ram_total_mb")
        ram_threshold = thresholds.get("ram_avail_mb", 2048)
        if ram_used is not None and ram_total is not None:
            ram_avail = ram_total - ram_used
            if ram_avail < ram_threshold:
                results.append((
                    "RAM",
                    False,
                    f"RAM available {ram_avail:.0f}MB (threshold {ram_threshold}MB)",
                ))
            else:
                results.append(("RAM", True, f"RAM available {ram_avail:.0f}MB"))

        disk = metrics.get("disk_used_percent")
        disk_threshold = thresholds.get("disk_percent", 90)
        if disk is not None and disk > disk_threshold:
            results.append((
                "Disk",
                False,
                f"Disk {disk:.1f}% used (threshold {disk_threshold}%)",
            ))
        elif disk is not None:
            results.append(("Disk", True, f"Disk {disk:.1f}% used"))

    return results


def format_message(host, name, ok, message, is_recovery, alert_count):
    """Format a Telegram HTML alert message."""
    safe_name = escape_html(name)
    safe_message = escape_html(message)
    safe_host = escape_html(host)

    if is_recovery:
        header = f"✅ <b>Recovered</b> · {safe_host}"
        body = f"🟢 <b>{safe_name}</b>: {safe_message}"
        return f"{header}\n\n{body}"

    if ok:
        # Should not normally alert on OK, but handle it gracefully.
        header = f"✅ <b>Recovered</b> · {safe_host}"
        body = f"🟢 <b>{safe_name}</b>: {safe_message}"
        return f"{header}\n\n{body}"

    if name == "netdata_alarms":
        header = f"⚠️ <b>Server alert</b> · {safe_host}"
        body = f"🔴 <b>Netdata alarms</b>: {safe_message}"
        return f"{header}\n\n{body}"

    if alert_count > 1:
        header = f"⏰ <b>Still down</b> · {safe_host}"
        body = f"🔴 <b>{safe_name}</b>: {safe_message}"
        return f"{header}\n\n{body}"

    header = f"⚠️ <b>Server alert</b> · {safe_host}"
    body = f"🔴 <b>{safe_name}</b>: {safe_message}"
    return f"{header}\n\n{body}"


def main_loop(config, state_store, alert_engine, once=False):
    """Run the monitoring loop until shutdown or a single cycle if ``once``."""
    reminder_interval = config.get("reminder_interval_sec", 1800)
    host = config["host"]
    bot_token = config["telegram"].get("bot_token", "")
    chat_id = config["telegram"].get("chat_id", "")
    timeout = config.get("timeout_sec", 5)

    baseline = not os.path.exists(state_store.path)

    while not shutdown_event.is_set():
        now = time.time()
        results = run_checks(config)

        for name, ok, message in results:
            logger.info("%s %s (%s)", name, "OK" if ok else "FAIL", message)

            current_state = "OK" if ok else "CRIT"
            alert_engine.set_state(name, current_state)
            should_send, is_recovery = alert_engine.should_alert(
                name, now, state_store, reminder_interval
            )

            if baseline:
                # First run with no state file: establish baseline, record state,
                # do not alert. alert_count stays 0 so the next real failure is
                # treated as the first alert.
                state_store.set(name, current_state, 0.0, 0)
                continue

            if should_send:
                state_entry = state_store.get(name)
                alert_count = state_entry["alert_count"]
                last_alert_ts = state_entry["last_alert_ts"]
                text = format_message(
                    host,
                    name,
                    ok,
                    message,
                    is_recovery,
                    alert_count,
                )
                logger.warning("Alert: %s", text.replace("\n", " "))
                send_telegram(bot_token, chat_id, text, timeout)

        baseline = False
        state_store.save()

        if once:
            break

        try:
            shutdown_event.wait(config.get("check_interval_sec", 60))
        except Exception:  # noqa: BLE001
            break


def main():
    parser = argparse.ArgumentParser(description="ServerMonitor")
    parser.add_argument("--once", action="store_true", help="Run one cycle and exit")
    args = parser.parse_args()

    setup_logging()

    script_dir = os.path.dirname(os.path.abspath(__file__))
    config_path = os.environ.get("CONFIG_FILE", os.path.join(script_dir, "config.json"))

    try:
        config = load_config(config_path)
    except Exception as exc:  # noqa: BLE001
        logger.error("Failed to load config from %s: %s", config_path, exc)
        sys.exit(1)

    apply_env_overrides(config)

    state_store = StateStore(config.get("state_file", "/data/state.json"))
    state_store.load()

    alert_engine = AlertEngine()

    signal.signal(signal.SIGTERM, shutdown_handler)
    signal.signal(signal.SIGINT, shutdown_handler)

    try:
        main_loop(config, state_store, alert_engine, once=args.once)
    except Exception as exc:  # noqa: BLE001
        logger.exception("Unexpected error in main loop: %s", exc)
    finally:
        state_store.save()
        logger.info("shutting down")


if __name__ == "__main__":
    main()