280 lines
11 KiB
Python
280 lines
11 KiB
Python
|
"""
|
|||
|
|
status_engine.py — Analyse les métriques reçues de l'agent.
|
|||
|
|
Détermine le statut global, la cause des ralentissements et une ETA.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from collections import deque
|
|||
|
|
from dataclasses import dataclass, field
|
|||
|
|
from typing import Optional
|
|||
|
|
import time
|
|||
|
|
|
|||
|
|
# ─── Constantes ───────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
STATUS_OK = "ok"
|
|||
|
|
STATUS_WARNING = "warning"
|
|||
|
|
STATUS_CRITICAL = "critical"
|
|||
|
|
|
|||
|
|
THRESHOLDS = {
|
|||
|
|
"cpu": {"warning": 70, "critical": 85},
|
|||
|
|
"ram": {"warning": 80, "critical": 90},
|
|||
|
|
"swap": {"warning": 40, "critical": 70},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Noms affichables pour chaque service
|
|||
|
|
SERVICE_DISPLAY = {
|
|||
|
|
"minecraft": "⛏️ Minecraft",
|
|||
|
|
"hytale": "🎮 Hytale",
|
|||
|
|
"satisfactory": "🏭 Satisfactory",
|
|||
|
|
"valheim": "🪓 Valheim",
|
|||
|
|
"terraria": "🌳 Terraria",
|
|||
|
|
"ark": "🦕 ARK",
|
|||
|
|
"ollama": "🤖 Ollama (IA)",
|
|||
|
|
"discord_bots": "🤖 Bots Discord",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
HISTORY_MAXLEN = 24 # 24 × 15s = 6 minutes de mémoire glissante
|
|||
|
|
PUSH_INTERVAL = 5 # secondes (doit correspondre à l'agent)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ─── Data classes ─────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
@dataclass
|
|||
|
|
class ServiceStatus:
|
|||
|
|
key: str
|
|||
|
|
name: str
|
|||
|
|
running: bool
|
|||
|
|
cpu: float = 0.0
|
|||
|
|
ram_mb: float = 0.0
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def ram_gb(self) -> float:
|
|||
|
|
return round(self.ram_mb / 1024, 2)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass
|
|||
|
|
class AnalysisResult:
|
|||
|
|
status: str # ok / warning / critical
|
|||
|
|
status_emoji: str
|
|||
|
|
cpu: float
|
|||
|
|
ram_pct: float
|
|||
|
|
ram_used_gb: float
|
|||
|
|
ram_total_gb: float
|
|||
|
|
swap_pct: float
|
|||
|
|
services: list[ServiceStatus] # triés par impact décroissant
|
|||
|
|
cause_lines: list[str] # explication humaine
|
|||
|
|
eta_text: str # estimation retour à la normale
|
|||
|
|
load_avg: tuple[float, float, float]
|
|||
|
|
cpu_temp: Optional[float]
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ─── Moteur ───────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
class StatusEngine:
|
|||
|
|
|
|||
|
|
def __init__(self):
|
|||
|
|
self._history: deque[dict] = deque(maxlen=HISTORY_MAXLEN)
|
|||
|
|
self.last_metrics: Optional[dict] = None
|
|||
|
|
self.seen_services: set[str] = set()
|
|||
|
|
|
|||
|
|
# ── Ingest ────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def push(self, metrics: dict):
|
|||
|
|
"""Enregistre un snapshot de métriques."""
|
|||
|
|
self._history.append(metrics)
|
|||
|
|
self.last_metrics = metrics
|
|||
|
|
|
|||
|
|
# On mémorise quels services ont été vus "actifs" au moins une fois
|
|||
|
|
raw_services = metrics.get("services", {})
|
|||
|
|
for key, data in raw_services.items():
|
|||
|
|
if data.get("running"):
|
|||
|
|
self.seen_services.add(key)
|
|||
|
|
|
|||
|
|
# ── Analyse ───────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def analyse(self) -> Optional[AnalysisResult]:
|
|||
|
|
if not self.last_metrics:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
m = self.last_metrics
|
|||
|
|
sys = m.get("system", {})
|
|||
|
|
|
|||
|
|
cpu = sys.get("cpu_percent", 0.0)
|
|||
|
|
ram_pct = sys.get("ram_percent", 0.0)
|
|||
|
|
ram_used = sys.get("ram_used_gb", 0.0)
|
|||
|
|
ram_total = sys.get("ram_total_gb", 1.0)
|
|||
|
|
swap_pct = sys.get("swap_percent", 0.0)
|
|||
|
|
load = (sys.get("load_avg_1m", 0.0),
|
|||
|
|
sys.get("load_avg_5m", 0.0),
|
|||
|
|
sys.get("load_avg_15m", 0.0))
|
|||
|
|
cpu_temp = sys.get("cpu_c", None)
|
|||
|
|
|
|||
|
|
status = self._compute_status(cpu, ram_pct, swap_pct)
|
|||
|
|
|
|||
|
|
services = self._parse_services(m.get("services", {}))
|
|||
|
|
causes = self._build_causes(cpu, ram_pct, ram_used, ram_total, swap_pct, services)
|
|||
|
|
eta = self._estimate_eta(status)
|
|||
|
|
|
|||
|
|
emoji_map = {
|
|||
|
|
STATUS_OK: "🟢",
|
|||
|
|
STATUS_WARNING: "🟡",
|
|||
|
|
STATUS_CRITICAL: "🔴",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return AnalysisResult(
|
|||
|
|
status=status,
|
|||
|
|
status_emoji=emoji_map[status],
|
|||
|
|
cpu=cpu,
|
|||
|
|
ram_pct=ram_pct,
|
|||
|
|
ram_used_gb=ram_used,
|
|||
|
|
ram_total_gb=ram_total,
|
|||
|
|
swap_pct=swap_pct,
|
|||
|
|
services=services,
|
|||
|
|
cause_lines=causes,
|
|||
|
|
eta_text=eta,
|
|||
|
|
load_avg=load,
|
|||
|
|
cpu_temp=cpu_temp,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# ── Méthodes internes ─────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def _compute_status(self, cpu: float, ram: float, swap: float) -> str:
|
|||
|
|
if (cpu >= THRESHOLDS["cpu"]["critical"]
|
|||
|
|
or ram >= THRESHOLDS["ram"]["critical"]
|
|||
|
|
or swap >= THRESHOLDS["swap"]["critical"]):
|
|||
|
|
return STATUS_CRITICAL
|
|||
|
|
if (cpu >= THRESHOLDS["cpu"]["warning"]
|
|||
|
|
or ram >= THRESHOLDS["ram"]["warning"]
|
|||
|
|
or swap >= THRESHOLDS["swap"]["warning"]):
|
|||
|
|
return STATUS_WARNING
|
|||
|
|
return STATUS_OK
|
|||
|
|
|
|||
|
|
def _parse_services(self, raw: dict) -> list[ServiceStatus]:
|
|||
|
|
result = []
|
|||
|
|
for key, data in raw.items():
|
|||
|
|
running = data.get("running", False)
|
|||
|
|
|
|||
|
|
# On ignore les services qui n'ont jamais été vus actifs sur ce serveur
|
|||
|
|
if not running and key not in self.seen_services:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
svc = ServiceStatus(
|
|||
|
|
key=key,
|
|||
|
|
name=SERVICE_DISPLAY.get(key, key.capitalize()),
|
|||
|
|
running=running,
|
|||
|
|
cpu=round(data.get("cpu", 0.0), 1),
|
|||
|
|
ram_mb=round(data.get("ram_mb", 0.0), 1),
|
|||
|
|
)
|
|||
|
|
result.append(svc)
|
|||
|
|
|
|||
|
|
# Tri : services actifs en premier, puis par impact (CPU + RAM normalisée)
|
|||
|
|
result.sort(key=lambda s: (
|
|||
|
|
not s.running,
|
|||
|
|
-(s.cpu + s.ram_mb / 100)
|
|||
|
|
))
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
def _build_causes(
|
|||
|
|
self,
|
|||
|
|
cpu: float, ram_pct: float, ram_used: float, ram_total: float,
|
|||
|
|
swap_pct: float, services: list[ServiceStatus]
|
|||
|
|
) -> list[str]:
|
|||
|
|
lines = []
|
|||
|
|
running = [s for s in services if s.running]
|
|||
|
|
|
|||
|
|
# ── Problème CPU ──
|
|||
|
|
if cpu >= THRESHOLDS["cpu"]["warning"]:
|
|||
|
|
top = max(running, key=lambda s: s.cpu, default=None)
|
|||
|
|
if top and top.cpu > 5:
|
|||
|
|
lines.append(
|
|||
|
|
f"CPU à **{cpu:.0f}%** — principal consommateur : "
|
|||
|
|
f"**{top.name}** ({top.cpu:.0f}%)"
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
lines.append(f"CPU à **{cpu:.0f}%** — charge élevée non attribuée à un service connu")
|
|||
|
|
|
|||
|
|
# ── Problème RAM ──
|
|||
|
|
if ram_pct >= THRESHOLDS["ram"]["warning"]:
|
|||
|
|
top = max(running, key=lambda s: s.ram_mb, default=None)
|
|||
|
|
if top and top.ram_mb > 200:
|
|||
|
|
lines.append(
|
|||
|
|
f"RAM à **{ram_pct:.0f}%** ({ram_used:.1f}/{ram_total:.1f} GB) — "
|
|||
|
|
f"**{top.name}** occupe {top.ram_gb:.1f} GB"
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
lines.append(
|
|||
|
|
f"RAM à **{ram_pct:.0f}%** ({ram_used:.1f}/{ram_total:.1f} GB)"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# ── Swap ──
|
|||
|
|
if swap_pct >= THRESHOLDS["swap"]["warning"]:
|
|||
|
|
lines.append(
|
|||
|
|
f"Swap à **{swap_pct:.0f}%** — le serveur manque de RAM "
|
|||
|
|
f"et utilise le disque comme mémoire d'appoint (ralentissement fort)"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# ── Combinaison Ollama + jeux ──
|
|||
|
|
ollama = next((s for s in running if s.key == "ollama"), None)
|
|||
|
|
games = [s for s in running if s.key not in ("ollama", "discord_bots")]
|
|||
|
|
if ollama and games and (cpu >= THRESHOLDS["cpu"]["critical"] or ram_pct >= THRESHOLDS["ram"]["critical"]):
|
|||
|
|
game_names = ", ".join(s.name for s in games)
|
|||
|
|
lines.append(
|
|||
|
|
f"💡 Conflit potentiel : **{ollama.name}** (IA) et **{game_names}** "
|
|||
|
|
f"semblent saturer les ressources"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return lines if lines else ["✅ Aucune anomalie détectée"]
|
|||
|
|
|
|||
|
|
def _estimate_eta(self, current_status: str) -> str:
|
|||
|
|
if current_status == STATUS_OK:
|
|||
|
|
return "✅ Tout est normal"
|
|||
|
|
|
|||
|
|
if len(self._history) < 4:
|
|||
|
|
return "⏳ Collecte en cours, ETA disponible dans quelques mesures…"
|
|||
|
|
|
|||
|
|
scores = [self._pressure_score(m) for m in self._history]
|
|||
|
|
|
|||
|
|
# Tendance : on compare la moyenne des 4 premières mesures vs les 4 dernières
|
|||
|
|
n = min(4, len(scores) // 2)
|
|||
|
|
old_avg = sum(scores[:n]) / n
|
|||
|
|
new_avg = sum(scores[-n:]) / n
|
|||
|
|
|
|||
|
|
delta = old_avg - new_avg # positif = amélioration
|
|||
|
|
|
|||
|
|
if delta > 0.08:
|
|||
|
|
# Le serveur s'améliore : on extrapole
|
|||
|
|
current = scores[-1]
|
|||
|
|
if current <= 1.0:
|
|||
|
|
return "⏱️ Retour à la normale **imminent**"
|
|||
|
|
|
|||
|
|
rate_per_step = delta / max(len(scores) - n, 1)
|
|||
|
|
if rate_per_step <= 0:
|
|||
|
|
return "📊 Situation stable — ETA indéterminée"
|
|||
|
|
|
|||
|
|
steps = (current - 1.0) / rate_per_step
|
|||
|
|
secs = int(steps * PUSH_INTERVAL)
|
|||
|
|
|
|||
|
|
if secs < 60:
|
|||
|
|
return "⏱️ Retour à la normale dans **moins d'1 minute**"
|
|||
|
|
elif secs < 3600:
|
|||
|
|
return f"⏱️ Retour à la normale dans **~{secs // 60} min**"
|
|||
|
|
else:
|
|||
|
|
h, m2 = divmod(secs // 60, 60)
|
|||
|
|
return f"⏱️ Retour à la normale dans **~{h}h{m2:02d}**"
|
|||
|
|
|
|||
|
|
elif delta < -0.08:
|
|||
|
|
return "⚠️ La charge **augmente** — ETA indéterminée, surveillez l'évolution"
|
|||
|
|
else:
|
|||
|
|
return "📊 Charge **stable** — ETA indéterminée (activité constante)"
|
|||
|
|
|
|||
|
|
def _pressure_score(self, m: dict) -> float:
|
|||
|
|
"""Score de pression normalisé — 1.0 = seuil critique."""
|
|||
|
|
sys = m.get("system", {})
|
|||
|
|
cpu = sys.get("cpu_percent", 0) / THRESHOLDS["cpu"]["critical"]
|
|||
|
|
ram = sys.get("ram_percent", 0) / THRESHOLDS["ram"]["critical"]
|
|||
|
|
swap = sys.get("swap_percent", 0) / THRESHOLDS["swap"]["critical"]
|
|||
|
|
return round(max(cpu, ram, swap), 3)
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def has_data(self) -> bool:
|
|||
|
|
return self.last_metrics is not None
|