Add live Prometheus telemetry endpoint
This commit is contained in:
Keith Casko
+23
-23
@@ -70,7 +70,7 @@
|
||||
<div class="overview-console">
|
||||
<div class="console-label">
|
||||
<span>INFRASTRUCTURE OVERVIEW</span>
|
||||
<span id="refresh-indicator">CONFIGURATION SNAPSHOT</span>
|
||||
<span id="refresh-indicator">CONNECTING TO TELEMETRY</span>
|
||||
</div>
|
||||
|
||||
<div class="identity-block">
|
||||
@@ -84,36 +84,36 @@
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="metric-board" aria-label="Infrastructure capabilities">
|
||||
<div class="metric-board" aria-label="Live infrastructure telemetry">
|
||||
<article class="metric metric-primary">
|
||||
<span>HOST METRICS</span>
|
||||
<strong>PROMETHEUS</strong>
|
||||
<span>CPU LOAD</span>
|
||||
<strong id="cpu-value" class="metric-value">--</strong><small>%</small>
|
||||
<p>PROMETHEUS / NODE EXPORTER</p>
|
||||
</article>
|
||||
<article class="metric">
|
||||
<span>MEMORY</span>
|
||||
<strong id="memory-value" class="metric-value">--</strong><small>%</small>
|
||||
<p>PROMETHEUS / NODE EXPORTER</p>
|
||||
</article>
|
||||
<article class="metric">
|
||||
<span>HOST UPTIME</span>
|
||||
<strong id="uptime-value" class="metric-value">--</strong>
|
||||
<p>NODE EXPORTER</p>
|
||||
</article>
|
||||
<article class="metric">
|
||||
<span>CONTAINER METRICS</span>
|
||||
<strong>CADVISOR</strong>
|
||||
<p>DOCKER TELEMETRY</p>
|
||||
<span>RUNNING CONTAINERS</span>
|
||||
<strong id="containers-value" class="metric-value">--</strong>
|
||||
<p>CADVISOR VIA PROMETHEUS</p>
|
||||
</article>
|
||||
<article class="metric">
|
||||
<span>AVAILABILITY MONITORING</span>
|
||||
<strong>KUMA</strong>
|
||||
<p>ENDPOINT CHECKS</p>
|
||||
<span>SCRAPE TARGETS</span>
|
||||
<strong id="targets-value" class="metric-value">--</strong>
|
||||
<p>PROMETHEUS TARGET HEALTH</p>
|
||||
</article>
|
||||
<article class="metric">
|
||||
<span>LOGS</span>
|
||||
<strong>LOKI</strong>
|
||||
<p>CENTRALIZED LOGGING</p>
|
||||
</article>
|
||||
<article class="metric">
|
||||
<span>DASHBOARDS</span>
|
||||
<strong>GRAFANA</strong>
|
||||
<p>OPERATIONS / ANALYSIS</p>
|
||||
</article>
|
||||
<article class="metric">
|
||||
<span>IDENTITY</span>
|
||||
<strong>AUTHENTIK</strong>
|
||||
<p>OIDC / SSO / 2FA</p>
|
||||
<span>TELEMETRY</span>
|
||||
<strong id="telemetry-value" class="metric-value">CONNECTING</strong>
|
||||
<p>PROMETHEUS / CADVISOR</p>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
location = /api/status {
|
||||
proxy_pass http://status-api:8080/status;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_read_timeout 4s;
|
||||
add_header Cache-Control "no-store, max-age=0" always;
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ =404;
|
||||
}
|
||||
|
||||
location ~ /. {
|
||||
deny all;
|
||||
return 404;
|
||||
}
|
||||
|
||||
location ^~ /status-api/ {
|
||||
return 404;
|
||||
}
|
||||
|
||||
location = /nginx.conf {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,17 @@
|
||||
const timeElement = document.querySelector("#local-time");
|
||||
const refreshElement = document.querySelector("#refresh-indicator");
|
||||
|
||||
const fields = {
|
||||
cpu: document.querySelector("#cpu-value"),
|
||||
memory: document.querySelector("#memory-value"),
|
||||
uptime: document.querySelector("#uptime-value"),
|
||||
containers: document.querySelector("#containers-value"),
|
||||
targets: document.querySelector("#targets-value"),
|
||||
telemetry: document.querySelector("#telemetry-value"),
|
||||
};
|
||||
|
||||
let lastSuccessfulUpdate = null;
|
||||
let requestInFlight = false;
|
||||
|
||||
function updateClock() {
|
||||
const now = new Date();
|
||||
@@ -11,5 +24,73 @@ function updateClock() {
|
||||
}).format(now);
|
||||
}
|
||||
|
||||
function formatUptime(totalSeconds) {
|
||||
const seconds = Math.max(0, Number(totalSeconds) || 0);
|
||||
const days = Math.floor(seconds / 86400);
|
||||
const hours = Math.floor((seconds % 86400) / 3600);
|
||||
const minutes = Math.floor((seconds % 3600) / 60);
|
||||
|
||||
if (days > 0) return days + "d " + hours + "h";
|
||||
if (hours > 0) return hours + "h " + minutes + "m";
|
||||
return minutes + "m";
|
||||
}
|
||||
|
||||
function setUnavailable() {
|
||||
for (const key of ["cpu", "memory", "uptime", "containers", "targets"]) {
|
||||
fields[key].textContent = "--";
|
||||
}
|
||||
fields.telemetry.textContent = "UNAVAILABLE";
|
||||
fields.telemetry.classList.add("metric-error");
|
||||
refreshElement.textContent = "TELEMETRY UNAVAILABLE";
|
||||
refreshElement.classList.add("telemetry-error");
|
||||
}
|
||||
|
||||
function renderStatus(data) {
|
||||
fields.cpu.textContent = Number(data.host.cpu_percent).toFixed(1);
|
||||
fields.memory.textContent = Number(data.host.memory_percent).toFixed(1);
|
||||
fields.uptime.textContent = formatUptime(data.host.uptime_seconds);
|
||||
fields.containers.textContent = String(data.containers.running);
|
||||
fields.targets.textContent = data.prometheus.up + "/" + data.prometheus.total;
|
||||
fields.telemetry.textContent = "LIVE";
|
||||
fields.telemetry.classList.remove("metric-error");
|
||||
refreshElement.classList.remove("telemetry-error");
|
||||
|
||||
lastSuccessfulUpdate = Date.now();
|
||||
updateRefreshLabel();
|
||||
}
|
||||
|
||||
function updateRefreshLabel() {
|
||||
if (!lastSuccessfulUpdate) return;
|
||||
const ageSeconds = Math.max(0, Math.floor((Date.now() - lastSuccessfulUpdate) / 1000));
|
||||
refreshElement.textContent = "LIVE TELEMETRY · UPDATED " + String(ageSeconds).padStart(2, "0") + "s AGO";
|
||||
}
|
||||
|
||||
async function refreshTelemetry() {
|
||||
if (requestInFlight) return;
|
||||
requestInFlight = true;
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/status", {
|
||||
cache: "no-store",
|
||||
headers: { "Accept": "application/json" },
|
||||
});
|
||||
|
||||
if (!response.ok) throw new Error("status " + response.status);
|
||||
const data = await response.json();
|
||||
if (!data.ok) throw new Error("telemetry unavailable");
|
||||
|
||||
renderStatus(data);
|
||||
} catch (error) {
|
||||
console.error("ByteGeist telemetry request failed:", error);
|
||||
setUnavailable();
|
||||
} finally {
|
||||
requestInFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
updateClock();
|
||||
refreshTelemetry();
|
||||
|
||||
window.setInterval(updateClock, 1000);
|
||||
window.setInterval(updateRefreshLabel, 1000);
|
||||
window.setInterval(refreshTelemetry, 15000);
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
FROM python:3.12-alpine
|
||||
WORKDIR /app
|
||||
COPY status_api.py /app/status_api.py
|
||||
USER nobody
|
||||
EXPOSE 8080
|
||||
CMD ["python", "/app/status_api.py"]
|
||||
@@ -0,0 +1,91 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
PROMETHEUS_URL = os.getenv("PROMETHEUS_URL", "http://host.docker.internal:9090").rstrip("/")
|
||||
TIMEOUT = float(os.getenv("PROMETHEUS_TIMEOUT", "2.5"))
|
||||
|
||||
QUERIES = {
|
||||
"cpu_percent": '100-(avg(rate(node_cpu_seconds_total{mode="idle"}[2m]))*100)',
|
||||
"memory_percent": '(1-(node_memory_MemAvailable_bytes/node_memory_MemTotal_bytes))*100',
|
||||
"host_uptime_seconds": 'time()-node_boot_time_seconds',
|
||||
"running_containers": 'count(count by (name) (container_last_seen{name!=""}))',
|
||||
}
|
||||
|
||||
def get_json(url):
|
||||
request = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
|
||||
return json.load(response)
|
||||
|
||||
def prometheus_query(expression):
|
||||
url = f"{PROMETHEUS_URL}/api/v1/query?{urllib.parse.urlencode({'query': expression})}"
|
||||
payload = get_json(url)
|
||||
if payload.get("status") != "success":
|
||||
raise RuntimeError("Prometheus query failed")
|
||||
results = payload.get("data", {}).get("result", [])
|
||||
if not results:
|
||||
raise RuntimeError("Prometheus query returned no data")
|
||||
return float(results[0]["value"][1])
|
||||
|
||||
def target_health():
|
||||
payload = get_json(f"{PROMETHEUS_URL}/api/v1/targets")
|
||||
targets = payload.get("data", {}).get("activeTargets", [])
|
||||
jobs = {}
|
||||
for target in targets:
|
||||
job = target.get("labels", {}).get("job", "unknown")
|
||||
jobs[job] = target.get("health") == "up"
|
||||
return {
|
||||
"up": sum(1 for target in targets if target.get("health") == "up"),
|
||||
"total": len(targets),
|
||||
"jobs": jobs,
|
||||
}
|
||||
|
||||
def build_status():
|
||||
values = {name: prometheus_query(query) for name, query in QUERIES.items()}
|
||||
targets = target_health()
|
||||
return {
|
||||
"ok": True,
|
||||
"source": "Prometheus / node_exporter / cAdvisor",
|
||||
"sampled_at": int(time.time()),
|
||||
"host": {
|
||||
"cpu_percent": round(values["cpu_percent"], 1),
|
||||
"memory_percent": round(values["memory_percent"], 1),
|
||||
"uptime_seconds": int(values["host_uptime_seconds"]),
|
||||
},
|
||||
"containers": {"running": int(values["running_containers"])},
|
||||
"prometheus": targets,
|
||||
}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def send_json(self, status_code, payload):
|
||||
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
self.send_response(status_code)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Cache-Control", "no-store, max-age=0")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/healthz":
|
||||
self.send_json(200, {"ok": True})
|
||||
return
|
||||
if self.path != "/status":
|
||||
self.send_json(404, {"ok": False, "error": "not_found"})
|
||||
return
|
||||
try:
|
||||
self.send_json(200, build_status())
|
||||
except Exception as exc:
|
||||
self.send_json(503, {"ok": False, "sampled_at": int(time.time()), "error": type(exc).__name__})
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
print(f"{self.address_string()} - {fmt % args}", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
server = ThreadingHTTPServer(("0.0.0.0", 8080), Handler)
|
||||
print("ByteGeist status API listening on :8080", flush=True)
|
||||
server.serve_forever()
|
||||
@@ -384,6 +384,19 @@ h1 {
|
||||
color: var(--green);
|
||||
}
|
||||
|
||||
.metric strong.metric-value {
|
||||
display: inline-block;
|
||||
margin-top: 16px;
|
||||
font: 500 clamp(1.35rem, 1.8vw, 2.15rem)/1 var(--mono);
|
||||
letter-spacing: -.045em;
|
||||
overflow-wrap: normal;
|
||||
}
|
||||
|
||||
.metric .metric-error,
|
||||
.telemetry-error {
|
||||
color: var(--amber) !important;
|
||||
}
|
||||
|
||||
.spark-bars {
|
||||
height: 23px;
|
||||
margin-top: 12px;
|
||||
|
||||
Reference in New Issue
Block a user