Add server-side health checks for service online status
- update-health.js: pings every service URL from script.js (single source of truth) in parallel every minute via cron and writes health.json; any HTTP response below 500 counts as online, 5xx as error, timeouts/DNS failures as offline; TLS validity ignored - Card badges now show real state instead of a static Online label: grey Checking, green Online (with latency tooltip), amber Error (HTTP code), red Offline (reason); refreshed every 30 s - Entrypoint runs a first sweep at boot; nginx serves /health.json with no-cache; README documents the behavior
This commit is contained in:
+2
-2
@@ -12,8 +12,8 @@ COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||
# Backend scripts (node_modules included so the build works offline).
|
||||
COPY api /opt/dashboard
|
||||
|
||||
# Cron refreshes the data feed every 15 minutes.
|
||||
RUN echo '*/15 * * * * cd /opt/dashboard && node update-data.js' > /etc/crontabs/root
|
||||
# Cron refreshes the data feed every 15 minutes and health checks every minute.
|
||||
RUN printf '*/15 * * * * cd /opt/dashboard && node update-data.js\n* * * * * cd /opt/dashboard && node update-health.js\n' > /etc/crontabs/root
|
||||
|
||||
# Entrypoint: run updater once, start API proxy + cron, then nginx.
|
||||
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
||||
|
||||
@@ -21,7 +21,9 @@ External: gitlab-ixsol, gem360.
|
||||
with their GPU/CPU split)
|
||||
- Inline **Open WebUI prompt line with model selection** in the Open WebUI card
|
||||
(requires `OPENWEBUI_TOKEN`)
|
||||
- Responsive grid of service cards with icons, categories and a live "Online" indicator
|
||||
- Responsive grid of service cards with icons, categories and **real online status**
|
||||
(server-side health checks every minute: green = online, amber = 5xx error,
|
||||
red = unreachable; hover the badge for latency/details)
|
||||
- **Open** (new tab) + **Copy URL** per service
|
||||
- Live search / filter across names, categories and URLs
|
||||
- Live local clock
|
||||
@@ -101,9 +103,12 @@ docker run -d --name homelab-dashboard -p 8888:80 --restart unless-stopped homel
|
||||
|
||||
## Notes
|
||||
|
||||
- The "Online" badge is cosmetic (a static site can't probe your services from the browser
|
||||
due to CORS). If you want real health checks later, a tiny backend that pings each URL
|
||||
would be the next step.
|
||||
- Online badges are real health checks: `api/update-health.js` (cron, every minute)
|
||||
pings each service URL from the container with a 5 s timeout and writes
|
||||
`health.json`; the frontend refreshes badges every 30 s. Any HTTP response below
|
||||
500 counts as online (401/404 still means the server is up); 5xx shows as
|
||||
"Error", timeouts/DNS failures as "Offline". TLS certificate validity is ignored
|
||||
(reachability only).
|
||||
- Calendar dependencies are bundled in `api/node_modules` so the Docker build works
|
||||
offline. To update them, run `npm install` in the `api/` directory.
|
||||
- Port 8888 is published by the Compose file; change it there if needed.
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
// ---------------------------------------------------------------------------
|
||||
// Health checks for all dashboard services.
|
||||
// Reads the service URLs from script.js (single source of truth), pings each
|
||||
// one in parallel and writes health.json. Scheduled every minute by cron.
|
||||
//
|
||||
// Status classification:
|
||||
// online - got any HTTP response below 500 (401/404 still means "up")
|
||||
// error - server responded with 5xx
|
||||
// offline - timeout / connection refused / DNS failure
|
||||
// TLS certificate validity is ignored (reachability check only).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const http = require("http");
|
||||
const https = require("https");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { URL } = require("url");
|
||||
|
||||
const OUT_FILE = process.env.OUT_FILE || "/usr/share/nginx/html/health.json";
|
||||
const TIMEOUT = parseInt(process.env.HEALTH_TIMEOUT || "5000", 10);
|
||||
|
||||
// Reachability only — don't fail on self-signed certificates.
|
||||
const httpAgent = new http.Agent({ rejectUnauthorized: false });
|
||||
const httpsAgent = new https.Agent({ rejectUnauthorized: false });
|
||||
|
||||
function serviceUrls() {
|
||||
const candidates = [
|
||||
process.env.SERVICES_JS,
|
||||
"/usr/share/nginx/html/script.js",
|
||||
path.join(__dirname, "..", "script.js"),
|
||||
].filter(Boolean);
|
||||
for (const p of candidates) {
|
||||
try {
|
||||
const src = fs.readFileSync(p, "utf8");
|
||||
const urls = [...src.matchAll(/url: *"(https?:\/\/[^"]+)"/g)].map((m) => m[1]);
|
||||
if (urls.length) return [...new Set(urls)];
|
||||
} catch {
|
||||
// try next candidate
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function checkUrl(url) {
|
||||
return new Promise((resolve) => {
|
||||
const started = Date.now();
|
||||
let settled = false;
|
||||
const done = (result) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
let req;
|
||||
try {
|
||||
const u = new URL(url);
|
||||
const client = u.protocol === "https:" ? https : http;
|
||||
req = client.request(
|
||||
u,
|
||||
{
|
||||
method: "GET",
|
||||
agent: u.protocol === "https:" ? httpsAgent : httpAgent,
|
||||
timeout: TIMEOUT,
|
||||
},
|
||||
(res) => {
|
||||
const code = res.statusCode;
|
||||
res.destroy(); // response headers are all we need
|
||||
done({
|
||||
status: code >= 500 ? "error" : "online",
|
||||
code,
|
||||
latencyMs: Date.now() - started,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
);
|
||||
} catch (e) {
|
||||
return done({ status: "offline", code: null, latencyMs: null, error: e.message });
|
||||
}
|
||||
|
||||
req.on("timeout", () => {
|
||||
req.destroy();
|
||||
done({ status: "offline", code: null, latencyMs: null, error: "Timeout" });
|
||||
});
|
||||
req.on("error", (e) => {
|
||||
done({ status: "offline", code: null, latencyMs: null, error: e.message });
|
||||
});
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const urls = serviceUrls();
|
||||
if (urls.length === 0) {
|
||||
console.error("No service URLs found — is script.js available?");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const results = await Promise.all(urls.map(async (u) => [u, await checkUrl(u)]));
|
||||
const services = Object.fromEntries(results);
|
||||
|
||||
const payload = { updatedAt: new Date().toISOString(), services };
|
||||
fs.writeFileSync(OUT_FILE, JSON.stringify(payload, null, 2));
|
||||
|
||||
const count = (s) => results.filter(([, r]) => r.status === s).length;
|
||||
console.log(`Health: ${urls.length} services — ${count("online")} online, ${count("error")} error, ${count("offline")} offline`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -13,5 +13,8 @@ node /opt/dashboard/server.js &
|
||||
# Run the initial data fetch in the background; cron keeps it fresh after that.
|
||||
( cd /opt/dashboard && node update-data.js ) &
|
||||
|
||||
# First health-check sweep in the background; cron re-runs it every minute.
|
||||
( cd /opt/dashboard && node update-health.js ) &
|
||||
|
||||
# Start nginx in the foreground.
|
||||
exec nginx -g "daemon off;"
|
||||
@@ -13,6 +13,11 @@ server {
|
||||
add_header Cache-Control "no-cache" always;
|
||||
}
|
||||
|
||||
# Service health checks; refreshed every minute by cron.
|
||||
location = /health.json {
|
||||
add_header Cache-Control "no-cache" always;
|
||||
}
|
||||
|
||||
# Open WebUI chat proxy.
|
||||
location /api/chat {
|
||||
proxy_pass http://127.0.0.1:3001;
|
||||
|
||||
@@ -424,7 +424,10 @@ function cardTemplate(s, idx) {
|
||||
style="--cat:${meta.color}; animation-delay:${Math.min(idx * 35, 400)}ms">
|
||||
<div class="card-head">
|
||||
<div class="card-icon" aria-hidden="true">${s.icon || "🔗"}</div>
|
||||
<span class="status"><span class="dot"></span>Online</span>
|
||||
<span class="status" data-status-url="${escapeHtml(s.url)}">
|
||||
<span class="dot" aria-hidden="true"></span>
|
||||
<span class="status-text">Checking…</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<span class="card-cat">${escapeHtml(meta.label)}</span>
|
||||
@@ -555,6 +558,51 @@ function bindOpenWebUIChat() {
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Service health badges (server-side checks via health.json)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function applyHealth(health) {
|
||||
const services = health?.services || {};
|
||||
document.querySelectorAll(".status[data-status-url]").forEach((badge) => {
|
||||
const url = badge.getAttribute("data-status-url");
|
||||
const h = services[url];
|
||||
const text = badge.querySelector(".status-text");
|
||||
if (!text) return;
|
||||
|
||||
if (!h) {
|
||||
badge.className = "status unknown";
|
||||
text.textContent = "Unknown";
|
||||
badge.title = "No health data yet";
|
||||
return;
|
||||
}
|
||||
|
||||
if (h.status === "online") {
|
||||
badge.className = "status online";
|
||||
text.textContent = "Online";
|
||||
badge.title = h.latencyMs != null ? `${h.latencyMs} ms` : "";
|
||||
} else if (h.status === "error") {
|
||||
badge.className = "status error";
|
||||
text.textContent = "Error";
|
||||
badge.title = `HTTP ${h.code}`;
|
||||
} else {
|
||||
badge.className = "status offline";
|
||||
text.textContent = "Offline";
|
||||
badge.title = h.error || "Unreachable";
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function loadHealth() {
|
||||
try {
|
||||
const res = await fetch("health.json", { cache: "no-store" });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
applyHealth(await res.json());
|
||||
} catch {
|
||||
// keep the current badges; the next poll will retry
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Search / filter
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -614,3 +662,5 @@ loadGpuStats();
|
||||
setInterval(loadGpuStats, 5000); // live GPU stats every 5 s
|
||||
loadOllamaStats();
|
||||
setInterval(loadOllamaStats, 5000); // live Ollama loaded models every 5 s
|
||||
loadHealth();
|
||||
setInterval(loadHealth, 30000); // service health badges every 30 s
|
||||
|
||||
@@ -301,11 +301,24 @@ main#app {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
flex: none;
|
||||
background: var(--muted-2); /* unknown / checking */
|
||||
}
|
||||
|
||||
.status.online .dot {
|
||||
background: var(--online);
|
||||
box-shadow: 0 0 0 3px color-mix(in srgb, var(--online) 18%, transparent);
|
||||
animation: pulse 2.4s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.status.offline .dot {
|
||||
background: var(--error_color, #e0554a);
|
||||
}
|
||||
|
||||
.status.error .dot {
|
||||
background: var(--warning_color, #f2a944);
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0%,
|
||||
100% {
|
||||
|
||||
Reference in New Issue
Block a user