feat: P0 通知（邮件 + MeoW）与 TLS 证书监控，事件时间线与维护窗口
通知（P0-1）： - 邮件：自写 SMTP 客户端，零依赖。465 隐式 TLS / 587·25 STARTTLS、 AUTH LOGIN+PLAIN、RFC2047 中文主题、base64 正文、多收件人； 服务器不支持 STARTTLS 时默认拒绝明文（需显式 allowInsecure） - MeoW：按官方文档实现，昵称寻址、msgType/htmlHeight 走查询参数、 以响应体 status 判定成功（HTTP 恒 200）、429/404/403 可读化、 本地限流器遵守 3s/15min/60h - 告警：连续 N 次失败去抖 + 恢复通知 + 同轮合并 + 去抖状态持久化 （data/notify-state.json，gitignore）+ 静默时段（跨零点） - --test-notify 自检命令、--data-dir / SHIROKO_DATA_DIR 证书（P0-2）： - 独立 TLS 握手取 notAfter（不动既有 fetch 探测路径），默认 6 小时一次 - 面板徽标 ≤14 天黄 / ≤7 天红，阈值 14/7/3 各告一次、已过期只报最严重一档 - 单个目标可用 tls: false 关闭 事件与维护（P1-1 的后端部分）： - data/events.json 追加式事件流（最近 500 条）+ GET /api/events + 快照内附最近 20 条；redactDetails 时脱敏 - 维护窗口：窗口内不计入可用率、不发告警、显示「维护中」徽标 测试抓到并修复的三个真 bug： 1. SMTP 空闲超时无人处理 → 服务器不响应会把探针循环挂死 2. MeoW 限流器把空历史当 0 → 第一条通知永远被自己拦掉 3. 去抖状态在无事轮次不落盘 → 连续失败计数丢失、永远达不到阈值 验证：新增 verify-notify.cjs 39 项（假 SMTP / 假 MeoW / 三轮集成 / 维护窗口 / 真实证书握手），六套合计 154 项全通过。
This commit is contained in:
tamakyi
+269
-3
@@ -20,6 +20,8 @@ import { probeHttp } from './lib/http-probe.js';
|
||||
import { probeTcp } from './lib/tcp-probe.js';
|
||||
import { probeBanner } from './lib/banner-probe.js';
|
||||
import { fetchUptimeRobotMonitors } from './lib/uptimerobot.js';
|
||||
import { notifyAll, sendTestNotifications, createMeoWLimiter } from './lib/notify.js';
|
||||
import { probeCertificate, tlsTargetOf } from './lib/tls-probe.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
@@ -35,13 +37,15 @@ const CONCURRENCY = 8;
|
||||
const WRITE_DEBOUNCE_MS = 300;
|
||||
const DEFAULT_INTERVAL_SEC = 60;
|
||||
const PORT = Number(process.env.PORT || 8787);
|
||||
const VERSION = '0.9.1';
|
||||
const VERSION = '0.10.0';
|
||||
const UR_DEFAULT_POLL_SEC = 300;
|
||||
const UR_MIN_REFRESH_GAP_MS = 60000;
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const ONCE = argv.includes('--once');
|
||||
const DEMO = argv.includes('--demo');
|
||||
// --test-notify：只给各通知渠道发一条自检消息后退出（不探测、不写快照）
|
||||
const TEST_NOTIFY = argv.includes('--test-notify');
|
||||
|
||||
// --out=<path> 或 --out <path>：指定快照输出路径（默认 public/status.json）
|
||||
function argValue(name) {
|
||||
@@ -54,6 +58,10 @@ const OUT_PATH = path.resolve(argValue('--out') || STATUS_PATH);
|
||||
// --config / --local-config：多实例或测试时指定配置文件（默认 public/monitors.json 与仓库根的私有覆盖）
|
||||
const CONFIG_OVERRIDE = argValue('--config');
|
||||
const LOCAL_CONFIG_OVERRIDE = argValue('--local-config');
|
||||
// --data-dir=<path>：通知状态与事件时间线的存放目录（默认 ./data，也可用环境变量 SHIROKO_DATA_DIR）
|
||||
if (argValue('--data-dir')) process.env.SHIROKO_DATA_DIR = argValue('--data-dir');
|
||||
// 私有数据目录：通知状态（告警去抖）与事件时间线，含本机运行痕迹，默认不入库
|
||||
const DATA_DIR = process.env.SHIROKO_DATA_DIR ? path.resolve(process.env.SHIROKO_DATA_DIR) : path.join(ROOT, 'data');
|
||||
const ACTIVE_CONFIG_PATH = CONFIG_OVERRIDE ? path.resolve(CONFIG_OVERRIDE) : CONFIG_PATH;
|
||||
const ACTIVE_LOCAL_CONFIG_PATH = LOCAL_CONFIG_OVERRIDE ? path.resolve(LOCAL_CONFIG_OVERRIDE) : LOCAL_CONFIG_PATH;
|
||||
|
||||
@@ -308,7 +316,208 @@ function memoryStats() {
|
||||
return { rssMB: mb(m.rss), heapUsedMB: mb(m.heapUsed), heapTotalMB: mb(m.heapTotal) };
|
||||
}
|
||||
|
||||
/* ---------- 数据源过滤 ---------- */
|
||||
/* ---------- 通知（邮件 / MeoW）、事件时间线、维护窗口与证书监控 ---------- */
|
||||
|
||||
const DEFAULT_TLS_CHECK_SEC = 21600; // 证书默认 6 小时查一次
|
||||
const TLS_ALERT_DAYS = [14, 7, 3];
|
||||
const EVENT_LIMIT = 500;
|
||||
const EVENT_IN_SNAPSHOT = 20;
|
||||
|
||||
function notifyConfig(config) {
|
||||
const n = config.settings?.notify ?? {};
|
||||
const channels = n.channels ?? {};
|
||||
return {
|
||||
enabled: n.enabled === true && Object.values(channels).some((c) => c?.enabled === true),
|
||||
failureThreshold: Math.max(1, Number(n.failureThreshold) || 2),
|
||||
recoveryNotify: n.recoveryNotify !== false,
|
||||
quietHours: n.quietHours ?? null,
|
||||
channels,
|
||||
};
|
||||
}
|
||||
|
||||
function tlsConfig(config) {
|
||||
const t = config.settings?.tls ?? {};
|
||||
const alertDays = Array.isArray(t.alertDays) ? t.alertDays.map(Number).filter(Number.isFinite) : null;
|
||||
return {
|
||||
enabled: t.enabled !== false, // 默认对 https 目标开启
|
||||
checkIntervalSec: Math.max(600, Number(t.checkIntervalSec) || DEFAULT_TLS_CHECK_SEC),
|
||||
alertDays: alertDays?.length ? alertDays.sort((a, b) => b - a) : TLS_ALERT_DAYS,
|
||||
};
|
||||
}
|
||||
|
||||
// 维护窗口：{ from, to, id?: string|string[], note? }；不写 id 表示对所有目标生效
|
||||
function activeMaintenanceFor(config, id, at = new Date()) {
|
||||
const list = config.settings?.maintenance;
|
||||
if (!Array.isArray(list) || !list.length) return null;
|
||||
const t = at.getTime();
|
||||
for (const w of list) {
|
||||
const from = Date.parse(w.from);
|
||||
const to = Date.parse(w.to);
|
||||
if (!Number.isFinite(from) || !Number.isFinite(to) || t < from || t > to) continue;
|
||||
if (!w.id) return w;
|
||||
const ids = Array.isArray(w.id) ? w.id : [w.id];
|
||||
if (ids.includes(id)) return w;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function readJson(file, fallback) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function writeJson(file, data) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, `${JSON.stringify(data, null, 2)}\n`);
|
||||
}
|
||||
|
||||
const NOTIFY_STATE_PATH = () => path.join(DATA_DIR, 'notify-state.json');
|
||||
const EVENTS_PATH = () => path.join(DATA_DIR, 'events.json');
|
||||
|
||||
function monitorNotifyState(runtime, id) {
|
||||
const list = runtime.notifyState.monitors;
|
||||
if (!list[id]) list[id] = { lastStatus: null, failStreak: 0, notifiedTls: [], tlsValidTo: null };
|
||||
return list[id];
|
||||
}
|
||||
|
||||
function appendEvents(runtime, entries) {
|
||||
if (!entries.length) return;
|
||||
runtime.events = [...runtime.events, ...entries].slice(-EVENT_LIMIT);
|
||||
writeJson(EVENTS_PATH(), runtime.events);
|
||||
}
|
||||
|
||||
// 通知 + 事件：每轮探测结束后评估状态翻转（带去抖）、恢复、维护窗口抑制
|
||||
async function evaluateRound(runtime) {
|
||||
const notify = notifyConfig(runtime.config);
|
||||
const events = [];
|
||||
const pendingNotifications = [];
|
||||
|
||||
for (const m of runtime.config.monitors) {
|
||||
const r = runtime.state.get(m.id)?.lastResult;
|
||||
if (!r || r.status === 'pending') continue;
|
||||
const st = monitorNotifyState(runtime, m.id);
|
||||
st.failStreak = r.status === 'down' ? (st.failStreak ?? 0) + 1 : 0;
|
||||
const maintenance = activeMaintenanceFor(runtime.config, m.id);
|
||||
const at = new Date().toISOString();
|
||||
|
||||
if (r.status === 'down') {
|
||||
// 去抖：连续失败达到阈值才算确认故障（阈值 1 表示立即告警）
|
||||
if (st.lastStatus === 'down' || st.failStreak < notify.failureThreshold) continue;
|
||||
st.lastStatus = 'down';
|
||||
st.lastNotifiedAt = at;
|
||||
events.push({ t: at, id: m.id, name: m.name, type: 'down', detail: r.detail ?? null, maintenance: Boolean(maintenance) });
|
||||
if (!maintenance) {
|
||||
pendingNotifications.push({
|
||||
subject: `【故障】${m.name} 连接失败`,
|
||||
text: `目标：${m.name}\n地址：${m.url ?? `${m.host}:${m.port}`}\n时间:${at}\n详情：${r.detail ?? '无'}\n连续失败：${st.failStreak} 次\n\n—— shiroko-monitor`,
|
||||
});
|
||||
}
|
||||
} else if (r.status === 'up') {
|
||||
if (st.lastStatus === null) {
|
||||
st.lastStatus = 'up'; // 首次观测不产生「恢复」噪音
|
||||
continue;
|
||||
}
|
||||
if (st.lastStatus === 'down') {
|
||||
st.lastStatus = 'up';
|
||||
st.lastNotifiedAt = at;
|
||||
events.push({ t: at, id: m.id, name: m.name, type: 'up', detail: '已恢复', maintenance: Boolean(maintenance) });
|
||||
if (notify.recoveryNotify && !maintenance) {
|
||||
pendingNotifications.push({
|
||||
subject: `【恢复】${m.name} 已恢复正常`,
|
||||
text: `目标：${m.name}\n地址：${m.url ?? `${m.host}:${m.port}`}\n恢复时间：${at}\n延迟：${r.latency ?? '—'} ms\n\n—— shiroko-monitor`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (events.length) {
|
||||
appendEvents(runtime, events);
|
||||
console.log(`[events] 新增 ${events.length} 条：${events.map((e) => `${e.name} ${e.type}`).join('、')}`);
|
||||
}
|
||||
// 汇总成一条通知，避免同一轮多点故障刷屏
|
||||
if (pendingNotifications.length) {
|
||||
const message = pendingNotifications.length === 1
|
||||
? pendingNotifications[0]
|
||||
: {
|
||||
subject: `【${pendingNotifications.length} 个目标状态变化】shiroko-monitor`,
|
||||
text: pendingNotifications.map((p) => p.text).join('\n\n──────────\n\n'),
|
||||
};
|
||||
const res = await notifyAll(notify, message, { logger: console });
|
||||
runtime.notifyState.lastResult = { at: new Date().toISOString(), ...res };
|
||||
}
|
||||
// 无论是否发通知都要落盘：否则「连续失败计数」会丢，重启或下一轮就达不到阈值
|
||||
saveNotifyState(runtime);
|
||||
}
|
||||
|
||||
// 证书检查：按 checkIntervalSec 逐个握手，跨阈值时告警（同一张证书每个阈值只告一次）
|
||||
async function checkCertificates(runtime) {
|
||||
const tlsCfg = tlsConfig(runtime.config);
|
||||
if (!tlsCfg.enabled) return;
|
||||
const now = Date.now();
|
||||
const events = [];
|
||||
const messages = [];
|
||||
|
||||
for (const m of runtime.config.monitors) {
|
||||
const target = tlsTargetOf(m);
|
||||
if (!target) continue;
|
||||
const prev = runtime.tls.get(m.id);
|
||||
if (prev && now - (prev.checkedAtMs ?? 0) < tlsCfg.checkIntervalSec * 1000) continue;
|
||||
const result = await probeCertificate({ ...target, timeoutMs: Math.max(5, Number(m.timeout) || 8) * 1000 });
|
||||
runtime.tls.set(m.id, { ...result, checkedAt: new Date().toISOString(), checkedAtMs: now });
|
||||
|
||||
const st = monitorNotifyState(runtime, m.id);
|
||||
if (result.error) {
|
||||
console.warn(`[tls] ${m.id} 证书检查失败：${result.error}`);
|
||||
continue;
|
||||
}
|
||||
// 换了证书（validTo 变化）→ 重置已通知阈值，新证书重新提醒
|
||||
if (st.tlsValidTo !== result.validTo) {
|
||||
st.tlsValidTo = result.validTo;
|
||||
st.notifiedTls = [];
|
||||
}
|
||||
const maintenance = activeMaintenanceFor(runtime.config, m.id);
|
||||
// 已过期的证书只报最严重的一档（避免一次刷三条）
|
||||
const thresholds = result.daysLeft <= 0 ? tlsCfg.alertDays.slice(-1) : tlsCfg.alertDays;
|
||||
for (const threshold of thresholds) {
|
||||
if (result.daysLeft > threshold || (st.notifiedTls ?? []).includes(threshold)) continue;
|
||||
st.notifiedTls = [...(st.notifiedTls ?? []), threshold];
|
||||
const at = new Date().toISOString();
|
||||
events.push({ t: at, id: m.id, name: m.name, type: 'tls', detail: `证书剩余 ${result.daysLeft} 天（阈值 ${threshold} 天）`, maintenance: Boolean(maintenance) });
|
||||
if (!maintenance) {
|
||||
messages.push(`- ${m.name}：证书剩余 ${result.daysLeft} 天（到期 ${result.validTo.slice(0, 10)}，签发者 ${result.issuer ?? '未知'}）`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (events.length) {
|
||||
appendEvents(runtime, events);
|
||||
if (messages.length) {
|
||||
const res = await notifyAll(notifyConfig(runtime.config), {
|
||||
subject: `【证书提醒】${messages.length} 个证书即将到期`,
|
||||
text: `${messages.join('\n')}\n\n签发者与到期时间取自实际握手，请及时续期。\n\n—— shiroko-monitor`,
|
||||
}, { logger: console });
|
||||
runtime.notifyState.lastResult = { at: new Date().toISOString(), ...res };
|
||||
}
|
||||
saveNotifyState(runtime);
|
||||
}
|
||||
}
|
||||
|
||||
function saveNotifyState(runtime) {
|
||||
writeJson(NOTIFY_STATE_PATH(), runtime.notifyState);
|
||||
}
|
||||
|
||||
// 通知与证书检查串行执行：避免并发轮次重复发同一条告警
|
||||
function queueEvaluation(runtime, task) {
|
||||
runtime.notifyChain = (runtime.notifyChain ?? Promise.resolve())
|
||||
.then(() => task(runtime))
|
||||
.catch((err) => console.error(`[notify] 评估异常：${err.stack || err.message}`));
|
||||
return runtime.notifyChain;
|
||||
}
|
||||
|
||||
|
||||
// 只影响「展示与快照输出」，不影响探测：配置文件里的目标始终按各自 interval 探测
|
||||
function dataFilter(config) {
|
||||
@@ -415,6 +624,14 @@ function snapshot(runtime, demo = false) {
|
||||
};
|
||||
entry.uptimeRatios = s.uptimeRatios ?? null;
|
||||
entry.downSeconds = s.downSeconds ?? null;
|
||||
} else {
|
||||
// 证书与维护窗口信息（来自独立的握手检查与配置），供面板打标
|
||||
const tlsInfo = runtime.tls?.get(id);
|
||||
if (tlsInfo) entry.tls = redact
|
||||
? { daysLeft: tlsInfo.daysLeft ?? null, error: tlsInfo.error ? '不可用' : null }
|
||||
: { daysLeft: tlsInfo.daysLeft ?? null, validTo: tlsInfo.validTo ?? null, issuer: tlsInfo.issuer ?? null, checkedAt: tlsInfo.checkedAt ?? null, error: tlsInfo.error ?? null };
|
||||
const maintenance = activeMaintenanceFor(config, id);
|
||||
if (maintenance) entry.maintenance = { note: maintenance.note ?? null, to: maintenance.to ?? null };
|
||||
}
|
||||
monitors[id] = entry;
|
||||
}
|
||||
@@ -438,6 +655,20 @@ function snapshot(runtime, demo = false) {
|
||||
memory: memoryStats(),
|
||||
},
|
||||
sources: { uptimerobot: { ...(runtime.ur ?? {}) } },
|
||||
// 通知能力与最近事件（事件含目标名与原因，redactDetails 时只保留类型与时间）
|
||||
notify: (() => {
|
||||
const n = notifyConfig(config);
|
||||
return {
|
||||
enabled: n.enabled,
|
||||
channels: Object.entries(n.channels ?? {}).filter(([, c]) => c?.enabled).map(([k]) => k),
|
||||
quietHours: n.quietHours,
|
||||
failureThreshold: n.failureThreshold,
|
||||
lastResult: runtime.notifyState?.lastResult ?? null,
|
||||
};
|
||||
})(),
|
||||
events: (runtime.events ?? []).slice(-EVENT_IN_SNAPSHOT).reverse().map((e) => (redact
|
||||
? { t: e.t, id: e.id, type: e.type, name: e.name, detail: e.type === 'down' ? '异常' : e.type === 'up' ? '恢复' : e.detail, maintenance: e.maintenance }
|
||||
: e)),
|
||||
monitors,
|
||||
};
|
||||
}
|
||||
@@ -486,6 +717,7 @@ function forceRound(runtime) {
|
||||
await settleInflight(runtime.state);
|
||||
runtime.forcedAt = Date.now();
|
||||
scheduleWrite(runtime);
|
||||
queueEvaluation(runtime, evaluateRound);
|
||||
} catch (err) {
|
||||
console.error(`[refresh] 强制探测异常：${err.message}`);
|
||||
} finally {
|
||||
@@ -654,6 +886,12 @@ function createServer(runtime) {
|
||||
return sendJson(res, toUptimeRobotResponse(runtime));
|
||||
}
|
||||
|
||||
// 事件时间线：面板「事件」视图取这里（含通知去抖前的全部状态翻转记录）
|
||||
if (url.pathname === '/api/events') {
|
||||
const list = [...(runtime.events ?? [])].reverse();
|
||||
return sendJson(res, { total: list.length, events: list.slice(0, 200) });
|
||||
}
|
||||
|
||||
let rel = decodeURIComponent(url.pathname);
|
||||
if (rel === '/') rel = '/index.html';
|
||||
const file = path.join(PUBLIC_DIR, path.normalize(rel));
|
||||
@@ -688,8 +926,26 @@ async function main() {
|
||||
config: loadConfig(),
|
||||
state: null,
|
||||
dataUpdatedAt: Date.now(),
|
||||
// 通知去抖状态与事件时间线从私有数据目录恢复，重启不会重复告警
|
||||
notifyState: readJson(NOTIFY_STATE_PATH(), { monitors: {} }),
|
||||
events: readJson(EVENTS_PATH(), []),
|
||||
tls: new Map(),
|
||||
tlsLastCheckMs: 0,
|
||||
ur: { enabled: false, ok: null, error: null, lastSyncAt: null, count: 0, syncing: false, lastSyncMs: 0 },
|
||||
};
|
||||
if (TEST_NOTIFY) {
|
||||
// 通道自检：只发一条测试消息，不探测、不写快照
|
||||
const cfg = notifyConfig(runtime.config);
|
||||
if (!cfg.enabled) {
|
||||
console.error('未启用任何通知渠道（settings.notify.enabled 或 channels.*.enabled 未开启）');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
console.log('通知通道自检：');
|
||||
const out = await sendTestNotifications(cfg, { logger: console });
|
||||
if (out.some((r) => !r.ok)) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (DEMO) {
|
||||
writeDemo(runtime.config);
|
||||
return;
|
||||
@@ -700,6 +956,8 @@ async function main() {
|
||||
|
||||
if (ONCE) {
|
||||
await probeDue(runtime, true);
|
||||
await queueEvaluation(runtime, evaluateRound);
|
||||
await queueEvaluation(runtime, checkCertificates);
|
||||
writeStatus(snapshot(runtime));
|
||||
const s = snapshot(runtime).stats;
|
||||
console.log(`[once] 完成：共 ${s.total}，在线 ${s.up}，离线 ${s.down}`);
|
||||
@@ -725,7 +983,10 @@ async function main() {
|
||||
|
||||
setInterval(() => {
|
||||
probeDue(runtime)
|
||||
.then(() => scheduleWrite(runtime))
|
||||
.then(() => {
|
||||
scheduleWrite(runtime);
|
||||
queueEvaluation(runtime, evaluateRound);
|
||||
})
|
||||
.catch((err) => console.error(`[tick] ${err.stack || err}`));
|
||||
// UptimeRobot 按自身 pollIntervalSec 拉取，失败不影响本地巡检
|
||||
if (Date.now() - runtime.ur.lastSyncMs >= uptimeRobotConfig(runtime.config).pollIntervalSec * 1000) {
|
||||
@@ -733,6 +994,11 @@ async function main() {
|
||||
.then(() => scheduleWrite(runtime))
|
||||
.catch(() => { /* syncUptimeRobot 内部已记录错误 */ });
|
||||
}
|
||||
// 证书到期检查：按 checkIntervalSec 周期执行（默认 6 小时），握手失败只记录不影响巡检
|
||||
if (Date.now() - runtime.tlsLastCheckMs >= tlsConfig(runtime.config).checkIntervalSec * 1000) {
|
||||
runtime.tlsLastCheckMs = Date.now();
|
||||
queueEvaluation(runtime, checkCertificates);
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
createServer(runtime).listen(PORT, () => {
|
||||
|
||||
Reference in New Issue
Block a user