231 lines
11 KiB
C#
231 lines
11 KiB
C#
using Model.Entity.Inspection;
|
||
using ORM;
|
||
using Service.Implement;
|
||
using Service.Interface;
|
||
using SqlSugar;
|
||
|
||
namespace WebAPI.Services
|
||
{
|
||
/// <summary>
|
||
/// 告警通知后台 Worker：消费 AlertNotifyBus 队列，按通知规则分发到飞书/钉钉/企微渠道
|
||
/// 流程：加载告警 → 匹配启用规则(按级别) → 静默期判断 → 直连推送(重试3次)或写跨网发件箱 → 记推送日志
|
||
/// </summary>
|
||
public class AlertNotifyWorker : BackgroundService
|
||
{
|
||
private readonly IServiceScopeFactory _scopeFactory;
|
||
private readonly ILogger<AlertNotifyWorker> _logger;
|
||
|
||
/// <summary>
|
||
/// 直连推送最大尝试次数（含首次）
|
||
/// </summary>
|
||
private const int MaxAttempts = 3;
|
||
|
||
public AlertNotifyWorker(IServiceScopeFactory scopeFactory, ILogger<AlertNotifyWorker> logger)
|
||
{
|
||
_scopeFactory = scopeFactory;
|
||
_logger = logger;
|
||
}
|
||
|
||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||
{
|
||
_logger.LogInformation("告警通知 Worker 已启动");
|
||
try
|
||
{
|
||
await foreach (var evt in AlertNotifyBus.Reader.ReadAllAsync(stoppingToken))
|
||
{
|
||
try
|
||
{
|
||
await ProcessEventAsync(evt, stoppingToken);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
_logger.LogError(ex, "处理告警通知事件失败, AlertId={AlertId}", evt.AlertId);
|
||
}
|
||
}
|
||
}
|
||
catch (OperationCanceledException)
|
||
{
|
||
// 正常停机
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 处理单个告警通知事件
|
||
/// </summary>
|
||
private async Task ProcessEventAsync(AlertNotifyEvent evt, CancellationToken ct)
|
||
{
|
||
var db = SqlSugarContext.DbContext;
|
||
|
||
// 1. 加载告警消息
|
||
var alert = await db.Queryable<AlertMessageEntity>()
|
||
.Where(x => x.Id == evt.AlertId && x.IsDel == 0)
|
||
.FirstAsync();
|
||
if (alert == null) return;
|
||
|
||
// 2. 匹配启用的通知规则（按告警级别）
|
||
var rules = await db.Queryable<AlertNotifyRuleEntity>()
|
||
.Where(x => x.IsDel == 0 && x.IsEnabled && x.AlertLevel == alert.AlertLevel)
|
||
.ToListAsync();
|
||
if (rules.Count == 0) return;
|
||
|
||
// 3. 每个事件独立创建作用域（Worker 是单例，服务是 Scoped）
|
||
using var scope = _scopeFactory.CreateScope();
|
||
var notifyService = scope.ServiceProvider.GetRequiredService<IAlertNotifyService>();
|
||
var notifiers = scope.ServiceProvider.GetServices<IAlertNotifier>().ToList();
|
||
|
||
foreach (var rule in rules)
|
||
{
|
||
var channel = await db.Queryable<AlertNotifyChannelEntity>()
|
||
.Where(x => x.Id == rule.ChannelId && x.IsDel == 0 && x.IsEnabled)
|
||
.FirstAsync();
|
||
if (channel == null) continue;
|
||
|
||
// 4. 静默期判断：合并触发的重复告警在静默期内不重复推送（新告警必推）
|
||
if (!evt.IsNewAlert && rule.SilenceMinutes > 0)
|
||
{
|
||
var silenceStart = DateTime.Now.AddMinutes(-rule.SilenceMinutes);
|
||
bool recentlyPushed = await db.Queryable<AlertNotifyLogEntity>()
|
||
.AnyAsync(x => x.AlertId == alert.Id && x.ChannelId == channel.Id
|
||
&& x.Success && x.NotifyTime >= silenceStart);
|
||
if (recentlyPushed) continue;
|
||
}
|
||
|
||
// 5. 组装消息与 @ 列表（规则接收人 + 当班值班人电话）
|
||
var message = BuildMessage(alert);
|
||
var atMobiles = new List<string>();
|
||
if (!string.IsNullOrWhiteSpace(rule.Receivers))
|
||
{
|
||
// 接收人常带 @ 前缀（如 "@18872509102"），但企微 mentioned_mobile_list / 钉钉 atMobiles 只认纯手机号，
|
||
// 带 @ 的非法值无法匹配群成员导致 @ 不生效，这里统一去掉前导 @ 与空白
|
||
atMobiles.AddRange(rule.Receivers.Split(',', '，', ';', '；')
|
||
.Select(s => s.Trim().TrimStart('@').Trim()).Where(s => s.Length > 0));
|
||
}
|
||
if (rule.NotifyDutyPerson)
|
||
{
|
||
var duty = await notifyService.GetDutyByDateAsync(alert.LastAlertTime.Date);
|
||
if (duty != null && !string.IsNullOrWhiteSpace(duty.Phone))
|
||
{
|
||
atMobiles.Add(duty.Phone.Trim());
|
||
message.Content += $"\n值班人: {duty.PersonName}";
|
||
}
|
||
}
|
||
atMobiles = atMobiles.Distinct().ToList();
|
||
|
||
// 飞书渠道：at 标签只认 open_id，不认手机号——把接收人里的手机号转成 open_id
|
||
// （"all" 与 ou_ 开头的原样保留）。转换在内网主站完成，直连与跨网(Outbox)两条路径都拿到 open_id，AlertBridge 无需改。
|
||
if (channel.ChannelType == NotifyChannelTypeEnum.Feishu && atMobiles.Count > 0)
|
||
{
|
||
var feishuContact = scope.ServiceProvider.GetRequiredService<IFeishuContactService>();
|
||
var openIds = new List<string>();
|
||
foreach (var at in atMobiles)
|
||
{
|
||
if (at.Equals("all", StringComparison.OrdinalIgnoreCase) || at.StartsWith("ou_"))
|
||
{
|
||
openIds.Add(at);
|
||
}
|
||
else
|
||
{
|
||
var openId = await feishuContact.GetOpenIdByMobileAsync(at);
|
||
if (!string.IsNullOrEmpty(openId)) openIds.Add(openId);
|
||
else _logger.LogWarning("飞书接收人 {Receiver} 未换取到 open_id，已跳过 @（检查飞书应用权限与通讯录范围）", at);
|
||
}
|
||
}
|
||
atMobiles = openIds.Distinct().ToList();
|
||
}
|
||
|
||
// 关键：@ 列表必须回填到 message——直连模式的 Notifier 是从 message.AtMobiles 读取的。
|
||
// 此前只把 atMobiles 传给了 Outbox 分支，直连 SendAsync(message) 的 message.AtMobiles 始终为空，导致企微/钉钉不 @ 任何人。
|
||
message.AtMobiles = atMobiles;
|
||
|
||
// 6. 按渠道推送模式分发
|
||
if (channel.PushMode == NotifyPushModeEnum.Outbox)
|
||
{
|
||
// 跨网模式：写发件箱，由跳板机 AlertBridge 拉取推送并回写结果
|
||
await notifyService.EnqueueOutboxAsync(alert.Id, channel.Id, message, atMobiles);
|
||
_logger.LogInformation("告警 {AlertId} 已转入跨网发件箱（渠道 {Channel}）", alert.Id, channel.Name);
|
||
continue;
|
||
}
|
||
|
||
// 直连模式：查找对应 Notifier，失败重试
|
||
var notifier = notifiers.FirstOrDefault(n => n.ChannelType == channel.ChannelType);
|
||
if (notifier == null)
|
||
{
|
||
await notifyService.WriteLogAsync(alert.Id, channel.Id, message.Title,
|
||
false, $"暂不支持的渠道类型: {channel.ChannelType}", 0);
|
||
continue;
|
||
}
|
||
|
||
NotifyResult result = NotifyResult.Fail("未执行");
|
||
int attempt = 0;
|
||
while (attempt < MaxAttempts && !ct.IsCancellationRequested)
|
||
{
|
||
attempt++;
|
||
result = await notifier.SendAsync(message, channel.WebhookUrl, channel.Secret, ct);
|
||
if (result.Success) break;
|
||
if (attempt < MaxAttempts)
|
||
{
|
||
await Task.Delay(TimeSpan.FromSeconds(attempt * 2), ct); // 2s/4s 退避
|
||
}
|
||
}
|
||
|
||
await notifyService.WriteLogAsync(alert.Id, channel.Id,
|
||
message.Title + " " + Truncate(message.Content, 800),
|
||
result.Success, result.ErrorMsg, attempt - 1);
|
||
|
||
if (!result.Success)
|
||
{
|
||
_logger.LogWarning("告警 {AlertId} 推送渠道 {Channel} 失败: {Error}", alert.Id, channel.Name, result.ErrorMsg);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 按内置模板构建通知消息（标题 + markdown 正文；后续可由通知模板模块接管）
|
||
/// </summary>
|
||
private static AlertNotifyMessage BuildMessage(AlertMessageEntity alert)
|
||
{
|
||
string levelLabel = AlertLevelMap.ToLabel(alert.AlertLevel);
|
||
var content = new System.Text.StringBuilder();
|
||
content.AppendLine($"- 设备: {alert.DeviceName}({alert.DeviceCode})");
|
||
if (!string.IsNullOrWhiteSpace(alert.DeviceType)) content.AppendLine($"- 类型: {alert.DeviceType}");
|
||
if (!string.IsNullOrWhiteSpace(alert.Metric)) content.AppendLine($"- 测点: {alert.Metric}");
|
||
if (alert.CurrentValue.HasValue) content.AppendLine($"- 当前值: {alert.CurrentValue.Value}");
|
||
if (alert.ThresholdValue.HasValue) content.AppendLine($"- 阈值: {alert.ThresholdValue.Value}");
|
||
content.AppendLine($"- 告警类型: {TypeLabel(alert.AlertType)}");
|
||
content.AppendLine($"- 来源: {SourceLabel(alert.Source)}");
|
||
content.AppendLine($"- 内容: {alert.Content}");
|
||
if (alert.TriggerCount > 1) content.AppendLine($"- 累计触发: {alert.TriggerCount} 次");
|
||
content.Append($"- 时间: {alert.LastAlertTime:yyyy-MM-dd HH:mm:ss}");
|
||
|
||
return new AlertNotifyMessage
|
||
{
|
||
Title = $"【{levelLabel}】设备告警 - {alert.DeviceName ?? alert.DeviceCode}",
|
||
Content = content.ToString()
|
||
};
|
||
}
|
||
|
||
private static string TypeLabel(AlertTypeEnum type) => type switch
|
||
{
|
||
AlertTypeEnum.ThresholdExceed => "阈值超限",
|
||
AlertTypeEnum.DeviceOffline => "设备离线",
|
||
AlertTypeEnum.CommFault => "通讯故障",
|
||
AlertTypeEnum.PatrolAbnormal => "巡检异常",
|
||
AlertTypeEnum.Custom => "自定义",
|
||
_ => "其它"
|
||
};
|
||
|
||
private static string SourceLabel(AlertSourceEnum source) => source switch
|
||
{
|
||
AlertSourceEnum.RuleEngine => "规则引擎",
|
||
AlertSourceEnum.AutoPatrol => "自动巡检",
|
||
AlertSourceEnum.Dashboard => "数据看板",
|
||
AlertSourceEnum.Collection => "数据采集",
|
||
AlertSourceEnum.Manual => "手动上报",
|
||
_ => "其它"
|
||
};
|
||
|
||
private static string Truncate(string s, int max) =>
|
||
string.IsNullOrEmpty(s) || s.Length <= max ? s : s.Substring(0, max) + "...";
|
||
}
|
||
}
|