592 lines
18 KiB
Python
592 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
告警模块
|
|
支持邮件告警、Webhook 告警、告警规则引擎
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import time
|
|
import threading
|
|
import logging
|
|
import smtplib
|
|
import requests
|
|
from datetime import datetime
|
|
from typing import Dict, List, Optional, Callable
|
|
from email.mime.text import MIMEText
|
|
from email.mime.multipart import MIMEMultipart
|
|
from enum import Enum
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class AlertSeverity(Enum):
|
|
"""告警级别"""
|
|
INFO = "info"
|
|
WARNING = "warning"
|
|
ERROR = "error"
|
|
CRITICAL = "critical"
|
|
|
|
|
|
class AlertType(Enum):
|
|
"""告警类型"""
|
|
DISK_HIGH = "disk_high"
|
|
DISK_CRITICAL = "disk_critical"
|
|
SYNC_FAILED = "sync_failed"
|
|
SOURCE_UNHEALTHY = "source_unhealthy"
|
|
CACHE_FULL = "cache_full"
|
|
SERVICE_DOWN = "service_down"
|
|
CUSTOM = "custom"
|
|
|
|
|
|
class Alert:
|
|
"""告警对象"""
|
|
|
|
def __init__(
|
|
self,
|
|
alert_type: str,
|
|
severity: AlertSeverity,
|
|
title: str,
|
|
message: str,
|
|
details: Dict = None,
|
|
source: str = None
|
|
):
|
|
self.id = f"{int(time.time())}_{threading.get_ident()}"
|
|
self.type = alert_type
|
|
self.severity = severity
|
|
self.title = title
|
|
self.message = message
|
|
self.details = details or {}
|
|
self.source = source
|
|
self.timestamp = datetime.now()
|
|
self.sent = False
|
|
self.acknowledged = False
|
|
|
|
def to_dict(self) -> Dict:
|
|
"""转换为字典"""
|
|
return {
|
|
'id': self.id,
|
|
'type': self.type,
|
|
'severity': self.severity.value,
|
|
'title': self.title,
|
|
'message': self.message,
|
|
'details': self.details,
|
|
'source': self.source,
|
|
'timestamp': self.timestamp.isoformat(),
|
|
'sent': self.sent,
|
|
'acknowledged': self.acknowledged
|
|
}
|
|
|
|
|
|
class EmailAlerter:
|
|
"""邮件告警器"""
|
|
|
|
def __init__(self, config: Dict = None):
|
|
"""
|
|
初始化邮件告警器
|
|
|
|
Args:
|
|
config: 邮件配置
|
|
"""
|
|
self.config = config or {}
|
|
self.enabled = self.config.get('enabled', False)
|
|
self.smtp_host = self.config.get('smtp_host', 'localhost')
|
|
self.smtp_port = self.config.get('smtp_port', 587)
|
|
self.smtp_user = self.config.get('smtp_user', '')
|
|
self.smtp_password = self.config.get('smtp_password', '')
|
|
self.from_address = self.config.get('from_address', 'hyc-mirror@localhost')
|
|
self.to_addresses = self.config.get('to_addresses', [])
|
|
self.use_tls = self.config.get('use_tls', True)
|
|
|
|
# 连接池
|
|
self._connection: Optional[smtplib.SMTP] = None
|
|
self._last_connect_time: Optional[datetime] = None
|
|
self._connection_timeout = 30
|
|
|
|
def _get_connection(self) -> smtplib.SMTP:
|
|
"""获取 SMTP 连接"""
|
|
if self._connection:
|
|
# 检查连接是否仍然有效
|
|
try:
|
|
self._connection.noop()
|
|
return self._connection
|
|
except Exception:
|
|
try:
|
|
self._connection.quit()
|
|
except Exception:
|
|
pass
|
|
self._connection = None
|
|
|
|
# 创建新连接
|
|
try:
|
|
self._connection = smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=self._connection_timeout)
|
|
if self.use_tls:
|
|
self._connection.starttls()
|
|
if self.smtp_user and self.smtp_password:
|
|
self._connection.login(self.smtp_user, self.smtp_password)
|
|
self._last_connect_time = datetime.now()
|
|
return self._connection
|
|
except Exception as e:
|
|
logger.error(f"Failed to connect to SMTP server: {e}")
|
|
raise
|
|
|
|
def send(self, alert: Alert) -> bool:
|
|
"""
|
|
发送告警邮件
|
|
|
|
Args:
|
|
alert: 告警对象
|
|
|
|
Returns:
|
|
是否发送成功
|
|
"""
|
|
if not self.enabled:
|
|
logger.debug("Email alerts disabled")
|
|
return False
|
|
|
|
if not self.to_addresses:
|
|
logger.warning("No recipients configured for email alerts")
|
|
return False
|
|
|
|
try:
|
|
msg = MIMEMultipart('alternative')
|
|
msg['Subject'] = f"[{alert.severity.value.upper()}] {alert.title}"
|
|
msg['From'] = self.from_address
|
|
msg['To'] = ', '.join(self.to_addresses)
|
|
|
|
# HTML 格式
|
|
html_content = self._format_html(alert)
|
|
msg.attach(MIMEText(html_content, 'html', 'utf-8'))
|
|
|
|
# 纯文本格式
|
|
text_content = self._format_text(alert)
|
|
msg.attach(MIMEText(text_content, 'plain', 'utf-8'))
|
|
|
|
# 发送邮件
|
|
server = self._get_connection()
|
|
server.send_message(msg)
|
|
logger.info(f"Alert email sent: {alert.title}")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to send alert email: {e}")
|
|
return False
|
|
|
|
def _format_html(self, alert: Alert) -> str:
|
|
"""格式化 HTML 内容"""
|
|
severity_colors = {
|
|
'info': '#2196F3',
|
|
'warning': '#FF9800',
|
|
'error': '#F44336',
|
|
'critical': '#9C27B0'
|
|
}
|
|
color = severity_colors.get(alert.severity.value, '#666666')
|
|
|
|
details_html = ''
|
|
if alert.details:
|
|
details_html = '<h3>Details</h3><table>'
|
|
for key, value in alert.details.items():
|
|
details_html += f'<tr><td><b>{key}:</b></td><td>{value}</td></tr>'
|
|
details_html += '</table>'
|
|
|
|
return f"""
|
|
<html>
|
|
<head>
|
|
<style>
|
|
body {{ font-family: Arial, sans-serif; margin: 20px; }}
|
|
.header {{ background-color: {color}; color: white; padding: 10px; }}
|
|
.title {{ font-size: 24px; margin: 20px 0; }}
|
|
.message {{ background-color: #f5f5f5; padding: 15px; border-radius: 5px; margin: 20px 0; }}
|
|
.details {{ margin-top: 20px; }}
|
|
.footer {{ margin-top: 30px; font-size: 12px; color: #666; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="header">
|
|
<h1>HYC Mirror Alert</h1>
|
|
</div>
|
|
<div class="title">[{alert.severity.value.upper()}] {alert.title}</div>
|
|
<div class="message">
|
|
<p><b>Message:</b> {alert.message}</p>
|
|
<p><b>Time:</b> {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}</p>
|
|
<p><b>Type:</b> {alert.type}</p>
|
|
{details_html}
|
|
</div>
|
|
<div class="footer">
|
|
<p>This is an automated alert from HYC Mirror Server</p>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
def _format_text(self, alert: Alert) -> str:
|
|
"""格式化纯文本内容"""
|
|
details_text = ''
|
|
if alert.details:
|
|
details_text = '\nDetails:\n'
|
|
for key, value in alert.details.items():
|
|
details_text += f" {key}: {value}\n"
|
|
|
|
return f"""
|
|
HYC Mirror Alert
|
|
================
|
|
|
|
Severity: {alert.severity.value.upper()}
|
|
Title: {alert.title}
|
|
Message: {alert.message}
|
|
Time: {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}
|
|
Type: {alert.type}
|
|
{details_text}
|
|
---
|
|
This is an automated alert from HYC Mirror Server
|
|
"""
|
|
|
|
def test_connection(self) -> Dict:
|
|
"""测试 SMTP 连接"""
|
|
try:
|
|
server = self._get_connection()
|
|
return {
|
|
'success': True,
|
|
'message': 'SMTP connection successful'
|
|
}
|
|
except Exception as e:
|
|
return {
|
|
'success': False,
|
|
'message': f'SMTP connection failed: {str(e)}'
|
|
}
|
|
|
|
def close(self):
|
|
"""关闭连接"""
|
|
if self._connection:
|
|
try:
|
|
self._connection.quit()
|
|
except Exception:
|
|
pass
|
|
self._connection = None
|
|
|
|
|
|
class WebhookAlerter:
|
|
"""Webhook 告警器"""
|
|
|
|
def __init__(self, config: Dict = None):
|
|
"""
|
|
初始化 Webhook 告警器
|
|
|
|
Args:
|
|
config: Webhook 配置
|
|
"""
|
|
self.config = config or {}
|
|
self.enabled = self.config.get('enabled', False)
|
|
self.webhook_url = self.config.get('webhook_url', '')
|
|
|
|
def send(self, alert: Alert) -> bool:
|
|
"""
|
|
发送告警到 Webhook
|
|
|
|
Args:
|
|
alert: 告警对象
|
|
|
|
Returns:
|
|
是否发送成功
|
|
"""
|
|
if not self.enabled:
|
|
logger.debug("Webhook alerts disabled")
|
|
return False
|
|
|
|
if not self.webhook_url:
|
|
logger.warning("No webhook URL configured")
|
|
return False
|
|
|
|
try:
|
|
payload = {
|
|
'event': 'alert',
|
|
'alert': alert.to_dict(),
|
|
'timestamp': datetime.now().isoformat()
|
|
}
|
|
|
|
headers = {
|
|
'Content-Type': 'application/json',
|
|
'User-Agent': 'HYC-Mirror-Alerts/1.0'
|
|
}
|
|
|
|
response = requests.post(
|
|
self.webhook_url,
|
|
json=payload,
|
|
headers=headers,
|
|
timeout=30
|
|
)
|
|
|
|
if response.status_code < 400:
|
|
logger.info(f"Alert webhook sent: {alert.title}")
|
|
return True
|
|
else:
|
|
logger.error(f"Webhook returned error: {response.status_code}")
|
|
return False
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to send webhook alert: {e}")
|
|
return False
|
|
|
|
|
|
class AlertManager:
|
|
"""告警管理器"""
|
|
|
|
def __init__(self, config: Dict = None):
|
|
"""
|
|
初始化告警管理器
|
|
|
|
Args:
|
|
config: 告警配置
|
|
"""
|
|
self.config = config or {}
|
|
self.enabled = self.config.get('enabled', False)
|
|
|
|
# 初始化告警器
|
|
self.email_alerter = EmailAlerter(self.config.get('email', {}))
|
|
self.webhook_alerter = WebhookAlerter(self.config.get('webhook', {}))
|
|
|
|
# 告警规则
|
|
self.rules = self.config.get('rules', {})
|
|
|
|
# 告警历史
|
|
self._alerts: List[Alert] = []
|
|
self._alerts_lock = threading.Lock()
|
|
self._max_history = 100
|
|
|
|
# 回调函数
|
|
self._on_alert: Optional[Callable] = None
|
|
self._on_ack: Optional[Callable] = None
|
|
|
|
# 告警冷却（防止重复告警）
|
|
self._alert_cooldowns: Dict[str, float] = {}
|
|
self._default_cooldown = 300 # 5 分钟
|
|
|
|
def set_alert_callback(self, callback: Callable):
|
|
"""设置告警回调"""
|
|
self._on_alert = callback
|
|
|
|
def set_ack_callback(self, callback: Callable):
|
|
"""设置确认回调"""
|
|
self._on_ack = callback
|
|
|
|
def check_rule(self, rule_name: str, data: Dict) -> Optional[Alert]:
|
|
"""
|
|
检查规则并生成告警
|
|
|
|
Args:
|
|
rule_name: 规则名称
|
|
data: 检查数据
|
|
|
|
Returns:
|
|
告警对象或 None
|
|
"""
|
|
if not self.enabled:
|
|
return None
|
|
|
|
rule = self.rules.get(rule_name, {})
|
|
if not rule.get('enabled', False):
|
|
return None
|
|
|
|
severity = AlertSeverity(rule.get('severity', 'warning'))
|
|
threshold = rule.get('threshold')
|
|
|
|
# 磁盘空间检查
|
|
if rule_name == 'disk_high' and threshold:
|
|
disk_percent = data.get('disk_percent', 0)
|
|
if disk_percent >= threshold:
|
|
return Alert(
|
|
alert_type=AlertType.DISK_HIGH.value,
|
|
severity=severity,
|
|
title=f"Disk usage is high: {disk_percent}%",
|
|
message=f"Disk usage has reached {disk_percent}%, which is above the {threshold}% threshold.",
|
|
details={'disk_percent': disk_percent, 'threshold': threshold},
|
|
source='monitor'
|
|
)
|
|
|
|
if rule_name == 'disk_critical' and threshold:
|
|
disk_percent = data.get('disk_percent', 0)
|
|
if disk_percent >= threshold:
|
|
return Alert(
|
|
alert_type=AlertType.DISK_CRITICAL.value,
|
|
severity=severity,
|
|
title=f"Disk usage is critical: {disk_percent}%",
|
|
message=f"Disk usage has reached {disk_percent}%, which is above the {critical_threshold}% threshold. Immediate action required!",
|
|
details={'disk_percent': disk_percent, 'threshold': threshold},
|
|
source='monitor'
|
|
)
|
|
|
|
# 同步失败检查
|
|
if rule_name == 'sync_failed':
|
|
sync_result = data.get('sync_result')
|
|
if sync_result and not sync_result.get('success', True):
|
|
return Alert(
|
|
alert_type=AlertType.SYNC_FAILED.value,
|
|
severity=severity,
|
|
title=f"Sync failed: {sync_result.get('source', 'unknown')}",
|
|
message=sync_result.get('error', 'Unknown sync error'),
|
|
details=sync_result,
|
|
source='sync'
|
|
)
|
|
|
|
# 源不健康检查
|
|
if rule_name == 'source_unhealthy':
|
|
unhealthy_sources = data.get('unhealthy_sources', [])
|
|
if unhealthy_sources:
|
|
return Alert(
|
|
alert_type=AlertType.SOURCE_UNHEALTHY.value,
|
|
severity=severity,
|
|
title=f"Unhealthy mirror sources detected: {len(unhealthy_sources)}",
|
|
message=f"The following mirror sources are unhealthy: {', '.join(unhealthy_sources)}",
|
|
details={'unhealthy_sources': unhealthy_sources},
|
|
source='health_check'
|
|
)
|
|
|
|
return None
|
|
|
|
def trigger_alert(self, alert: Alert) -> bool:
|
|
"""
|
|
触发告警
|
|
|
|
Args:
|
|
alert: 告警对象
|
|
|
|
Returns:
|
|
是否发送成功
|
|
"""
|
|
if not self.enabled:
|
|
return False
|
|
|
|
# 检查冷却时间
|
|
cooldown_key = f"{alert.type}:{alert.source or 'unknown'}"
|
|
last_alert = self._alert_cooldowns.get(cooldown_key, 0)
|
|
if time.time() - last_alert < self._default_cooldown:
|
|
logger.debug(f"Alert {alert.type} in cooldown, skipping")
|
|
return False
|
|
|
|
# 发送告警
|
|
email_sent = self.email_alerter.send(alert)
|
|
webhook_sent = self.webhook_alerter.send(alert)
|
|
|
|
alert.sent = email_sent or webhook_sent
|
|
|
|
# 记录告警
|
|
with self._alerts_lock:
|
|
self._alerts.append(alert)
|
|
if len(self._alerts) > self._max_history:
|
|
self._alerts = self._alerts[-self._max_history:]
|
|
|
|
# 更新冷却时间
|
|
self._alert_cooldowns[cooldown_key] = time.time()
|
|
|
|
# 触发回调
|
|
if self._on_alert and alert.sent:
|
|
try:
|
|
self._on_alert(alert)
|
|
except Exception as e:
|
|
logger.error(f"Alert callback failed: {e}")
|
|
|
|
return alert.sent
|
|
|
|
def acknowledge_alert(self, alert_id: str) -> bool:
|
|
"""
|
|
确认告警
|
|
|
|
Args:
|
|
alert_id: 告警 ID
|
|
|
|
Returns:
|
|
是否成功
|
|
"""
|
|
with self._alerts_lock:
|
|
for alert in self._alerts:
|
|
if alert.id == alert_id:
|
|
alert.acknowledged = True
|
|
if self._on_ack:
|
|
try:
|
|
self._on_ack(alert)
|
|
except Exception as e:
|
|
logger.error(f"Ack callback failed: {e}")
|
|
return True
|
|
return False
|
|
|
|
def get_alerts(
|
|
self,
|
|
acknowledged: bool = None,
|
|
severity: str = None,
|
|
limit: int = 50
|
|
) -> List[Dict]:
|
|
"""
|
|
获取告警列表
|
|
|
|
Args:
|
|
acknowledged: 过滤已确认状态
|
|
severity: 过滤级别
|
|
limit: 返回数量限制
|
|
|
|
Returns:
|
|
告警列表
|
|
"""
|
|
with self._alerts_lock:
|
|
alerts = [a.to_dict() for a in self._alerts]
|
|
|
|
# 过滤
|
|
if acknowledged is not None:
|
|
alerts = [a for a in alerts if a['acknowledged'] == acknowledged]
|
|
|
|
if severity:
|
|
alerts = [a for a in alerts if a['severity'] == severity]
|
|
|
|
# 返回最近的告警
|
|
return alerts[-limit:]
|
|
|
|
def get_stats(self) -> Dict:
|
|
"""获取告警统计"""
|
|
with self._alerts_lock:
|
|
total = len(self._alerts)
|
|
unack = sum(1 for a in self._alerts if not a['acknowledged'])
|
|
by_severity = {}
|
|
for a in self._alerts:
|
|
by_severity[a['severity']] = by_severity.get(a['severity'], 0) + 1
|
|
|
|
return {
|
|
'total_alerts': total,
|
|
'unacknowledged': unack,
|
|
'by_severity': by_severity,
|
|
'email_enabled': self.email_alerter.enabled,
|
|
'webhook_enabled': self.webhook_alerter.enabled,
|
|
'rules_enabled': sum(1 for r in self.rules.values() if r.get('enabled', False))
|
|
}
|
|
|
|
def clear_history(self) -> bool:
|
|
"""清除告警历史"""
|
|
with self._alerts_lock:
|
|
self._alerts = []
|
|
return True
|
|
|
|
def test_email(self, to_address: str) -> Dict:
|
|
"""测试邮件发送"""
|
|
test_alert = Alert(
|
|
alert_type=AlertType.CUSTOM.value,
|
|
severity=AlertSeverity.INFO,
|
|
title="Test Alert",
|
|
message="This is a test alert from HYC Mirror Server",
|
|
details={'test': True}
|
|
)
|
|
|
|
# 临时添加收件人
|
|
original_recipients = self.email_alerter.to_addresses
|
|
self.email_alerter.to_addresses = [to_address]
|
|
|
|
success = self.email_alerter.send(test_alert)
|
|
|
|
# 恢复收件人
|
|
self.email_alerter.to_addresses = original_recipients
|
|
|
|
return {
|
|
'success': success,
|
|
'message': 'Test email sent successfully' if success else 'Failed to send test email'
|
|
}
|