427 lines
15 KiB
Python
427 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
镜像源健康检查模块
|
|
自动检测上游镜像源的可用性，支持故障切换
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import time
|
|
import threading
|
|
import logging
|
|
import urllib.request
|
|
import urllib.error
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, List, Optional, Callable
|
|
from enum import Enum
|
|
from dataclasses import dataclass, field
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class HealthStatus(Enum):
|
|
"""健康状态"""
|
|
UNKNOWN = "unknown"
|
|
HEALTHY = "healthy"
|
|
DEGRADED = "degraded"
|
|
UNHEALTHY = "unhealthy"
|
|
|
|
|
|
@dataclass
|
|
class HealthCheckResult:
|
|
"""健康检查结果"""
|
|
source_name: str
|
|
status: HealthStatus
|
|
response_time: float # 毫秒
|
|
http_status: Optional[int] = None
|
|
error_message: Optional[str] = None
|
|
last_check: Optional[datetime] = None
|
|
consecutive_failures: int = 0
|
|
total_checks: int = 0
|
|
success_rate: float = 100.0
|
|
details: Dict = field(default_factory=dict)
|
|
|
|
|
|
class HealthChecker:
|
|
"""健康检查器"""
|
|
|
|
def __init__(self, config: dict = None):
|
|
"""
|
|
初始化健康检查器
|
|
|
|
Args:
|
|
config: 健康检查配置
|
|
"""
|
|
self.config = config or {}
|
|
self.default_timeout = self.config.get('timeout', 10) # 秒
|
|
self.default_interval = self.config.get('interval', 60) # 秒
|
|
self.max_retries = self.config.get('max_retries', 3)
|
|
self.failure_threshold = self.config.get('failure_threshold', 3) # 连续失败次数阈值
|
|
|
|
# 检查结果存储
|
|
self._results: Dict[str, HealthCheckResult] = {}
|
|
self._lock = threading.Lock()
|
|
|
|
# 状态变化回调
|
|
self._on_status_change: Optional[Callable] = None
|
|
|
|
def set_status_change_callback(self, callback: Callable):
|
|
"""设置状态变化回调"""
|
|
self._on_status_change = callback
|
|
|
|
def check_source(self, source_name: str, source_config: dict) -> HealthCheckResult:
|
|
"""
|
|
检查单个镜像源的健康状态
|
|
|
|
Args:
|
|
source_name: 镜像源名称
|
|
source_config: 镜像源配置
|
|
|
|
Returns:
|
|
HealthCheckResult: 检查结果
|
|
"""
|
|
url = source_config.get('url', '')
|
|
if not url:
|
|
return HealthCheckResult(
|
|
source_name=source_name,
|
|
status=HealthStatus.UNHEALTHY,
|
|
response_time=0,
|
|
error_message="No URL configured",
|
|
last_check=datetime.now()
|
|
)
|
|
|
|
start_time = time.time()
|
|
http_status = None
|
|
error_message = None
|
|
details = {}
|
|
|
|
try:
|
|
# 发送 HEAD 请求（更轻量）
|
|
req = urllib.request.Request(
|
|
url.rstrip('/') + '/',
|
|
method='HEAD',
|
|
headers={
|
|
'User-Agent': 'HYC-Mirror-HealthCheck/1.0',
|
|
'Accept': '*/*'
|
|
}
|
|
)
|
|
|
|
with urllib.request.urlopen(req, timeout=self.default_timeout) as response:
|
|
http_status = response.status
|
|
details['headers'] = dict(response.headers)
|
|
|
|
except urllib.error.HTTPError as e:
|
|
http_status = e.code
|
|
error_message = f"HTTP {e.code}"
|
|
except urllib.error.URLError as e:
|
|
error_message = f"Connection error: {str(e.reason)}"
|
|
except Exception as e:
|
|
error_message = str(e)
|
|
|
|
response_time = (time.time() - start_time) * 1000 # 转换为毫秒
|
|
|
|
# 判断健康状态
|
|
if error_message:
|
|
if http_status and 400 <= http_status < 500:
|
|
status = HealthStatus.DEGRADED # 客户端错误，可能暂时
|
|
else:
|
|
status = HealthStatus.UNHEALTHY
|
|
elif http_status and 200 <= http_status < 400:
|
|
status = HealthStatus.HEALTHY
|
|
elif http_status:
|
|
status = HealthStatus.DEGRADED
|
|
else:
|
|
status = HealthStatus.UNHEALTHY
|
|
|
|
# 计算统计数据
|
|
with self._lock:
|
|
if source_name not in self._results:
|
|
self._results[source_name] = HealthCheckResult(
|
|
source_name=source_name,
|
|
status=status,
|
|
response_time=response_time,
|
|
last_check=datetime.now(),
|
|
consecutive_failures=0,
|
|
total_checks=1
|
|
)
|
|
else:
|
|
old_result = self._results[source_name]
|
|
consecutive_failures = old_result.consecutive_failures + (1 if status == HealthStatus.UNHEALTHY else 0)
|
|
total_checks = old_result.total_checks + 1
|
|
success_rate = ((total_checks - consecutive_failures) / total_checks) * 100
|
|
|
|
self._results[source_name] = HealthCheckResult(
|
|
source_name=source_name,
|
|
status=status,
|
|
response_time=response_time,
|
|
http_status=http_status,
|
|
error_message=error_message,
|
|
last_check=datetime.now(),
|
|
consecutive_failures=consecutive_failures,
|
|
total_checks=total_checks,
|
|
success_rate=success_rate,
|
|
details=details
|
|
)
|
|
|
|
# 检查状态变化，触发回调
|
|
if self._on_status_change and source_name in self._results:
|
|
old_status = self._results[source_name].status
|
|
if old_status != status:
|
|
try:
|
|
self._on_status_change(source_name, old_status, status)
|
|
except Exception as e:
|
|
logger.error(f"状态变化回调执行失败: {e}")
|
|
|
|
return self._results[source_name]
|
|
|
|
def get_all_results(self) -> List[HealthCheckResult]:
|
|
"""获取所有检查结果"""
|
|
with self._lock:
|
|
return list(self._results.values())
|
|
|
|
def get_result(self, source_name: str) -> Optional[HealthCheckResult]:
|
|
"""获取指定源的结果"""
|
|
with self._lock:
|
|
return self._results.get(source_name)
|
|
|
|
def is_healthy(self, source_name: str) -> bool:
|
|
"""检查源是否健康"""
|
|
result = self.get_result(source_name)
|
|
if result is None:
|
|
return True # 未检查过的默认健康
|
|
return result.status == HealthStatus.HEALTHY
|
|
|
|
def get_unhealthy_sources(self) -> List[str]:
|
|
"""获取不健康的源列表"""
|
|
with self._lock:
|
|
return [name for name, result in self._results.items()
|
|
if result.status == HealthStatus.UNHEALTHY]
|
|
|
|
def get_stats(self) -> dict:
|
|
"""获取健康检查统计"""
|
|
with self._lock:
|
|
total = len(self._results)
|
|
healthy = sum(1 for r in self._results.values() if r.status == HealthStatus.HEALTHY)
|
|
degraded = sum(1 for r in self._results.values() if r.status == HealthStatus.DEGRADED)
|
|
unhealthy = sum(1 for r in self._results.values() if r.status == HealthStatus.UNHEALTHY)
|
|
|
|
avg_response_time = 0
|
|
if total > 0:
|
|
avg_response_time = sum(r.response_time for r in self._results.values()) / total
|
|
|
|
return {
|
|
'total_sources': total,
|
|
'healthy': healthy,
|
|
'degraded': degraded,
|
|
'unhealthy': unhealthy,
|
|
'avg_response_time_ms': round(avg_response_time, 2),
|
|
'timestamp': datetime.now().isoformat()
|
|
}
|
|
|
|
|
|
class MirrorFailoverManager:
|
|
"""镜像源故障切换管理器"""
|
|
|
|
def __init__(self, config: dict = None):
|
|
"""
|
|
初始化故障切换管理器
|
|
|
|
Args:
|
|
config: 配置，包含镜像源列表
|
|
"""
|
|
self.config = config or {}
|
|
self.mirrors: Dict[str, Dict] = self.config.get('mirrors', {})
|
|
|
|
# 启用故障切换
|
|
self.failover_enabled = self.config.get('failover_enabled', True)
|
|
self.failover_threshold = self.config.get('failover_threshold', 3) # 连续失败次数
|
|
|
|
# 健康检查器
|
|
self.health_checker = HealthChecker(self.config.get('health_check', {}))
|
|
|
|
# 当前活跃源
|
|
self._active_source: Dict[str, str] = {} # mirror_type -> source_name
|
|
self._source_priority: Dict[str, List[str]] = {} # mirror_type -> [优先列表]
|
|
|
|
# 故障切换历史
|
|
self._failover_history: List[dict] = []
|
|
|
|
# 回调
|
|
self._on_failover: Optional[Callable] = None
|
|
|
|
def set_failover_callback(self, callback: Callable):
|
|
"""设置故障切换回调"""
|
|
self._on_failover = callback
|
|
|
|
def initialize(self):
|
|
"""初始化，确定各镜像类型的首选源"""
|
|
for mirror_type, mirror_config in self.mirrors.items():
|
|
if not isinstance(mirror_config, dict):
|
|
continue
|
|
|
|
sources = mirror_config.get('sources', [])
|
|
if sources:
|
|
# 使用配置的优先列表
|
|
self._source_priority[mirror_type] = sources
|
|
else:
|
|
# 使用内置的默认优先列表
|
|
self._source_priority[mirror_type] = self._get_default_priority(mirror_type)
|
|
|
|
# 选择首选源
|
|
if self._source_priority[mirror_type]:
|
|
self._active_source[mirror_type] = self._source_priority[mirror_type][0]
|
|
|
|
def _get_default_priority(self, mirror_type: str) -> List[str]:
|
|
"""获取默认的镜像源优先列表"""
|
|
priorities = {
|
|
'docker': ['docker.io', 'docker.mirrors.aliyun.com', 'dockerhub.azk8s.cn'],
|
|
'apt': ['archive.ubuntu.com', 'mirrors.aliyun.com', 'security.ubuntu.com'],
|
|
'yum': ['mirror.centos.org', 'mirrors.aliyun.com'],
|
|
'pypi': ['pypi.org', 'pypi.mirrors.aliyun.com'],
|
|
'npm': ['registry.npmjs.org', 'registry.npmmirror.com'],
|
|
'go': ['proxy.golang.org', 'goproxy.cn']
|
|
}
|
|
return priorities.get(mirror_type, [])
|
|
|
|
def check_all(self) -> Dict[str, HealthCheckResult]:
|
|
"""检查所有镜像源"""
|
|
results = {}
|
|
|
|
for mirror_type, mirror_config in self.mirrors.items():
|
|
if not isinstance(mirror_config, dict):
|
|
continue
|
|
|
|
sources = mirror_config.get('sources', [])
|
|
for source_name in sources:
|
|
if source_name not in results:
|
|
result = self.health_checker.check_source(source_name, {'url': self._get_source_url(mirror_type, source_name)})
|
|
results[source_name] = result
|
|
|
|
return results
|
|
|
|
def _get_source_url(self, mirror_type: str, source_name: str) -> str:
|
|
"""获取源 URL"""
|
|
# 从配置中获取
|
|
sources_config = self.mirrors.get(mirror_type, {}).get('sources_config', {})
|
|
if source_name in sources_config:
|
|
return sources_config[source_name].get('url', '')
|
|
|
|
# 从 URL 模板生成
|
|
url_template = self.mirrors.get(mirror_type, {}).get('url_template', '')
|
|
if url_template and '{mirror}' in url_template:
|
|
return url_template.replace('{mirror}', source_name)
|
|
|
|
return ''
|
|
|
|
def get_active_source(self, mirror_type: str) -> Optional[str]:
|
|
"""获取当前活跃的镜像源"""
|
|
return self._active_source.get(mirror_type)
|
|
|
|
def get_source_for_request(self, mirror_type: str, original_url: str) -> str:
|
|
"""
|
|
根据故障切换策略获取请求的源 URL
|
|
|
|
Args:
|
|
mirror_type: 镜像类型
|
|
original_url: 原始 URL
|
|
|
|
Returns:
|
|
str: 实际请求的 URL
|
|
"""
|
|
if not self.failover_enabled:
|
|
return original_url
|
|
|
|
active_source = self._active_source.get(mirror_type)
|
|
if not active_source:
|
|
return original_url
|
|
|
|
source_url = self._get_source_url(mirror_type, active_source)
|
|
if not source_url:
|
|
return original_url
|
|
|
|
# 替换 URL 中的主机部分
|
|
try:
|
|
from urllib.parse import urlparse
|
|
parsed = urlparse(original_url)
|
|
|
|
# 构建新 URL
|
|
new_url = f"{parsed.scheme}://{source_url}{parsed.path}"
|
|
if parsed.query:
|
|
new_url += f"?{parsed.query}"
|
|
|
|
return new_url
|
|
except Exception:
|
|
return original_url
|
|
|
|
def perform_failover(self, mirror_type: str) -> bool:
|
|
"""
|
|
对指定镜像类型执行故障切换
|
|
|
|
Args:
|
|
mirror_type: 镜像类型
|
|
|
|
Returns:
|
|
bool: 是否成功切换
|
|
"""
|
|
priority_list = self._source_priority.get(mirror_type, [])
|
|
if not priority_list:
|
|
return False
|
|
|
|
current_index = 0
|
|
if mirror_type in self._active_source:
|
|
try:
|
|
current_index = priority_list.index(self._active_source[mirror_type])
|
|
except ValueError:
|
|
pass
|
|
|
|
# 查找下一个健康的源
|
|
old_source = self._active_source.get(mirror_type)
|
|
|
|
for i in range(current_index + 1, len(priority_list)):
|
|
source_name = priority_list[i]
|
|
result = self.health_checker.get_result(source_name)
|
|
|
|
if result and result.status == HealthStatus.HEALTHY:
|
|
self._active_source[mirror_type] = source_name
|
|
|
|
# 记录故障切换
|
|
failover_record = {
|
|
'timestamp': datetime.now().isoformat(),
|
|
'mirror_type': mirror_type,
|
|
'old_source': old_source,
|
|
'new_source': source_name,
|
|
'reason': 'Health check failed'
|
|
}
|
|
self._failover_history.append(failover_record)
|
|
|
|
# 保持历史记录在合理范围内
|
|
if len(self._failover_history) > 100:
|
|
self._failover_history = self._failover_history[-50:]
|
|
|
|
# 触发回调
|
|
if self._on_failover:
|
|
try:
|
|
self._on_failover(mirror_type, old_source, source_name)
|
|
except Exception as e:
|
|
logger.error(f"故障切换回调执行失败: {e}")
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
def get_failover_history(self, limit: int = 10) -> List[dict]:
|
|
"""获取故障切换历史"""
|
|
return self._failover_history[-limit:]
|
|
|
|
def get_health_summary(self) -> dict:
|
|
"""获取健康状态摘要"""
|
|
return {
|
|
'failover_enabled': self.failover_enabled,
|
|
'health': self.health_checker.get_stats(),
|
|
'active_sources': self._active_source.copy(),
|
|
'failover_history_count': len(self._failover_history)
|
|
}
|