复查修复(二): 同步模块与安全遗留
- 修复 remove_sync_source 引入的 stop_sync KeyError 回归(容错 get) - stop_sync 不再删除存活线程条目(防双 worker);FTP 子目录递归传真名(不再 KeyError) - 远程文件名统一 _safe_remote_name 校验(FTP/SFTP/HTTP 防路径穿越) - temp 任务名加随机后缀防碰撞;temp 状态/线程完成后清理(防 sync_state.json 膨胀) - save_sync_state 快照+原子写(临时文件+os.replace),持锁调用不死锁 - URL 打印脱敏(user:pass@ -> ***@) - _need_sync_http size=0 不再全量重下(仅按存在性) - cron 同一分钟去重;PyPI 进度只更新当前源 - v2 start_sync 不再把 bool 当 task_id - UserRecord.to_dict 脱敏(不返回 password_hash/token),新增 to_dict_private/get_user_with_password - config_hotreload 单配置源(set+persist 与热重载一致);server.py 用 get_all() - 会话创建时顺带清理过期项;JSON stats/历史读改写加锁 - 解压目标目录先校验;FTP RETR 命令注入防护;rsync --delete 目标保护 - git/urlopen 补超时;cleanup_completed_tasks 删旧留新
This commit is contained in:
@@ -1297,6 +1297,10 @@ class APIv1:
|
||||
return
|
||||
|
||||
extract_dir = os.path.join(self.config['base_dir'], target_dir)
|
||||
# 先校验目标目录本身,防止 target_dir 含 .. 把目录建到 base_dir 外
|
||||
if not is_safe_path(self.config['base_dir'], extract_dir):
|
||||
handler.send_json_response({"error": "Invalid target directory"}, 403)
|
||||
return
|
||||
os.makedirs(extract_dir, exist_ok=True)
|
||||
|
||||
with zipfile.ZipFile(archive_path, 'r') as zipf:
|
||||
@@ -1305,12 +1309,13 @@ class APIv1:
|
||||
if not is_safe_path(self.config['base_dir'], member_path):
|
||||
handler.send_json_response({"error": "Unsafe archive contents"}, 403)
|
||||
return
|
||||
members = zipf.namelist()
|
||||
zipf.extractall(extract_dir)
|
||||
|
||||
# 同步提取的文件到数据库
|
||||
if self.db:
|
||||
extracted_files = []
|
||||
for member in zipf.namelist():
|
||||
for member in members:
|
||||
member_path = os.path.join(extract_dir, member)
|
||||
if os.path.isfile(member_path):
|
||||
rel_member_path = os.path.relpath(member_path, self.config['base_dir']).replace("\\", "/")
|
||||
@@ -1327,7 +1332,7 @@ class APIv1:
|
||||
handler.send_json_response({
|
||||
"operation": "extract",
|
||||
"extract_dir": target_dir,
|
||||
"extracted_files": len(zipf.namelist())
|
||||
"extracted_files": len(members)
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -24,6 +24,37 @@ class APIv2(APIv1):
|
||||
# 管理员API处理器(始终创建,auth_type检查在装饰器中处理)
|
||||
self.admin_api = AdminAPI(config)
|
||||
|
||||
# ==================== 状态型管理器(惰性单例,跨请求保留状态) ====================
|
||||
|
||||
def _get_health_checker(self):
|
||||
"""健康检查器单例(原每请求新建导致状态丢失)"""
|
||||
if not hasattr(self, '_health_checker'):
|
||||
from core.health_check import HealthChecker
|
||||
self._health_checker = self._get_health_checker()
|
||||
return self._health_checker
|
||||
|
||||
def _get_failover_manager(self):
|
||||
"""故障转移管理器单例"""
|
||||
if not hasattr(self, '_failover_manager'):
|
||||
from core.health_check import MirrorFailoverManager
|
||||
self._failover_manager = MirrorFailoverManager(self.config)
|
||||
self._failover_manager.initialize()
|
||||
return self._failover_manager
|
||||
|
||||
def _get_restart_manager(self):
|
||||
"""优雅重启管理器单例"""
|
||||
if not hasattr(self, '_restart_manager'):
|
||||
from core.graceful_restart import GracefulRestartManager
|
||||
self._restart_manager = self._get_restart_manager()
|
||||
return self._restart_manager
|
||||
|
||||
def _get_prewarmer(self):
|
||||
"""缓存预热器单例(原每请求新建导致队列/状态丢失)"""
|
||||
if not hasattr(self, '_prewarmer'):
|
||||
from core.cache_prewarm import CachePrewarmer
|
||||
self._prewarmer = self._get_prewarmer()
|
||||
return self._prewarmer
|
||||
|
||||
def handle_request(self, handler, method, path, query_params):
|
||||
"""处理API v2请求"""
|
||||
import sys
|
||||
@@ -271,7 +302,7 @@ class APIv2(APIv1):
|
||||
elif path == 'health/stats':
|
||||
if method == 'GET':
|
||||
from core.health_check import HealthChecker
|
||||
checker = HealthChecker(self.config.get('health_check', {}))
|
||||
checker = self._get_health_checker()
|
||||
handler.send_json_response(checker.get_stats())
|
||||
else:
|
||||
handler.send_error(405)
|
||||
@@ -2283,11 +2314,12 @@ class APIv2(APIv1):
|
||||
if source_name is None:
|
||||
return super().api_start_sync(handler)
|
||||
if hasattr(handler, 'sync_manager') and handler.sync_manager:
|
||||
task_id = handler.sync_manager.start_sync(source_name)
|
||||
if task_id:
|
||||
# MirrorSyncManager.start_sync 返回 bool(True=已启动/已在运行)
|
||||
started = handler.sync_manager.start_sync(source_name)
|
||||
if started:
|
||||
handler.send_json_response({
|
||||
"success": True,
|
||||
"task_id": task_id,
|
||||
"started": True,
|
||||
"source_name": source_name
|
||||
})
|
||||
else:
|
||||
@@ -2573,7 +2605,7 @@ class APIv2(APIv1):
|
||||
from core.health_check import HealthChecker, HealthStatus
|
||||
|
||||
mirrors = self.config.get('mirrors', {})
|
||||
checker = HealthChecker(self.config.get('health_check', {}))
|
||||
checker = self._get_health_checker()
|
||||
|
||||
results = []
|
||||
for mirror_type, mirror_config in mirrors.items():
|
||||
@@ -2614,7 +2646,7 @@ class APIv2(APIv1):
|
||||
try:
|
||||
from core.health_check import HealthChecker, HealthStatus
|
||||
|
||||
checker = HealthChecker(self.config.get('health_check', {}))
|
||||
checker = self._get_health_checker()
|
||||
mirrors = self.config.get('mirrors', {})
|
||||
|
||||
# 查找源对应的镜像类型
|
||||
@@ -2659,7 +2691,7 @@ class APIv2(APIv1):
|
||||
try:
|
||||
from core.health_check import MirrorFailoverManager
|
||||
|
||||
failover = MirrorFailoverManager(self.config)
|
||||
failover = self._get_failover_manager()
|
||||
failover.initialize()
|
||||
|
||||
handler.send_json_response({
|
||||
@@ -2679,7 +2711,7 @@ class APIv2(APIv1):
|
||||
try:
|
||||
from core.health_check import MirrorFailoverManager
|
||||
|
||||
failover = MirrorFailoverManager(self.config)
|
||||
failover = self._get_failover_manager()
|
||||
failover.initialize()
|
||||
|
||||
success = failover.perform_failover(mirror_type)
|
||||
@@ -3285,7 +3317,7 @@ class APIv2(APIv1):
|
||||
|
||||
# 数据库验证
|
||||
if db:
|
||||
user = db.get_user(username)
|
||||
user = db.get_user_with_password(username)
|
||||
if user and db.verify_password(password, user['password_hash']):
|
||||
# 数据库验证成功,生成 token
|
||||
import secrets
|
||||
@@ -3388,7 +3420,7 @@ class APIv2(APIv1):
|
||||
config_pass = config.get('auth_pass', '')
|
||||
|
||||
# 验证旧密码(优先验证数据库,没有则验证配置文件)—— 强制要求,防止无旧密码改密
|
||||
user = db.get_user(username) if db else None
|
||||
user = db.get_user_with_password(username) if db else None
|
||||
if user:
|
||||
# 验证数据库密码
|
||||
if not db.verify_password(old_password, user['password_hash']):
|
||||
@@ -3985,7 +4017,7 @@ class APIv2(APIv1):
|
||||
"""获取重启状态"""
|
||||
from core.graceful_restart import GracefulRestartManager, ServerState
|
||||
|
||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
||||
restart_manager = self._get_restart_manager()
|
||||
stats = restart_manager.get_stats()
|
||||
|
||||
handler.send_json_response({
|
||||
@@ -3999,7 +4031,7 @@ class APIv2(APIv1):
|
||||
"""获取待处理请求"""
|
||||
from core.graceful_restart import GracefulRestartManager
|
||||
|
||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
||||
restart_manager = self._get_restart_manager()
|
||||
pending = restart_manager.get_pending_requests()
|
||||
|
||||
handler.send_json_response({
|
||||
@@ -4018,7 +4050,7 @@ class APIv2(APIv1):
|
||||
except ValueError:
|
||||
strategy = RestartStrategy.GRACEFUL
|
||||
|
||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
||||
restart_manager = self._get_restart_manager()
|
||||
|
||||
# 准备重启
|
||||
prepare_result = restart_manager.prepare_restart()
|
||||
@@ -4061,7 +4093,7 @@ class APIv2(APIv1):
|
||||
except json.JSONDecodeError:
|
||||
restart_strategy = RestartStrategy.GRACEFUL
|
||||
|
||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
||||
restart_manager = self._get_restart_manager()
|
||||
|
||||
# 执行重启
|
||||
result = restart_manager.perform_restart(strategy=restart_strategy)
|
||||
@@ -4072,7 +4104,7 @@ class APIv2(APIv1):
|
||||
"""立即重启服务器"""
|
||||
from core.graceful_restart import GracefulRestartManager
|
||||
|
||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
||||
restart_manager = self._get_restart_manager()
|
||||
|
||||
# 获取脚本路径
|
||||
script_path = self.config.get('main_script', 'main.py')
|
||||
@@ -4088,7 +4120,7 @@ class APIv2(APIv1):
|
||||
"""获取重启历史"""
|
||||
from core.graceful_restart import GracefulRestartManager
|
||||
|
||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
||||
restart_manager = self._get_restart_manager()
|
||||
history = restart_manager.get_restart_history()
|
||||
|
||||
handler.send_json_response({
|
||||
@@ -4182,7 +4214,7 @@ class APIv2(APIv1):
|
||||
"""获取缓存预热状态"""
|
||||
from core.cache_prewarm import CachePrewarmer
|
||||
|
||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
||||
prewarmer = self._get_prewarmer()
|
||||
status = prewarmer.get_status()
|
||||
|
||||
handler.send_json_response(status)
|
||||
@@ -4191,7 +4223,7 @@ class APIv2(APIv1):
|
||||
"""获取缓存预热统计"""
|
||||
from core.cache_prewarm import CachePrewarmer
|
||||
|
||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
||||
prewarmer = self._get_prewarmer()
|
||||
stats = prewarmer.get_stats()
|
||||
|
||||
handler.send_json_response(stats)
|
||||
@@ -4205,7 +4237,7 @@ class APIv2(APIv1):
|
||||
mirror_type = query_params.get('mirror_type', [None])[0]
|
||||
limit = int(query_params.get('limit', [50])[0])
|
||||
|
||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
||||
prewarmer = self._get_prewarmer()
|
||||
items = prewarmer.get_items(status=status, mirror_type=mirror_type, limit=limit)
|
||||
|
||||
handler.send_json_response({
|
||||
@@ -4217,7 +4249,7 @@ class APIv2(APIv1):
|
||||
"""获取预热历史"""
|
||||
from core.cache_prewarm import CachePrewarmer
|
||||
|
||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
||||
prewarmer = self._get_prewarmer()
|
||||
history = prewarmer.get_history()
|
||||
|
||||
handler.send_json_response({
|
||||
@@ -4234,7 +4266,7 @@ class APIv2(APIv1):
|
||||
limit = int(query_params.get('limit', [50])[0])
|
||||
priority = query_params.get('priority', ['medium'])[0]
|
||||
|
||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
||||
prewarmer = self._get_prewarmer()
|
||||
|
||||
# 如果指定了镜像类型,只预热该类型
|
||||
targets = None
|
||||
@@ -4277,7 +4309,7 @@ class APIv2(APIv1):
|
||||
|
||||
from core.cache_prewarm import CachePrewarmer
|
||||
|
||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
||||
prewarmer = self._get_prewarmer()
|
||||
prewarmer.add_items_batch(mirror_type, items, priority)
|
||||
|
||||
handler.send_json_response({
|
||||
@@ -4313,7 +4345,7 @@ class APIv2(APIv1):
|
||||
}, 400)
|
||||
return
|
||||
|
||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
||||
prewarmer = self._get_prewarmer()
|
||||
popular = prewarmer.get_popular_items(mirror_type)
|
||||
|
||||
if limit:
|
||||
@@ -4335,7 +4367,7 @@ class APIv2(APIv1):
|
||||
|
||||
mirror_type = query_params.get('mirror_type', [None])[0]
|
||||
|
||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
||||
prewarmer = self._get_prewarmer()
|
||||
|
||||
if mirror_type:
|
||||
items = prewarmer.get_popular_items(mirror_type)
|
||||
@@ -4355,7 +4387,7 @@ class APIv2(APIv1):
|
||||
"""清空预热队列"""
|
||||
from core.cache_prewarm import CachePrewarmer
|
||||
|
||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
||||
prewarmer = self._get_prewarmer()
|
||||
prewarmer.clear_items()
|
||||
|
||||
handler.send_json_response({
|
||||
|
||||
@@ -396,6 +396,10 @@ class APIAuthManager:
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
# 顺带清理过期会话(防止字典无限增长,配合懒清理)
|
||||
now = time.time()
|
||||
for sid in [sid for sid, sess in self.sessions.items() if sess.expires_at <= now]:
|
||||
del self.sessions[sid]
|
||||
self.sessions[session_id] = session
|
||||
self._save_sessions()
|
||||
|
||||
|
||||
@@ -270,12 +270,18 @@ class ConfigHotReloader:
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
"""配置管理器 - 支持热更新"""
|
||||
"""配置管理器 - 支持热更新(单配置源: 热重载开启时以 reloader 的配置为准)"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any] = None):
|
||||
self.config = config or {}
|
||||
self._hot_reloader: Optional[ConfigHotReloader] = None
|
||||
|
||||
def _effective_config(self) -> Dict[str, Any]:
|
||||
"""获取当前生效配置(避免与 reloader._config 双配置源分裂)"""
|
||||
if self._hot_reloader is not None:
|
||||
return self._hot_reloader._config
|
||||
return self.config
|
||||
|
||||
def load_from_file(self, path: str, enable_watch: bool = False) -> bool:
|
||||
"""从文件加载配置"""
|
||||
if not os.path.exists(path):
|
||||
@@ -301,13 +307,13 @@ class ConfigManager:
|
||||
return False
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
"""获取配置值"""
|
||||
return self.config.get(key, default)
|
||||
"""获取配置值(读当前生效配置)"""
|
||||
return self._effective_config().get(key, default)
|
||||
|
||||
def set(self, key: str, value: Any, persist: bool = False, path: str = None) -> bool:
|
||||
"""设置配置值"""
|
||||
"""设置配置值(写入当前生效配置,持久化时与 reloader 一致)"""
|
||||
keys = key.split('.')
|
||||
current = self.config
|
||||
current = self._effective_config()
|
||||
|
||||
for k in keys[:-1]:
|
||||
if k not in current:
|
||||
@@ -318,11 +324,12 @@ class ConfigManager:
|
||||
|
||||
if persist:
|
||||
if self._hot_reloader:
|
||||
# set 已写入 reloader._config, save 落盘的就是新值
|
||||
return self._hot_reloader.save(path)
|
||||
elif path:
|
||||
try:
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
json.dump(self.config, f, ensure_ascii=False, indent=4)
|
||||
json.dump(self._effective_config(), f, ensure_ascii=False, indent=4)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存配置失败: {e}")
|
||||
@@ -336,5 +343,5 @@ class ConfigManager:
|
||||
self._hot_reloader.add_change_listener(callback)
|
||||
|
||||
def get_all(self) -> Dict:
|
||||
"""获取完整配置"""
|
||||
return self.config.copy()
|
||||
"""获取完整配置(当前生效版本)"""
|
||||
return self._effective_config().copy()
|
||||
|
||||
+17
-3
@@ -369,11 +369,10 @@ class UserRecord(Base):
|
||||
enabled = Column(Boolean, default=True)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""安全序列化: 不返回密码哈希与 token(防止泄露给前端/日志)"""
|
||||
return {
|
||||
'id': self.id,
|
||||
'username': self.username,
|
||||
'password_hash': self.password_hash,
|
||||
'token': self.token,
|
||||
'token_expires_at': self.token_expires_at,
|
||||
'role': self.role,
|
||||
'email': self.email,
|
||||
@@ -387,6 +386,13 @@ class UserRecord(Base):
|
||||
'enabled': self.enabled
|
||||
}
|
||||
|
||||
def to_dict_private(self) -> dict:
|
||||
"""内部序列化: 含密码哈希与 token,仅供认证内部逻辑使用"""
|
||||
data = self.to_dict()
|
||||
data['password_hash'] = self.password_hash
|
||||
data['token'] = self.token
|
||||
return data
|
||||
|
||||
|
||||
class LoginLogRecord(Base):
|
||||
"""登录日志记录表"""
|
||||
@@ -1212,13 +1218,21 @@ class DatabaseManager:
|
||||
return {'success': True, 'user_id': user.id}
|
||||
|
||||
def get_user(self, username: str) -> dict:
|
||||
"""获取用户信息"""
|
||||
"""获取用户信息(安全版,不含 password_hash/token)"""
|
||||
with self.session() as session:
|
||||
user = session.query(UserRecord).filter_by(username=username).first()
|
||||
if user:
|
||||
return user.to_dict()
|
||||
return None
|
||||
|
||||
def get_user_with_password(self, username: str) -> dict:
|
||||
"""获取用户信息(含密码哈希,仅供密码验证内部使用)"""
|
||||
with self.session() as session:
|
||||
user = session.query(UserRecord).filter_by(username=username).first()
|
||||
if user:
|
||||
return user.to_dict_private()
|
||||
return None
|
||||
|
||||
def get_user_by_id(self, user_id: int) -> dict:
|
||||
"""通过ID获取用户信息"""
|
||||
with self.session() as session:
|
||||
|
||||
+120
-49
@@ -97,10 +97,14 @@ class MirrorSyncManager:
|
||||
self.sync_status = {}
|
||||
|
||||
def save_sync_state(self):
|
||||
"""保存同步状态"""
|
||||
"""保存同步状态(快照 + 原子写,可在持锁状态下调用,不会死锁)"""
|
||||
try:
|
||||
with open(self.sync_state_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(self.sync_status, f, ensure_ascii=False, indent=2)
|
||||
# dict() 浅拷贝是单条 C 操作,并发修改安全
|
||||
snapshot = dict(self.sync_status)
|
||||
tmp_file = self.sync_state_file + '.tmp'
|
||||
with open(tmp_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(snapshot, f, ensure_ascii=False, indent=2)
|
||||
os.replace(tmp_file, self.sync_state_file)
|
||||
except Exception as e:
|
||||
print(f"保存同步状态失败: {e}")
|
||||
|
||||
@@ -249,18 +253,18 @@ class MirrorSyncManager:
|
||||
if name not in self.sync_sources:
|
||||
return False
|
||||
|
||||
if name in self.sync_threads and self.sync_threads[name].is_alive():
|
||||
return True # 已经在运行
|
||||
|
||||
# 设置运行标志,确保同步循环可以执行
|
||||
self.running = True
|
||||
|
||||
with self.sync_lock:
|
||||
existing = self.sync_threads.get(name)
|
||||
if existing and existing.is_alive():
|
||||
return True # 已经在运行(检查+记录在同一把锁内,防双 worker 竞态)
|
||||
|
||||
# 设置运行标志,确保同步循环可以执行
|
||||
self.running = True
|
||||
self.sync_status[name]['status'] = 'syncing'
|
||||
self.sync_status[name]['error'] = None
|
||||
|
||||
thread = Thread(target=self._sync_worker, args=(name,), daemon=True)
|
||||
self.sync_threads[name] = thread
|
||||
thread = Thread(target=self._sync_worker, args=(name,), daemon=True)
|
||||
self.sync_threads[name] = thread
|
||||
thread.start()
|
||||
return True
|
||||
|
||||
@@ -280,12 +284,14 @@ class MirrorSyncManager:
|
||||
if not packages or not isinstance(packages, list):
|
||||
return {"success": False, "error": "请提供有效的包名列表"}
|
||||
|
||||
# 生成临时任务ID
|
||||
# 生成临时任务ID(随机后缀防同秒碰撞)
|
||||
import time
|
||||
task_id = f"temp_sync_{int(time.time())}"
|
||||
import uuid
|
||||
_ts = int(time.time())
|
||||
task_id = f"temp_sync_{_ts}_{uuid.uuid4().hex[:6]}"
|
||||
|
||||
# 使用临时任务名进行同步
|
||||
temp_name = f"{source_name}_temp_{int(time.time())}"
|
||||
temp_name = f"{source_name}_temp_{_ts}_{uuid.uuid4().hex[:6]}"
|
||||
|
||||
# 设置状态
|
||||
with self.sync_lock:
|
||||
@@ -333,17 +339,27 @@ class MirrorSyncManager:
|
||||
self.sync_status[temp_name]['status'] = 'error'
|
||||
self.sync_status[temp_name]['error'] = str(e)
|
||||
print(f"[Temp Sync] 临时同步失败: {e}")
|
||||
finally:
|
||||
# 清理临时条目,防止 sync_state.json 无限膨胀
|
||||
with self.sync_lock:
|
||||
self.sync_status.pop(temp_name, None)
|
||||
self.sync_threads.pop(temp_name, None)
|
||||
self.save_sync_state()
|
||||
|
||||
def stop_sync(self, name):
|
||||
"""停止同步指定源 - 立即停止"""
|
||||
# 立即设置状态为停止
|
||||
"""停止同步指定源 - 请求停止
|
||||
|
||||
注意: 不删除 sync_threads 条目(删除后 start_sync 会再起一个 worker,
|
||||
与仍在运行的旧线程形成双 worker)。线程结束后 is_alive() 为 False,
|
||||
start_sync 会自然复用该条目。
|
||||
"""
|
||||
# 设置状态为停止(源可能已从 sync_status 移除,容错)
|
||||
with self.sync_lock:
|
||||
self.sync_status[name]['status'] = 'stopped'
|
||||
status = self.sync_status.get(name)
|
||||
if status is not None:
|
||||
status['status'] = 'stopped'
|
||||
# 设置停止标志,让线程提前退出
|
||||
self.running = False
|
||||
# 立即返回,不等待线程结束
|
||||
if name in self.sync_threads:
|
||||
del self.sync_threads[name]
|
||||
# 短暂等待后重置运行标志
|
||||
import time
|
||||
time.sleep(0.5)
|
||||
@@ -364,7 +380,7 @@ class MirrorSyncManager:
|
||||
print(f"[SYNC] 开始同步: {name}")
|
||||
source_config = self.sync_sources[name]
|
||||
sync_type = source_config.get('type', 'http')
|
||||
print(f"[SYNC] 类型: {sync_type}, URL: {source_config.get('url', 'N/A')}")
|
||||
print(f"[SYNC] 类型: {sync_type}, URL: {self._redact_url(source_config.get('url', 'N/A'))}")
|
||||
|
||||
try:
|
||||
if sync_type in ('http', 'https'):
|
||||
@@ -394,8 +410,12 @@ class MirrorSyncManager:
|
||||
raise ValueError(f"不支持的同步类型: {sync_type}")
|
||||
|
||||
with self.sync_lock:
|
||||
self.sync_status[name]['status'] = 'completed'
|
||||
self.sync_status[name]['last_sync'] = datetime.now().isoformat()
|
||||
if not self.running:
|
||||
# 被 stop_sync 中断:标记为停止而非完成
|
||||
self.sync_status[name]['status'] = 'stopped'
|
||||
else:
|
||||
self.sync_status[name]['status'] = 'completed'
|
||||
self.sync_status[name]['last_sync'] = datetime.now().isoformat()
|
||||
|
||||
except Exception as e:
|
||||
with self.sync_lock:
|
||||
@@ -403,6 +423,9 @@ class MirrorSyncManager:
|
||||
self.sync_status[name]['error'] = str(e)
|
||||
|
||||
finally:
|
||||
# 清理线程条目(start_sync 通过 is_alive 判断, 清理后允许重新 start)
|
||||
with self.sync_lock:
|
||||
self.sync_threads.pop(name, None)
|
||||
self.save_sync_state()
|
||||
|
||||
def _sync_http(self, name, config, specific_packages=None):
|
||||
@@ -420,7 +443,7 @@ class MirrorSyncManager:
|
||||
|
||||
# 获取文件列表
|
||||
source_url = config.get('url', '')
|
||||
print(f"[HTTP Sync] 源URL: {source_url}")
|
||||
print(f"[HTTP Sync] 源URL: {self._redact_url(source_url)}")
|
||||
file_list = self._get_http_file_list(source_url, config, specific_packages)
|
||||
|
||||
total_files = len(file_list)
|
||||
@@ -445,10 +468,13 @@ class MirrorSyncManager:
|
||||
with self.sync_lock:
|
||||
self.sync_status[name]['files_synced'] = synced_count
|
||||
|
||||
# 获取本地路径
|
||||
# 获取本地路径(先做路径穿越校验)
|
||||
filename = file_info.get('name', '')
|
||||
if not filename:
|
||||
continue
|
||||
if self._safe_remote_name(filename) is None:
|
||||
print(f"[HTTP Sync] 跳过不安全的文件名: {filename!r}")
|
||||
continue
|
||||
|
||||
# 处理子目录
|
||||
local_path = os.path.join(target_dir, filename)
|
||||
@@ -476,6 +502,29 @@ class MirrorSyncManager:
|
||||
|
||||
print(f"[HTTP Sync] 完成: 成功 {synced_count}, 失败 {failed_count}")
|
||||
|
||||
@staticmethod
|
||||
def _safe_remote_name(filename):
|
||||
"""校验远程文件名/相对路径,拒绝路径穿越(.. 段、绝对路径、反斜杠)"
|
||||
|
||||
允许 '/' 分隔的多级子路径(HTTP 列表常见),但拒绝任何 '..' 段。
|
||||
"""
|
||||
if not filename or filename in ('.', '..'):
|
||||
return None
|
||||
if filename.startswith('/') or '\\' in filename:
|
||||
return None
|
||||
for seg in filename.split('/'):
|
||||
if seg in ('', '.', '..'):
|
||||
return None
|
||||
return filename
|
||||
|
||||
@staticmethod
|
||||
def _redact_url(url):
|
||||
"""脱敏 URL 中的 user:pass@ 段(防止凭证进日志/状态文件)"""
|
||||
if not url:
|
||||
return url
|
||||
import re as _re
|
||||
return _re.sub(r'(://)([^/@\s]+)@', r'\1***@', str(url))
|
||||
|
||||
def _sync_ftp(self, name, config):
|
||||
"""FTP同步实现"""
|
||||
try:
|
||||
@@ -519,15 +568,19 @@ class MirrorSyncManager:
|
||||
for filename, file_info in remote_files.items():
|
||||
if not self.running:
|
||||
break
|
||||
local_path = os.path.join(target_dir, filename)
|
||||
safe_name = self._safe_remote_name(filename)
|
||||
if safe_name is None:
|
||||
continue
|
||||
local_path = os.path.join(target_dir, safe_name)
|
||||
if file_info['is_dir']:
|
||||
sub_config = config.copy()
|
||||
sub_config['remote_path'] = os.path.join(remote_path, filename).replace('\\', '/')
|
||||
sub_config['target'] = os.path.join(config.get('target', name), filename)
|
||||
self._sync_ftp(name + '/' + filename, sub_config)
|
||||
sub_config['remote_path'] = os.path.join(remote_path, safe_name).replace('\\', '/')
|
||||
sub_config['target'] = os.path.join(config.get('target', name), safe_name)
|
||||
# 递归沿用顶层 name,避免子目录用假名导致 sync_status KeyError
|
||||
self._sync_ftp(name, sub_config)
|
||||
else:
|
||||
if self._need_sync_ftp(ftp, filename, local_path, file_info):
|
||||
if self._download_ftp_file(ftp, filename, local_path, file_info):
|
||||
if self._need_sync_ftp(ftp, safe_name, local_path, file_info):
|
||||
if self._download_ftp_file(ftp, safe_name, local_path, file_info):
|
||||
synced_count += 1
|
||||
|
||||
with self.sync_lock:
|
||||
@@ -603,13 +656,12 @@ class MirrorSyncManager:
|
||||
for file_attr in file_list:
|
||||
if not self.running:
|
||||
break
|
||||
remote_filename = file_attr.filename
|
||||
remote_filename = self._safe_remote_name(file_attr.filename)
|
||||
if remote_filename is None:
|
||||
continue
|
||||
remote_filepath = os.path.join(remote_path, remote_filename).replace('\\', '/')
|
||||
local_filepath = os.path.join(local_path, remote_filename)
|
||||
|
||||
if remote_filename in ['.', '..']:
|
||||
continue
|
||||
|
||||
if file_attr.st_mode & 0o40000:
|
||||
sub_synced = self._sync_sftp_directory(sftp, remote_filepath, local_filepath, sync_name)
|
||||
synced_count += sub_synced
|
||||
@@ -679,7 +731,13 @@ class MirrorSyncManager:
|
||||
import subprocess
|
||||
|
||||
source = config.get('source', config.get('url', ''))
|
||||
target_dir = os.path.join(self.config['base_dir'], config.get('target', name))
|
||||
target_rel = config.get('target', name) or name
|
||||
if target_rel in ('.', './', '') or os.path.isabs(target_rel):
|
||||
raise Exception(f"Rsync 同步目标不安全: {target_rel!r} (禁止同步到 '.' 或绝对路径, 防止 --delete 清空目录)")
|
||||
target_dir = os.path.join(self.config['base_dir'], target_rel)
|
||||
# 目标目录不得等于 base_dir 本身
|
||||
if os.path.realpath(target_dir) == os.path.realpath(self.config['base_dir']):
|
||||
raise Exception("Rsync 同步目标不允许是下载根目录本身 (--delete 会清空全部文件)")
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
|
||||
print(f"开始Rsync同步 {name} -> {target_dir}")
|
||||
@@ -725,10 +783,15 @@ class MirrorSyncManager:
|
||||
if os.path.exists(os.path.join(target_dir, '.git')):
|
||||
# 已存在,执行git pull
|
||||
try:
|
||||
subprocess.run(['git', 'fetch', '--all'], cwd=target_dir, check=True, capture_output=True)
|
||||
subprocess.run(['git', 'reset', '--hard', f'origin/{branch}'], cwd=target_dir, check=True, capture_output=True)
|
||||
git_timeout = config.get('git_timeout', 1800)
|
||||
subprocess.run(['git', 'fetch', '--all'], cwd=target_dir, check=True,
|
||||
capture_output=True, timeout=git_timeout)
|
||||
subprocess.run(['git', 'reset', '--hard', f'origin/{branch}'], cwd=target_dir,
|
||||
check=True, capture_output=True, timeout=git_timeout)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise Exception(f"Git pull失败: {e}")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Exception(f"Git pull超时({git_timeout}s)")
|
||||
else:
|
||||
# 克隆新仓库
|
||||
cmd = ['git', 'clone']
|
||||
@@ -739,9 +802,12 @@ class MirrorSyncManager:
|
||||
cmd.extend([repo_url, target_dir])
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=True)
|
||||
git_timeout = config.get('git_timeout', 1800)
|
||||
subprocess.run(cmd, check=True, capture_output=True, timeout=git_timeout)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise Exception(f"Git clone失败: {e}")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Exception(f"Git clone超时({git_timeout}s)")
|
||||
|
||||
def _sync_s3(self, name, config):
|
||||
"""AWS S3兼容存储同步 (S3/OSS/COS/MinIO等)"""
|
||||
@@ -888,7 +954,7 @@ class MirrorSyncManager:
|
||||
|
||||
if is_pypi:
|
||||
print(f"[HTTP] 检测到PyPI索引,使用PyPI专用解析")
|
||||
return self._get_pypi_file_list(base_url, config, specific_packages)
|
||||
return self._get_pypi_file_list(base_url, config, specific_packages, sync_name=name)
|
||||
|
||||
if 'api_url' in config:
|
||||
try:
|
||||
@@ -898,7 +964,7 @@ class MirrorSyncManager:
|
||||
encoded_auth = base64.b64encode(auth_string.encode()).decode()
|
||||
req.add_header('Authorization', f'Basic {encoded_auth}')
|
||||
|
||||
with urllib.request.urlopen(req) as response:
|
||||
with urllib.request.urlopen(req, timeout=60) as response:
|
||||
api_data = json.loads(response.read().decode())
|
||||
if 'files' in api_data:
|
||||
for file_info in api_data['files']:
|
||||
@@ -920,7 +986,7 @@ class MirrorSyncManager:
|
||||
|
||||
return file_list
|
||||
|
||||
def _get_pypi_file_list(self, base_url, config, specific_packages=None):
|
||||
def _get_pypi_file_list(self, base_url, config, specific_packages=None, sync_name=None):
|
||||
"""获取PyPI文件列表
|
||||
|
||||
Args:
|
||||
@@ -986,12 +1052,10 @@ class MirrorSyncManager:
|
||||
if not self.running:
|
||||
break
|
||||
|
||||
# 更新进度
|
||||
# 更新进度(只更新当前源,避免污染其他源状态)
|
||||
with self.sync_lock:
|
||||
if hasattr(self, 'sync_status') and self.sync_status:
|
||||
for name in self.sync_status:
|
||||
if 'total_files' in self.sync_status[name]:
|
||||
self.sync_status[name]['files_synced'] = i
|
||||
if sync_name and sync_name in self.sync_status:
|
||||
self.sync_status[sync_name]['files_synced'] = i
|
||||
|
||||
# 获取每个包的文件列表
|
||||
package_url = f"{package_index_url}{package_name}/"
|
||||
@@ -1098,8 +1162,11 @@ class MirrorSyncManager:
|
||||
"""检查HTTP文件是否需要同步"""
|
||||
if not os.path.exists(local_path):
|
||||
return True
|
||||
local_size = os.path.getsize(local_path)
|
||||
remote_size = file_info.get('size', 0)
|
||||
if remote_size <= 0:
|
||||
# 远端大小未知(PyPI/HTML 列表恒为 0):仅按存在性判断,避免每次全量重下
|
||||
return False
|
||||
local_size = os.path.getsize(local_path)
|
||||
return local_size != remote_size
|
||||
|
||||
def _need_sync_ftp(self, ftp, remote_filename, local_path, file_info):
|
||||
@@ -1213,7 +1280,7 @@ class MirrorSyncManager:
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"下载HTTP文件失败 {url}: {e}")
|
||||
print(f"下载HTTP文件失败 {self._redact_url(url)}: {e}")
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
return False
|
||||
@@ -1230,6 +1297,10 @@ class MirrorSyncManager:
|
||||
else:
|
||||
return True
|
||||
|
||||
# 防 FTP 命令注入: 文件名含 CR/LF 时拒绝
|
||||
if any(c in remote_filename for c in ('\r', '\n')):
|
||||
raise Exception(f"文件名含非法字符: {remote_filename!r}")
|
||||
|
||||
with open(local_path, mode) as f:
|
||||
if start_pos > 0:
|
||||
ftp.voidcmd('TYPE I')
|
||||
|
||||
+8
-1
@@ -133,7 +133,14 @@ class ScheduledTask:
|
||||
now = datetime.now()
|
||||
|
||||
if self.task_type == 'cron' and self.cron_parts:
|
||||
return self._matches_cron(now)
|
||||
if not self._matches_cron(now):
|
||||
return False
|
||||
# 同一分钟内只触发一次(轮询间隔 < 60s 时避免重复触发)
|
||||
if self.last_run is not None:
|
||||
last_minute = self.last_run.strftime('%Y-%m-%d %H:%M')
|
||||
if last_minute == now.strftime('%Y-%m-%d %H:%M'):
|
||||
return False
|
||||
return True
|
||||
elif self.task_type == 'interval':
|
||||
if self.last_run is None:
|
||||
return True
|
||||
|
||||
+1
-1
@@ -95,7 +95,7 @@ class MirrorServer:
|
||||
else:
|
||||
self.config_manager = config
|
||||
|
||||
self.config = self.config_manager.config
|
||||
self.config = self.config_manager.get_all()
|
||||
self.server = None
|
||||
self.sync_manager = None
|
||||
self.is_running = False
|
||||
|
||||
+4
-2
@@ -935,8 +935,10 @@ class SyncEngine:
|
||||
if task.status in [SyncStatus.COMPLETED, SyncStatus.FAILED, SyncStatus.CANCELLED]
|
||||
]
|
||||
|
||||
for task_id in completed_ids[-keep_count:]:
|
||||
del self.active_tasks[task_id]
|
||||
# 删除最旧的已完成任务,保留最近 keep_count 个(原逻辑删最新留最旧,逻辑倒置)
|
||||
if len(completed_ids) > keep_count:
|
||||
for task_id in completed_ids[:len(completed_ids) - keep_count]:
|
||||
self.active_tasks.pop(task_id, None)
|
||||
|
||||
|
||||
class SyncSource:
|
||||
|
||||
@@ -12,6 +12,7 @@ import mimetypes
|
||||
import base64
|
||||
import hashlib
|
||||
import shutil
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from urllib.parse import unquote, urlparse, parse_qs
|
||||
@@ -40,6 +41,7 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
|
||||
debug_log_file = None # 调试日志文件路径
|
||||
_debug_categories = set() # 启用的调试类别
|
||||
_mirror_handlers = {} # 镜像处理器实例缓存
|
||||
_stats_lock = threading.Lock() # 保护 JSON 统计/历史文件读改写
|
||||
|
||||
@classmethod
|
||||
def _setup_debug(cls, config):
|
||||
@@ -1421,10 +1423,11 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
|
||||
except Exception as e:
|
||||
print(f"Error updating download count in database: {e}")
|
||||
|
||||
# 回退到 JSON 文件
|
||||
stats = self.load_stats()
|
||||
stats[filepath] = stats.get(filepath, 0) + 1
|
||||
self.save_stats(stats)
|
||||
# 回退到 JSON 文件(加锁防并发读改写丢计数)
|
||||
with self._stats_lock:
|
||||
stats = self.load_stats()
|
||||
stats[filepath] = stats.get(filepath, 0) + 1
|
||||
self.save_stats(stats)
|
||||
|
||||
# ==================== 下载历史记录 ====================
|
||||
|
||||
@@ -1501,8 +1504,9 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
|
||||
except Exception as e:
|
||||
print(f"Error logging download to database: {e}")
|
||||
|
||||
# 回退到 JSON 文件
|
||||
history = self.load_download_history(1000)
|
||||
# 回退到 JSON 文件(加锁防并发写坏/丢记录)
|
||||
with self._stats_lock:
|
||||
history = self.load_download_history(1000)
|
||||
|
||||
entry = {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
@@ -1513,5 +1517,6 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
|
||||
'method': self.command if hasattr(self, 'command') else 'GET'
|
||||
}
|
||||
|
||||
history.append(entry)
|
||||
self.save_download_history(history)
|
||||
with self._stats_lock:
|
||||
history.append(entry)
|
||||
self.save_download_history(history)
|
||||
|
||||
Reference in New Issue
Block a user