复查修复(二): 同步模块与安全遗留
- 修复 remove_sync_source 引入的 stop_sync KeyError 回归(容错 get) - stop_sync 不再删除存活线程条目(防双 worker);FTP 子目录递归传真名(不再 KeyError) - 远程文件名统一 _safe_remote_name 校验(FTP/SFTP/HTTP 防路径穿越) - temp 任务名加随机后缀防碰撞;temp 状态/线程完成后清理(防 sync_state.json 膨胀) - save_sync_state 快照+原子写(临时文件+os.replace),持锁调用不死锁 - URL 打印脱敏(user:pass@ -> ***@) - _need_sync_http size=0 不再全量重下(仅按存在性) - cron 同一分钟去重;PyPI 进度只更新当前源 - v2 start_sync 不再把 bool 当 task_id - UserRecord.to_dict 脱敏(不返回 password_hash/token),新增 to_dict_private/get_user_with_password - config_hotreload 单配置源(set+persist 与热重载一致);server.py 用 get_all() - 会话创建时顺带清理过期项;JSON stats/历史读改写加锁 - 解压目标目录先校验;FTP RETR 命令注入防护;rsync --delete 目标保护 - git/urlopen 补超时;cleanup_completed_tasks 删旧留新
This commit is contained in:
@@ -1297,6 +1297,10 @@ class APIv1:
|
|||||||
return
|
return
|
||||||
|
|
||||||
extract_dir = os.path.join(self.config['base_dir'], target_dir)
|
extract_dir = os.path.join(self.config['base_dir'], target_dir)
|
||||||
|
# 先校验目标目录本身,防止 target_dir 含 .. 把目录建到 base_dir 外
|
||||||
|
if not is_safe_path(self.config['base_dir'], extract_dir):
|
||||||
|
handler.send_json_response({"error": "Invalid target directory"}, 403)
|
||||||
|
return
|
||||||
os.makedirs(extract_dir, exist_ok=True)
|
os.makedirs(extract_dir, exist_ok=True)
|
||||||
|
|
||||||
with zipfile.ZipFile(archive_path, 'r') as zipf:
|
with zipfile.ZipFile(archive_path, 'r') as zipf:
|
||||||
@@ -1305,12 +1309,13 @@ class APIv1:
|
|||||||
if not is_safe_path(self.config['base_dir'], member_path):
|
if not is_safe_path(self.config['base_dir'], member_path):
|
||||||
handler.send_json_response({"error": "Unsafe archive contents"}, 403)
|
handler.send_json_response({"error": "Unsafe archive contents"}, 403)
|
||||||
return
|
return
|
||||||
|
members = zipf.namelist()
|
||||||
zipf.extractall(extract_dir)
|
zipf.extractall(extract_dir)
|
||||||
|
|
||||||
# 同步提取的文件到数据库
|
# 同步提取的文件到数据库
|
||||||
if self.db:
|
if self.db:
|
||||||
extracted_files = []
|
extracted_files = []
|
||||||
for member in zipf.namelist():
|
for member in members:
|
||||||
member_path = os.path.join(extract_dir, member)
|
member_path = os.path.join(extract_dir, member)
|
||||||
if os.path.isfile(member_path):
|
if os.path.isfile(member_path):
|
||||||
rel_member_path = os.path.relpath(member_path, self.config['base_dir']).replace("\\", "/")
|
rel_member_path = os.path.relpath(member_path, self.config['base_dir']).replace("\\", "/")
|
||||||
@@ -1327,7 +1332,7 @@ class APIv1:
|
|||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
"operation": "extract",
|
"operation": "extract",
|
||||||
"extract_dir": target_dir,
|
"extract_dir": target_dir,
|
||||||
"extracted_files": len(zipf.namelist())
|
"extracted_files": len(members)
|
||||||
})
|
})
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -24,6 +24,37 @@ class APIv2(APIv1):
|
|||||||
# 管理员API处理器(始终创建,auth_type检查在装饰器中处理)
|
# 管理员API处理器(始终创建,auth_type检查在装饰器中处理)
|
||||||
self.admin_api = AdminAPI(config)
|
self.admin_api = AdminAPI(config)
|
||||||
|
|
||||||
|
# ==================== 状态型管理器(惰性单例,跨请求保留状态) ====================
|
||||||
|
|
||||||
|
def _get_health_checker(self):
|
||||||
|
"""健康检查器单例(原每请求新建导致状态丢失)"""
|
||||||
|
if not hasattr(self, '_health_checker'):
|
||||||
|
from core.health_check import HealthChecker
|
||||||
|
self._health_checker = self._get_health_checker()
|
||||||
|
return self._health_checker
|
||||||
|
|
||||||
|
def _get_failover_manager(self):
|
||||||
|
"""故障转移管理器单例"""
|
||||||
|
if not hasattr(self, '_failover_manager'):
|
||||||
|
from core.health_check import MirrorFailoverManager
|
||||||
|
self._failover_manager = MirrorFailoverManager(self.config)
|
||||||
|
self._failover_manager.initialize()
|
||||||
|
return self._failover_manager
|
||||||
|
|
||||||
|
def _get_restart_manager(self):
|
||||||
|
"""优雅重启管理器单例"""
|
||||||
|
if not hasattr(self, '_restart_manager'):
|
||||||
|
from core.graceful_restart import GracefulRestartManager
|
||||||
|
self._restart_manager = self._get_restart_manager()
|
||||||
|
return self._restart_manager
|
||||||
|
|
||||||
|
def _get_prewarmer(self):
|
||||||
|
"""缓存预热器单例(原每请求新建导致队列/状态丢失)"""
|
||||||
|
if not hasattr(self, '_prewarmer'):
|
||||||
|
from core.cache_prewarm import CachePrewarmer
|
||||||
|
self._prewarmer = self._get_prewarmer()
|
||||||
|
return self._prewarmer
|
||||||
|
|
||||||
def handle_request(self, handler, method, path, query_params):
|
def handle_request(self, handler, method, path, query_params):
|
||||||
"""处理API v2请求"""
|
"""处理API v2请求"""
|
||||||
import sys
|
import sys
|
||||||
@@ -271,7 +302,7 @@ class APIv2(APIv1):
|
|||||||
elif path == 'health/stats':
|
elif path == 'health/stats':
|
||||||
if method == 'GET':
|
if method == 'GET':
|
||||||
from core.health_check import HealthChecker
|
from core.health_check import HealthChecker
|
||||||
checker = HealthChecker(self.config.get('health_check', {}))
|
checker = self._get_health_checker()
|
||||||
handler.send_json_response(checker.get_stats())
|
handler.send_json_response(checker.get_stats())
|
||||||
else:
|
else:
|
||||||
handler.send_error(405)
|
handler.send_error(405)
|
||||||
@@ -2283,11 +2314,12 @@ class APIv2(APIv1):
|
|||||||
if source_name is None:
|
if source_name is None:
|
||||||
return super().api_start_sync(handler)
|
return super().api_start_sync(handler)
|
||||||
if hasattr(handler, 'sync_manager') and handler.sync_manager:
|
if hasattr(handler, 'sync_manager') and handler.sync_manager:
|
||||||
task_id = handler.sync_manager.start_sync(source_name)
|
# MirrorSyncManager.start_sync 返回 bool(True=已启动/已在运行)
|
||||||
if task_id:
|
started = handler.sync_manager.start_sync(source_name)
|
||||||
|
if started:
|
||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
"success": True,
|
"success": True,
|
||||||
"task_id": task_id,
|
"started": True,
|
||||||
"source_name": source_name
|
"source_name": source_name
|
||||||
})
|
})
|
||||||
else:
|
else:
|
||||||
@@ -2573,7 +2605,7 @@ class APIv2(APIv1):
|
|||||||
from core.health_check import HealthChecker, HealthStatus
|
from core.health_check import HealthChecker, HealthStatus
|
||||||
|
|
||||||
mirrors = self.config.get('mirrors', {})
|
mirrors = self.config.get('mirrors', {})
|
||||||
checker = HealthChecker(self.config.get('health_check', {}))
|
checker = self._get_health_checker()
|
||||||
|
|
||||||
results = []
|
results = []
|
||||||
for mirror_type, mirror_config in mirrors.items():
|
for mirror_type, mirror_config in mirrors.items():
|
||||||
@@ -2614,7 +2646,7 @@ class APIv2(APIv1):
|
|||||||
try:
|
try:
|
||||||
from core.health_check import HealthChecker, HealthStatus
|
from core.health_check import HealthChecker, HealthStatus
|
||||||
|
|
||||||
checker = HealthChecker(self.config.get('health_check', {}))
|
checker = self._get_health_checker()
|
||||||
mirrors = self.config.get('mirrors', {})
|
mirrors = self.config.get('mirrors', {})
|
||||||
|
|
||||||
# 查找源对应的镜像类型
|
# 查找源对应的镜像类型
|
||||||
@@ -2659,7 +2691,7 @@ class APIv2(APIv1):
|
|||||||
try:
|
try:
|
||||||
from core.health_check import MirrorFailoverManager
|
from core.health_check import MirrorFailoverManager
|
||||||
|
|
||||||
failover = MirrorFailoverManager(self.config)
|
failover = self._get_failover_manager()
|
||||||
failover.initialize()
|
failover.initialize()
|
||||||
|
|
||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
@@ -2679,7 +2711,7 @@ class APIv2(APIv1):
|
|||||||
try:
|
try:
|
||||||
from core.health_check import MirrorFailoverManager
|
from core.health_check import MirrorFailoverManager
|
||||||
|
|
||||||
failover = MirrorFailoverManager(self.config)
|
failover = self._get_failover_manager()
|
||||||
failover.initialize()
|
failover.initialize()
|
||||||
|
|
||||||
success = failover.perform_failover(mirror_type)
|
success = failover.perform_failover(mirror_type)
|
||||||
@@ -3285,7 +3317,7 @@ class APIv2(APIv1):
|
|||||||
|
|
||||||
# 数据库验证
|
# 数据库验证
|
||||||
if db:
|
if db:
|
||||||
user = db.get_user(username)
|
user = db.get_user_with_password(username)
|
||||||
if user and db.verify_password(password, user['password_hash']):
|
if user and db.verify_password(password, user['password_hash']):
|
||||||
# 数据库验证成功,生成 token
|
# 数据库验证成功,生成 token
|
||||||
import secrets
|
import secrets
|
||||||
@@ -3388,7 +3420,7 @@ class APIv2(APIv1):
|
|||||||
config_pass = config.get('auth_pass', '')
|
config_pass = config.get('auth_pass', '')
|
||||||
|
|
||||||
# 验证旧密码(优先验证数据库,没有则验证配置文件)—— 强制要求,防止无旧密码改密
|
# 验证旧密码(优先验证数据库,没有则验证配置文件)—— 强制要求,防止无旧密码改密
|
||||||
user = db.get_user(username) if db else None
|
user = db.get_user_with_password(username) if db else None
|
||||||
if user:
|
if user:
|
||||||
# 验证数据库密码
|
# 验证数据库密码
|
||||||
if not db.verify_password(old_password, user['password_hash']):
|
if not db.verify_password(old_password, user['password_hash']):
|
||||||
@@ -3985,7 +4017,7 @@ class APIv2(APIv1):
|
|||||||
"""获取重启状态"""
|
"""获取重启状态"""
|
||||||
from core.graceful_restart import GracefulRestartManager, ServerState
|
from core.graceful_restart import GracefulRestartManager, ServerState
|
||||||
|
|
||||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
restart_manager = self._get_restart_manager()
|
||||||
stats = restart_manager.get_stats()
|
stats = restart_manager.get_stats()
|
||||||
|
|
||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
@@ -3999,7 +4031,7 @@ class APIv2(APIv1):
|
|||||||
"""获取待处理请求"""
|
"""获取待处理请求"""
|
||||||
from core.graceful_restart import GracefulRestartManager
|
from core.graceful_restart import GracefulRestartManager
|
||||||
|
|
||||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
restart_manager = self._get_restart_manager()
|
||||||
pending = restart_manager.get_pending_requests()
|
pending = restart_manager.get_pending_requests()
|
||||||
|
|
||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
@@ -4018,7 +4050,7 @@ class APIv2(APIv1):
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
strategy = RestartStrategy.GRACEFUL
|
strategy = RestartStrategy.GRACEFUL
|
||||||
|
|
||||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
restart_manager = self._get_restart_manager()
|
||||||
|
|
||||||
# 准备重启
|
# 准备重启
|
||||||
prepare_result = restart_manager.prepare_restart()
|
prepare_result = restart_manager.prepare_restart()
|
||||||
@@ -4061,7 +4093,7 @@ class APIv2(APIv1):
|
|||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
restart_strategy = RestartStrategy.GRACEFUL
|
restart_strategy = RestartStrategy.GRACEFUL
|
||||||
|
|
||||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
restart_manager = self._get_restart_manager()
|
||||||
|
|
||||||
# 执行重启
|
# 执行重启
|
||||||
result = restart_manager.perform_restart(strategy=restart_strategy)
|
result = restart_manager.perform_restart(strategy=restart_strategy)
|
||||||
@@ -4072,7 +4104,7 @@ class APIv2(APIv1):
|
|||||||
"""立即重启服务器"""
|
"""立即重启服务器"""
|
||||||
from core.graceful_restart import GracefulRestartManager
|
from core.graceful_restart import GracefulRestartManager
|
||||||
|
|
||||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
restart_manager = self._get_restart_manager()
|
||||||
|
|
||||||
# 获取脚本路径
|
# 获取脚本路径
|
||||||
script_path = self.config.get('main_script', 'main.py')
|
script_path = self.config.get('main_script', 'main.py')
|
||||||
@@ -4088,7 +4120,7 @@ class APIv2(APIv1):
|
|||||||
"""获取重启历史"""
|
"""获取重启历史"""
|
||||||
from core.graceful_restart import GracefulRestartManager
|
from core.graceful_restart import GracefulRestartManager
|
||||||
|
|
||||||
restart_manager = GracefulRestartManager(self.config.get('restart', {}))
|
restart_manager = self._get_restart_manager()
|
||||||
history = restart_manager.get_restart_history()
|
history = restart_manager.get_restart_history()
|
||||||
|
|
||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
@@ -4182,7 +4214,7 @@ class APIv2(APIv1):
|
|||||||
"""获取缓存预热状态"""
|
"""获取缓存预热状态"""
|
||||||
from core.cache_prewarm import CachePrewarmer
|
from core.cache_prewarm import CachePrewarmer
|
||||||
|
|
||||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
prewarmer = self._get_prewarmer()
|
||||||
status = prewarmer.get_status()
|
status = prewarmer.get_status()
|
||||||
|
|
||||||
handler.send_json_response(status)
|
handler.send_json_response(status)
|
||||||
@@ -4191,7 +4223,7 @@ class APIv2(APIv1):
|
|||||||
"""获取缓存预热统计"""
|
"""获取缓存预热统计"""
|
||||||
from core.cache_prewarm import CachePrewarmer
|
from core.cache_prewarm import CachePrewarmer
|
||||||
|
|
||||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
prewarmer = self._get_prewarmer()
|
||||||
stats = prewarmer.get_stats()
|
stats = prewarmer.get_stats()
|
||||||
|
|
||||||
handler.send_json_response(stats)
|
handler.send_json_response(stats)
|
||||||
@@ -4205,7 +4237,7 @@ class APIv2(APIv1):
|
|||||||
mirror_type = query_params.get('mirror_type', [None])[0]
|
mirror_type = query_params.get('mirror_type', [None])[0]
|
||||||
limit = int(query_params.get('limit', [50])[0])
|
limit = int(query_params.get('limit', [50])[0])
|
||||||
|
|
||||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
prewarmer = self._get_prewarmer()
|
||||||
items = prewarmer.get_items(status=status, mirror_type=mirror_type, limit=limit)
|
items = prewarmer.get_items(status=status, mirror_type=mirror_type, limit=limit)
|
||||||
|
|
||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
@@ -4217,7 +4249,7 @@ class APIv2(APIv1):
|
|||||||
"""获取预热历史"""
|
"""获取预热历史"""
|
||||||
from core.cache_prewarm import CachePrewarmer
|
from core.cache_prewarm import CachePrewarmer
|
||||||
|
|
||||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
prewarmer = self._get_prewarmer()
|
||||||
history = prewarmer.get_history()
|
history = prewarmer.get_history()
|
||||||
|
|
||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
@@ -4234,7 +4266,7 @@ class APIv2(APIv1):
|
|||||||
limit = int(query_params.get('limit', [50])[0])
|
limit = int(query_params.get('limit', [50])[0])
|
||||||
priority = query_params.get('priority', ['medium'])[0]
|
priority = query_params.get('priority', ['medium'])[0]
|
||||||
|
|
||||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
prewarmer = self._get_prewarmer()
|
||||||
|
|
||||||
# 如果指定了镜像类型,只预热该类型
|
# 如果指定了镜像类型,只预热该类型
|
||||||
targets = None
|
targets = None
|
||||||
@@ -4277,7 +4309,7 @@ class APIv2(APIv1):
|
|||||||
|
|
||||||
from core.cache_prewarm import CachePrewarmer
|
from core.cache_prewarm import CachePrewarmer
|
||||||
|
|
||||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
prewarmer = self._get_prewarmer()
|
||||||
prewarmer.add_items_batch(mirror_type, items, priority)
|
prewarmer.add_items_batch(mirror_type, items, priority)
|
||||||
|
|
||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
@@ -4313,7 +4345,7 @@ class APIv2(APIv1):
|
|||||||
}, 400)
|
}, 400)
|
||||||
return
|
return
|
||||||
|
|
||||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
prewarmer = self._get_prewarmer()
|
||||||
popular = prewarmer.get_popular_items(mirror_type)
|
popular = prewarmer.get_popular_items(mirror_type)
|
||||||
|
|
||||||
if limit:
|
if limit:
|
||||||
@@ -4335,7 +4367,7 @@ class APIv2(APIv1):
|
|||||||
|
|
||||||
mirror_type = query_params.get('mirror_type', [None])[0]
|
mirror_type = query_params.get('mirror_type', [None])[0]
|
||||||
|
|
||||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
prewarmer = self._get_prewarmer()
|
||||||
|
|
||||||
if mirror_type:
|
if mirror_type:
|
||||||
items = prewarmer.get_popular_items(mirror_type)
|
items = prewarmer.get_popular_items(mirror_type)
|
||||||
@@ -4355,7 +4387,7 @@ class APIv2(APIv1):
|
|||||||
"""清空预热队列"""
|
"""清空预热队列"""
|
||||||
from core.cache_prewarm import CachePrewarmer
|
from core.cache_prewarm import CachePrewarmer
|
||||||
|
|
||||||
prewarmer = CachePrewarmer(self.config.get('cache_prewarm', {}))
|
prewarmer = self._get_prewarmer()
|
||||||
prewarmer.clear_items()
|
prewarmer.clear_items()
|
||||||
|
|
||||||
handler.send_json_response({
|
handler.send_json_response({
|
||||||
|
|||||||
@@ -396,6 +396,10 @@ class APIAuthManager:
|
|||||||
)
|
)
|
||||||
|
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
# 顺带清理过期会话(防止字典无限增长,配合懒清理)
|
||||||
|
now = time.time()
|
||||||
|
for sid in [sid for sid, sess in self.sessions.items() if sess.expires_at <= now]:
|
||||||
|
del self.sessions[sid]
|
||||||
self.sessions[session_id] = session
|
self.sessions[session_id] = session
|
||||||
self._save_sessions()
|
self._save_sessions()
|
||||||
|
|
||||||
|
|||||||
@@ -270,12 +270,18 @@ class ConfigHotReloader:
|
|||||||
|
|
||||||
|
|
||||||
class ConfigManager:
|
class ConfigManager:
|
||||||
"""配置管理器 - 支持热更新"""
|
"""配置管理器 - 支持热更新(单配置源: 热重载开启时以 reloader 的配置为准)"""
|
||||||
|
|
||||||
def __init__(self, config: Dict[str, Any] = None):
|
def __init__(self, config: Dict[str, Any] = None):
|
||||||
self.config = config or {}
|
self.config = config or {}
|
||||||
self._hot_reloader: Optional[ConfigHotReloader] = None
|
self._hot_reloader: Optional[ConfigHotReloader] = None
|
||||||
|
|
||||||
|
def _effective_config(self) -> Dict[str, Any]:
|
||||||
|
"""获取当前生效配置(避免与 reloader._config 双配置源分裂)"""
|
||||||
|
if self._hot_reloader is not None:
|
||||||
|
return self._hot_reloader._config
|
||||||
|
return self.config
|
||||||
|
|
||||||
def load_from_file(self, path: str, enable_watch: bool = False) -> bool:
|
def load_from_file(self, path: str, enable_watch: bool = False) -> bool:
|
||||||
"""从文件加载配置"""
|
"""从文件加载配置"""
|
||||||
if not os.path.exists(path):
|
if not os.path.exists(path):
|
||||||
@@ -301,13 +307,13 @@ class ConfigManager:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
def get(self, key: str, default: Any = None) -> Any:
|
def get(self, key: str, default: Any = None) -> Any:
|
||||||
"""获取配置值"""
|
"""获取配置值(读当前生效配置)"""
|
||||||
return self.config.get(key, default)
|
return self._effective_config().get(key, default)
|
||||||
|
|
||||||
def set(self, key: str, value: Any, persist: bool = False, path: str = None) -> bool:
|
def set(self, key: str, value: Any, persist: bool = False, path: str = None) -> bool:
|
||||||
"""设置配置值"""
|
"""设置配置值(写入当前生效配置,持久化时与 reloader 一致)"""
|
||||||
keys = key.split('.')
|
keys = key.split('.')
|
||||||
current = self.config
|
current = self._effective_config()
|
||||||
|
|
||||||
for k in keys[:-1]:
|
for k in keys[:-1]:
|
||||||
if k not in current:
|
if k not in current:
|
||||||
@@ -318,11 +324,12 @@ class ConfigManager:
|
|||||||
|
|
||||||
if persist:
|
if persist:
|
||||||
if self._hot_reloader:
|
if self._hot_reloader:
|
||||||
|
# set 已写入 reloader._config, save 落盘的就是新值
|
||||||
return self._hot_reloader.save(path)
|
return self._hot_reloader.save(path)
|
||||||
elif path:
|
elif path:
|
||||||
try:
|
try:
|
||||||
with open(path, 'w', encoding='utf-8') as f:
|
with open(path, 'w', encoding='utf-8') as f:
|
||||||
json.dump(self.config, f, ensure_ascii=False, indent=4)
|
json.dump(self._effective_config(), f, ensure_ascii=False, indent=4)
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"保存配置失败: {e}")
|
logger.error(f"保存配置失败: {e}")
|
||||||
@@ -336,5 +343,5 @@ class ConfigManager:
|
|||||||
self._hot_reloader.add_change_listener(callback)
|
self._hot_reloader.add_change_listener(callback)
|
||||||
|
|
||||||
def get_all(self) -> Dict:
|
def get_all(self) -> Dict:
|
||||||
"""获取完整配置"""
|
"""获取完整配置(当前生效版本)"""
|
||||||
return self.config.copy()
|
return self._effective_config().copy()
|
||||||
|
|||||||
+17
-3
@@ -369,11 +369,10 @@ class UserRecord(Base):
|
|||||||
enabled = Column(Boolean, default=True)
|
enabled = Column(Boolean, default=True)
|
||||||
|
|
||||||
def to_dict(self) -> dict:
|
def to_dict(self) -> dict:
|
||||||
|
"""安全序列化: 不返回密码哈希与 token(防止泄露给前端/日志)"""
|
||||||
return {
|
return {
|
||||||
'id': self.id,
|
'id': self.id,
|
||||||
'username': self.username,
|
'username': self.username,
|
||||||
'password_hash': self.password_hash,
|
|
||||||
'token': self.token,
|
|
||||||
'token_expires_at': self.token_expires_at,
|
'token_expires_at': self.token_expires_at,
|
||||||
'role': self.role,
|
'role': self.role,
|
||||||
'email': self.email,
|
'email': self.email,
|
||||||
@@ -387,6 +386,13 @@ class UserRecord(Base):
|
|||||||
'enabled': self.enabled
|
'enabled': self.enabled
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def to_dict_private(self) -> dict:
|
||||||
|
"""内部序列化: 含密码哈希与 token,仅供认证内部逻辑使用"""
|
||||||
|
data = self.to_dict()
|
||||||
|
data['password_hash'] = self.password_hash
|
||||||
|
data['token'] = self.token
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
class LoginLogRecord(Base):
|
class LoginLogRecord(Base):
|
||||||
"""登录日志记录表"""
|
"""登录日志记录表"""
|
||||||
@@ -1212,13 +1218,21 @@ class DatabaseManager:
|
|||||||
return {'success': True, 'user_id': user.id}
|
return {'success': True, 'user_id': user.id}
|
||||||
|
|
||||||
def get_user(self, username: str) -> dict:
|
def get_user(self, username: str) -> dict:
|
||||||
"""获取用户信息"""
|
"""获取用户信息(安全版,不含 password_hash/token)"""
|
||||||
with self.session() as session:
|
with self.session() as session:
|
||||||
user = session.query(UserRecord).filter_by(username=username).first()
|
user = session.query(UserRecord).filter_by(username=username).first()
|
||||||
if user:
|
if user:
|
||||||
return user.to_dict()
|
return user.to_dict()
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def get_user_with_password(self, username: str) -> dict:
|
||||||
|
"""获取用户信息(含密码哈希,仅供密码验证内部使用)"""
|
||||||
|
with self.session() as session:
|
||||||
|
user = session.query(UserRecord).filter_by(username=username).first()
|
||||||
|
if user:
|
||||||
|
return user.to_dict_private()
|
||||||
|
return None
|
||||||
|
|
||||||
def get_user_by_id(self, user_id: int) -> dict:
|
def get_user_by_id(self, user_id: int) -> dict:
|
||||||
"""通过ID获取用户信息"""
|
"""通过ID获取用户信息"""
|
||||||
with self.session() as session:
|
with self.session() as session:
|
||||||
|
|||||||
+114
-43
@@ -97,10 +97,14 @@ class MirrorSyncManager:
|
|||||||
self.sync_status = {}
|
self.sync_status = {}
|
||||||
|
|
||||||
def save_sync_state(self):
|
def save_sync_state(self):
|
||||||
"""保存同步状态"""
|
"""保存同步状态(快照 + 原子写,可在持锁状态下调用,不会死锁)"""
|
||||||
try:
|
try:
|
||||||
with open(self.sync_state_file, 'w', encoding='utf-8') as f:
|
# dict() 浅拷贝是单条 C 操作,并发修改安全
|
||||||
json.dump(self.sync_status, f, ensure_ascii=False, indent=2)
|
snapshot = dict(self.sync_status)
|
||||||
|
tmp_file = self.sync_state_file + '.tmp'
|
||||||
|
with open(tmp_file, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(snapshot, f, ensure_ascii=False, indent=2)
|
||||||
|
os.replace(tmp_file, self.sync_state_file)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"保存同步状态失败: {e}")
|
print(f"保存同步状态失败: {e}")
|
||||||
|
|
||||||
@@ -249,13 +253,13 @@ class MirrorSyncManager:
|
|||||||
if name not in self.sync_sources:
|
if name not in self.sync_sources:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if name in self.sync_threads and self.sync_threads[name].is_alive():
|
with self.sync_lock:
|
||||||
return True # 已经在运行
|
existing = self.sync_threads.get(name)
|
||||||
|
if existing and existing.is_alive():
|
||||||
|
return True # 已经在运行(检查+记录在同一把锁内,防双 worker 竞态)
|
||||||
|
|
||||||
# 设置运行标志,确保同步循环可以执行
|
# 设置运行标志,确保同步循环可以执行
|
||||||
self.running = True
|
self.running = True
|
||||||
|
|
||||||
with self.sync_lock:
|
|
||||||
self.sync_status[name]['status'] = 'syncing'
|
self.sync_status[name]['status'] = 'syncing'
|
||||||
self.sync_status[name]['error'] = None
|
self.sync_status[name]['error'] = None
|
||||||
|
|
||||||
@@ -280,12 +284,14 @@ class MirrorSyncManager:
|
|||||||
if not packages or not isinstance(packages, list):
|
if not packages or not isinstance(packages, list):
|
||||||
return {"success": False, "error": "请提供有效的包名列表"}
|
return {"success": False, "error": "请提供有效的包名列表"}
|
||||||
|
|
||||||
# 生成临时任务ID
|
# 生成临时任务ID(随机后缀防同秒碰撞)
|
||||||
import time
|
import time
|
||||||
task_id = f"temp_sync_{int(time.time())}"
|
import uuid
|
||||||
|
_ts = int(time.time())
|
||||||
|
task_id = f"temp_sync_{_ts}_{uuid.uuid4().hex[:6]}"
|
||||||
|
|
||||||
# 使用临时任务名进行同步
|
# 使用临时任务名进行同步
|
||||||
temp_name = f"{source_name}_temp_{int(time.time())}"
|
temp_name = f"{source_name}_temp_{_ts}_{uuid.uuid4().hex[:6]}"
|
||||||
|
|
||||||
# 设置状态
|
# 设置状态
|
||||||
with self.sync_lock:
|
with self.sync_lock:
|
||||||
@@ -333,17 +339,27 @@ class MirrorSyncManager:
|
|||||||
self.sync_status[temp_name]['status'] = 'error'
|
self.sync_status[temp_name]['status'] = 'error'
|
||||||
self.sync_status[temp_name]['error'] = str(e)
|
self.sync_status[temp_name]['error'] = str(e)
|
||||||
print(f"[Temp Sync] 临时同步失败: {e}")
|
print(f"[Temp Sync] 临时同步失败: {e}")
|
||||||
|
finally:
|
||||||
|
# 清理临时条目,防止 sync_state.json 无限膨胀
|
||||||
|
with self.sync_lock:
|
||||||
|
self.sync_status.pop(temp_name, None)
|
||||||
|
self.sync_threads.pop(temp_name, None)
|
||||||
|
self.save_sync_state()
|
||||||
|
|
||||||
def stop_sync(self, name):
|
def stop_sync(self, name):
|
||||||
"""停止同步指定源 - 立即停止"""
|
"""停止同步指定源 - 请求停止
|
||||||
# 立即设置状态为停止
|
|
||||||
|
注意: 不删除 sync_threads 条目(删除后 start_sync 会再起一个 worker,
|
||||||
|
与仍在运行的旧线程形成双 worker)。线程结束后 is_alive() 为 False,
|
||||||
|
start_sync 会自然复用该条目。
|
||||||
|
"""
|
||||||
|
# 设置状态为停止(源可能已从 sync_status 移除,容错)
|
||||||
with self.sync_lock:
|
with self.sync_lock:
|
||||||
self.sync_status[name]['status'] = 'stopped'
|
status = self.sync_status.get(name)
|
||||||
|
if status is not None:
|
||||||
|
status['status'] = 'stopped'
|
||||||
# 设置停止标志,让线程提前退出
|
# 设置停止标志,让线程提前退出
|
||||||
self.running = False
|
self.running = False
|
||||||
# 立即返回,不等待线程结束
|
|
||||||
if name in self.sync_threads:
|
|
||||||
del self.sync_threads[name]
|
|
||||||
# 短暂等待后重置运行标志
|
# 短暂等待后重置运行标志
|
||||||
import time
|
import time
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
@@ -364,7 +380,7 @@ class MirrorSyncManager:
|
|||||||
print(f"[SYNC] 开始同步: {name}")
|
print(f"[SYNC] 开始同步: {name}")
|
||||||
source_config = self.sync_sources[name]
|
source_config = self.sync_sources[name]
|
||||||
sync_type = source_config.get('type', 'http')
|
sync_type = source_config.get('type', 'http')
|
||||||
print(f"[SYNC] 类型: {sync_type}, URL: {source_config.get('url', 'N/A')}")
|
print(f"[SYNC] 类型: {sync_type}, URL: {self._redact_url(source_config.get('url', 'N/A'))}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if sync_type in ('http', 'https'):
|
if sync_type in ('http', 'https'):
|
||||||
@@ -394,6 +410,10 @@ class MirrorSyncManager:
|
|||||||
raise ValueError(f"不支持的同步类型: {sync_type}")
|
raise ValueError(f"不支持的同步类型: {sync_type}")
|
||||||
|
|
||||||
with self.sync_lock:
|
with self.sync_lock:
|
||||||
|
if not self.running:
|
||||||
|
# 被 stop_sync 中断:标记为停止而非完成
|
||||||
|
self.sync_status[name]['status'] = 'stopped'
|
||||||
|
else:
|
||||||
self.sync_status[name]['status'] = 'completed'
|
self.sync_status[name]['status'] = 'completed'
|
||||||
self.sync_status[name]['last_sync'] = datetime.now().isoformat()
|
self.sync_status[name]['last_sync'] = datetime.now().isoformat()
|
||||||
|
|
||||||
@@ -403,6 +423,9 @@ class MirrorSyncManager:
|
|||||||
self.sync_status[name]['error'] = str(e)
|
self.sync_status[name]['error'] = str(e)
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
|
# 清理线程条目(start_sync 通过 is_alive 判断, 清理后允许重新 start)
|
||||||
|
with self.sync_lock:
|
||||||
|
self.sync_threads.pop(name, None)
|
||||||
self.save_sync_state()
|
self.save_sync_state()
|
||||||
|
|
||||||
def _sync_http(self, name, config, specific_packages=None):
|
def _sync_http(self, name, config, specific_packages=None):
|
||||||
@@ -420,7 +443,7 @@ class MirrorSyncManager:
|
|||||||
|
|
||||||
# 获取文件列表
|
# 获取文件列表
|
||||||
source_url = config.get('url', '')
|
source_url = config.get('url', '')
|
||||||
print(f"[HTTP Sync] 源URL: {source_url}")
|
print(f"[HTTP Sync] 源URL: {self._redact_url(source_url)}")
|
||||||
file_list = self._get_http_file_list(source_url, config, specific_packages)
|
file_list = self._get_http_file_list(source_url, config, specific_packages)
|
||||||
|
|
||||||
total_files = len(file_list)
|
total_files = len(file_list)
|
||||||
@@ -445,10 +468,13 @@ class MirrorSyncManager:
|
|||||||
with self.sync_lock:
|
with self.sync_lock:
|
||||||
self.sync_status[name]['files_synced'] = synced_count
|
self.sync_status[name]['files_synced'] = synced_count
|
||||||
|
|
||||||
# 获取本地路径
|
# 获取本地路径(先做路径穿越校验)
|
||||||
filename = file_info.get('name', '')
|
filename = file_info.get('name', '')
|
||||||
if not filename:
|
if not filename:
|
||||||
continue
|
continue
|
||||||
|
if self._safe_remote_name(filename) is None:
|
||||||
|
print(f"[HTTP Sync] 跳过不安全的文件名: {filename!r}")
|
||||||
|
continue
|
||||||
|
|
||||||
# 处理子目录
|
# 处理子目录
|
||||||
local_path = os.path.join(target_dir, filename)
|
local_path = os.path.join(target_dir, filename)
|
||||||
@@ -476,6 +502,29 @@ class MirrorSyncManager:
|
|||||||
|
|
||||||
print(f"[HTTP Sync] 完成: 成功 {synced_count}, 失败 {failed_count}")
|
print(f"[HTTP Sync] 完成: 成功 {synced_count}, 失败 {failed_count}")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _safe_remote_name(filename):
|
||||||
|
"""校验远程文件名/相对路径,拒绝路径穿越(.. 段、绝对路径、反斜杠)"
|
||||||
|
|
||||||
|
允许 '/' 分隔的多级子路径(HTTP 列表常见),但拒绝任何 '..' 段。
|
||||||
|
"""
|
||||||
|
if not filename or filename in ('.', '..'):
|
||||||
|
return None
|
||||||
|
if filename.startswith('/') or '\\' in filename:
|
||||||
|
return None
|
||||||
|
for seg in filename.split('/'):
|
||||||
|
if seg in ('', '.', '..'):
|
||||||
|
return None
|
||||||
|
return filename
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _redact_url(url):
|
||||||
|
"""脱敏 URL 中的 user:pass@ 段(防止凭证进日志/状态文件)"""
|
||||||
|
if not url:
|
||||||
|
return url
|
||||||
|
import re as _re
|
||||||
|
return _re.sub(r'(://)([^/@\s]+)@', r'\1***@', str(url))
|
||||||
|
|
||||||
def _sync_ftp(self, name, config):
|
def _sync_ftp(self, name, config):
|
||||||
"""FTP同步实现"""
|
"""FTP同步实现"""
|
||||||
try:
|
try:
|
||||||
@@ -519,15 +568,19 @@ class MirrorSyncManager:
|
|||||||
for filename, file_info in remote_files.items():
|
for filename, file_info in remote_files.items():
|
||||||
if not self.running:
|
if not self.running:
|
||||||
break
|
break
|
||||||
local_path = os.path.join(target_dir, filename)
|
safe_name = self._safe_remote_name(filename)
|
||||||
|
if safe_name is None:
|
||||||
|
continue
|
||||||
|
local_path = os.path.join(target_dir, safe_name)
|
||||||
if file_info['is_dir']:
|
if file_info['is_dir']:
|
||||||
sub_config = config.copy()
|
sub_config = config.copy()
|
||||||
sub_config['remote_path'] = os.path.join(remote_path, filename).replace('\\', '/')
|
sub_config['remote_path'] = os.path.join(remote_path, safe_name).replace('\\', '/')
|
||||||
sub_config['target'] = os.path.join(config.get('target', name), filename)
|
sub_config['target'] = os.path.join(config.get('target', name), safe_name)
|
||||||
self._sync_ftp(name + '/' + filename, sub_config)
|
# 递归沿用顶层 name,避免子目录用假名导致 sync_status KeyError
|
||||||
|
self._sync_ftp(name, sub_config)
|
||||||
else:
|
else:
|
||||||
if self._need_sync_ftp(ftp, filename, local_path, file_info):
|
if self._need_sync_ftp(ftp, safe_name, local_path, file_info):
|
||||||
if self._download_ftp_file(ftp, filename, local_path, file_info):
|
if self._download_ftp_file(ftp, safe_name, local_path, file_info):
|
||||||
synced_count += 1
|
synced_count += 1
|
||||||
|
|
||||||
with self.sync_lock:
|
with self.sync_lock:
|
||||||
@@ -603,13 +656,12 @@ class MirrorSyncManager:
|
|||||||
for file_attr in file_list:
|
for file_attr in file_list:
|
||||||
if not self.running:
|
if not self.running:
|
||||||
break
|
break
|
||||||
remote_filename = file_attr.filename
|
remote_filename = self._safe_remote_name(file_attr.filename)
|
||||||
|
if remote_filename is None:
|
||||||
|
continue
|
||||||
remote_filepath = os.path.join(remote_path, remote_filename).replace('\\', '/')
|
remote_filepath = os.path.join(remote_path, remote_filename).replace('\\', '/')
|
||||||
local_filepath = os.path.join(local_path, remote_filename)
|
local_filepath = os.path.join(local_path, remote_filename)
|
||||||
|
|
||||||
if remote_filename in ['.', '..']:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if file_attr.st_mode & 0o40000:
|
if file_attr.st_mode & 0o40000:
|
||||||
sub_synced = self._sync_sftp_directory(sftp, remote_filepath, local_filepath, sync_name)
|
sub_synced = self._sync_sftp_directory(sftp, remote_filepath, local_filepath, sync_name)
|
||||||
synced_count += sub_synced
|
synced_count += sub_synced
|
||||||
@@ -679,7 +731,13 @@ class MirrorSyncManager:
|
|||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
source = config.get('source', config.get('url', ''))
|
source = config.get('source', config.get('url', ''))
|
||||||
target_dir = os.path.join(self.config['base_dir'], config.get('target', name))
|
target_rel = config.get('target', name) or name
|
||||||
|
if target_rel in ('.', './', '') or os.path.isabs(target_rel):
|
||||||
|
raise Exception(f"Rsync 同步目标不安全: {target_rel!r} (禁止同步到 '.' 或绝对路径, 防止 --delete 清空目录)")
|
||||||
|
target_dir = os.path.join(self.config['base_dir'], target_rel)
|
||||||
|
# 目标目录不得等于 base_dir 本身
|
||||||
|
if os.path.realpath(target_dir) == os.path.realpath(self.config['base_dir']):
|
||||||
|
raise Exception("Rsync 同步目标不允许是下载根目录本身 (--delete 会清空全部文件)")
|
||||||
os.makedirs(target_dir, exist_ok=True)
|
os.makedirs(target_dir, exist_ok=True)
|
||||||
|
|
||||||
print(f"开始Rsync同步 {name} -> {target_dir}")
|
print(f"开始Rsync同步 {name} -> {target_dir}")
|
||||||
@@ -725,10 +783,15 @@ class MirrorSyncManager:
|
|||||||
if os.path.exists(os.path.join(target_dir, '.git')):
|
if os.path.exists(os.path.join(target_dir, '.git')):
|
||||||
# 已存在,执行git pull
|
# 已存在,执行git pull
|
||||||
try:
|
try:
|
||||||
subprocess.run(['git', 'fetch', '--all'], cwd=target_dir, check=True, capture_output=True)
|
git_timeout = config.get('git_timeout', 1800)
|
||||||
subprocess.run(['git', 'reset', '--hard', f'origin/{branch}'], cwd=target_dir, check=True, capture_output=True)
|
subprocess.run(['git', 'fetch', '--all'], cwd=target_dir, check=True,
|
||||||
|
capture_output=True, timeout=git_timeout)
|
||||||
|
subprocess.run(['git', 'reset', '--hard', f'origin/{branch}'], cwd=target_dir,
|
||||||
|
check=True, capture_output=True, timeout=git_timeout)
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
raise Exception(f"Git pull失败: {e}")
|
raise Exception(f"Git pull失败: {e}")
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
raise Exception(f"Git pull超时({git_timeout}s)")
|
||||||
else:
|
else:
|
||||||
# 克隆新仓库
|
# 克隆新仓库
|
||||||
cmd = ['git', 'clone']
|
cmd = ['git', 'clone']
|
||||||
@@ -739,9 +802,12 @@ class MirrorSyncManager:
|
|||||||
cmd.extend([repo_url, target_dir])
|
cmd.extend([repo_url, target_dir])
|
||||||
|
|
||||||
try:
|
try:
|
||||||
subprocess.run(cmd, check=True, capture_output=True)
|
git_timeout = config.get('git_timeout', 1800)
|
||||||
|
subprocess.run(cmd, check=True, capture_output=True, timeout=git_timeout)
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
raise Exception(f"Git clone失败: {e}")
|
raise Exception(f"Git clone失败: {e}")
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
raise Exception(f"Git clone超时({git_timeout}s)")
|
||||||
|
|
||||||
def _sync_s3(self, name, config):
|
def _sync_s3(self, name, config):
|
||||||
"""AWS S3兼容存储同步 (S3/OSS/COS/MinIO等)"""
|
"""AWS S3兼容存储同步 (S3/OSS/COS/MinIO等)"""
|
||||||
@@ -888,7 +954,7 @@ class MirrorSyncManager:
|
|||||||
|
|
||||||
if is_pypi:
|
if is_pypi:
|
||||||
print(f"[HTTP] 检测到PyPI索引,使用PyPI专用解析")
|
print(f"[HTTP] 检测到PyPI索引,使用PyPI专用解析")
|
||||||
return self._get_pypi_file_list(base_url, config, specific_packages)
|
return self._get_pypi_file_list(base_url, config, specific_packages, sync_name=name)
|
||||||
|
|
||||||
if 'api_url' in config:
|
if 'api_url' in config:
|
||||||
try:
|
try:
|
||||||
@@ -898,7 +964,7 @@ class MirrorSyncManager:
|
|||||||
encoded_auth = base64.b64encode(auth_string.encode()).decode()
|
encoded_auth = base64.b64encode(auth_string.encode()).decode()
|
||||||
req.add_header('Authorization', f'Basic {encoded_auth}')
|
req.add_header('Authorization', f'Basic {encoded_auth}')
|
||||||
|
|
||||||
with urllib.request.urlopen(req) as response:
|
with urllib.request.urlopen(req, timeout=60) as response:
|
||||||
api_data = json.loads(response.read().decode())
|
api_data = json.loads(response.read().decode())
|
||||||
if 'files' in api_data:
|
if 'files' in api_data:
|
||||||
for file_info in api_data['files']:
|
for file_info in api_data['files']:
|
||||||
@@ -920,7 +986,7 @@ class MirrorSyncManager:
|
|||||||
|
|
||||||
return file_list
|
return file_list
|
||||||
|
|
||||||
def _get_pypi_file_list(self, base_url, config, specific_packages=None):
|
def _get_pypi_file_list(self, base_url, config, specific_packages=None, sync_name=None):
|
||||||
"""获取PyPI文件列表
|
"""获取PyPI文件列表
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -986,12 +1052,10 @@ class MirrorSyncManager:
|
|||||||
if not self.running:
|
if not self.running:
|
||||||
break
|
break
|
||||||
|
|
||||||
# 更新进度
|
# 更新进度(只更新当前源,避免污染其他源状态)
|
||||||
with self.sync_lock:
|
with self.sync_lock:
|
||||||
if hasattr(self, 'sync_status') and self.sync_status:
|
if sync_name and sync_name in self.sync_status:
|
||||||
for name in self.sync_status:
|
self.sync_status[sync_name]['files_synced'] = i
|
||||||
if 'total_files' in self.sync_status[name]:
|
|
||||||
self.sync_status[name]['files_synced'] = i
|
|
||||||
|
|
||||||
# 获取每个包的文件列表
|
# 获取每个包的文件列表
|
||||||
package_url = f"{package_index_url}{package_name}/"
|
package_url = f"{package_index_url}{package_name}/"
|
||||||
@@ -1098,8 +1162,11 @@ class MirrorSyncManager:
|
|||||||
"""检查HTTP文件是否需要同步"""
|
"""检查HTTP文件是否需要同步"""
|
||||||
if not os.path.exists(local_path):
|
if not os.path.exists(local_path):
|
||||||
return True
|
return True
|
||||||
local_size = os.path.getsize(local_path)
|
|
||||||
remote_size = file_info.get('size', 0)
|
remote_size = file_info.get('size', 0)
|
||||||
|
if remote_size <= 0:
|
||||||
|
# 远端大小未知(PyPI/HTML 列表恒为 0):仅按存在性判断,避免每次全量重下
|
||||||
|
return False
|
||||||
|
local_size = os.path.getsize(local_path)
|
||||||
return local_size != remote_size
|
return local_size != remote_size
|
||||||
|
|
||||||
def _need_sync_ftp(self, ftp, remote_filename, local_path, file_info):
|
def _need_sync_ftp(self, ftp, remote_filename, local_path, file_info):
|
||||||
@@ -1213,7 +1280,7 @@ class MirrorSyncManager:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"下载HTTP文件失败 {url}: {e}")
|
print(f"下载HTTP文件失败 {self._redact_url(url)}: {e}")
|
||||||
if os.path.exists(temp_path):
|
if os.path.exists(temp_path):
|
||||||
os.remove(temp_path)
|
os.remove(temp_path)
|
||||||
return False
|
return False
|
||||||
@@ -1230,6 +1297,10 @@ class MirrorSyncManager:
|
|||||||
else:
|
else:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
# 防 FTP 命令注入: 文件名含 CR/LF 时拒绝
|
||||||
|
if any(c in remote_filename for c in ('\r', '\n')):
|
||||||
|
raise Exception(f"文件名含非法字符: {remote_filename!r}")
|
||||||
|
|
||||||
with open(local_path, mode) as f:
|
with open(local_path, mode) as f:
|
||||||
if start_pos > 0:
|
if start_pos > 0:
|
||||||
ftp.voidcmd('TYPE I')
|
ftp.voidcmd('TYPE I')
|
||||||
|
|||||||
+8
-1
@@ -133,7 +133,14 @@ class ScheduledTask:
|
|||||||
now = datetime.now()
|
now = datetime.now()
|
||||||
|
|
||||||
if self.task_type == 'cron' and self.cron_parts:
|
if self.task_type == 'cron' and self.cron_parts:
|
||||||
return self._matches_cron(now)
|
if not self._matches_cron(now):
|
||||||
|
return False
|
||||||
|
# 同一分钟内只触发一次(轮询间隔 < 60s 时避免重复触发)
|
||||||
|
if self.last_run is not None:
|
||||||
|
last_minute = self.last_run.strftime('%Y-%m-%d %H:%M')
|
||||||
|
if last_minute == now.strftime('%Y-%m-%d %H:%M'):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
elif self.task_type == 'interval':
|
elif self.task_type == 'interval':
|
||||||
if self.last_run is None:
|
if self.last_run is None:
|
||||||
return True
|
return True
|
||||||
|
|||||||
+1
-1
@@ -95,7 +95,7 @@ class MirrorServer:
|
|||||||
else:
|
else:
|
||||||
self.config_manager = config
|
self.config_manager = config
|
||||||
|
|
||||||
self.config = self.config_manager.config
|
self.config = self.config_manager.get_all()
|
||||||
self.server = None
|
self.server = None
|
||||||
self.sync_manager = None
|
self.sync_manager = None
|
||||||
self.is_running = False
|
self.is_running = False
|
||||||
|
|||||||
+4
-2
@@ -935,8 +935,10 @@ class SyncEngine:
|
|||||||
if task.status in [SyncStatus.COMPLETED, SyncStatus.FAILED, SyncStatus.CANCELLED]
|
if task.status in [SyncStatus.COMPLETED, SyncStatus.FAILED, SyncStatus.CANCELLED]
|
||||||
]
|
]
|
||||||
|
|
||||||
for task_id in completed_ids[-keep_count:]:
|
# 删除最旧的已完成任务,保留最近 keep_count 个(原逻辑删最新留最旧,逻辑倒置)
|
||||||
del self.active_tasks[task_id]
|
if len(completed_ids) > keep_count:
|
||||||
|
for task_id in completed_ids[:len(completed_ids) - keep_count]:
|
||||||
|
self.active_tasks.pop(task_id, None)
|
||||||
|
|
||||||
|
|
||||||
class SyncSource:
|
class SyncSource:
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import mimetypes
|
|||||||
import base64
|
import base64
|
||||||
import hashlib
|
import hashlib
|
||||||
import shutil
|
import shutil
|
||||||
|
import threading
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from http.server import BaseHTTPRequestHandler
|
from http.server import BaseHTTPRequestHandler
|
||||||
from urllib.parse import unquote, urlparse, parse_qs
|
from urllib.parse import unquote, urlparse, parse_qs
|
||||||
@@ -40,6 +41,7 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
|
|||||||
debug_log_file = None # 调试日志文件路径
|
debug_log_file = None # 调试日志文件路径
|
||||||
_debug_categories = set() # 启用的调试类别
|
_debug_categories = set() # 启用的调试类别
|
||||||
_mirror_handlers = {} # 镜像处理器实例缓存
|
_mirror_handlers = {} # 镜像处理器实例缓存
|
||||||
|
_stats_lock = threading.Lock() # 保护 JSON 统计/历史文件读改写
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _setup_debug(cls, config):
|
def _setup_debug(cls, config):
|
||||||
@@ -1421,7 +1423,8 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error updating download count in database: {e}")
|
print(f"Error updating download count in database: {e}")
|
||||||
|
|
||||||
# 回退到 JSON 文件
|
# 回退到 JSON 文件(加锁防并发读改写丢计数)
|
||||||
|
with self._stats_lock:
|
||||||
stats = self.load_stats()
|
stats = self.load_stats()
|
||||||
stats[filepath] = stats.get(filepath, 0) + 1
|
stats[filepath] = stats.get(filepath, 0) + 1
|
||||||
self.save_stats(stats)
|
self.save_stats(stats)
|
||||||
@@ -1501,7 +1504,8 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error logging download to database: {e}")
|
print(f"Error logging download to database: {e}")
|
||||||
|
|
||||||
# 回退到 JSON 文件
|
# 回退到 JSON 文件(加锁防并发写坏/丢记录)
|
||||||
|
with self._stats_lock:
|
||||||
history = self.load_download_history(1000)
|
history = self.load_download_history(1000)
|
||||||
|
|
||||||
entry = {
|
entry = {
|
||||||
@@ -1513,5 +1517,6 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
|
|||||||
'method': self.command if hasattr(self, 'command') else 'GET'
|
'method': self.command if hasattr(self, 'command') else 'GET'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
with self._stats_lock:
|
||||||
history.append(entry)
|
history.append(entry)
|
||||||
self.save_download_history(history)
|
self.save_download_history(history)
|
||||||
|
|||||||
Reference in New Issue
Block a user