HYCEXITADMIN/mirror_server
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P2: 清理与一致性

Browse Source 
- CLI: --preset 实际生效;--directory-listing/--enable-stats/--enable-*/--ignore-hidden 支持 true/false(修复 type=bool 陷阱),未显式传入时不再覆盖 settings.json
- 版本号统一 v2.3(settings.json/config.py/v1/v2/Dockerfile)
- 移除硬编码 /tmp/pypi_debug.log 写文件;v2 裸 except 改记录错误返回 500
- webhook id 非法输入返回 400;下载趋势 timedelta 导入提升修复 NameError
- 打包模式 auth_token 写入 exe 目录(不再写入 _MEIPASS)
- README 修正: 线程池架构、pyinstaller 构建方式
- mirrors/__init__.py ' Quay.io' 笔误;chunked 响应去掉 Content-Length;docs/ui realpath 边界
- SQLite WAL+busy_timeout;下载计数原子 UPDATE
- requirements 移除未使用的 cachetools/apscheduler
This commit is contained in:
HYC Fixer
2026-08-30 12:25:26 +08:00
parent 237b99b480
commit dfae749612
12 changed files with 4020 additions and 3968 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6 -9
View File
@@ -471,7 +471,7 @@ HTTPS配置:
- **config.py**: 配置管理和验证，支持多级配置合并
- **config_hotreload.py**: 配置文件热重载，无需重启生效
- **utils.py**: 通用工具函数（文件大小格式化、哈希计算等）
- **server.py**: 服务器核心类，基于aiohttp
- **server.py**: 服务器核心类（基于标准库 http.server 的线程池服务器）
- **database.py**: 数据库管理，支持SQLite/MySQL/PostgreSQL
- **mirror_sync.py**: 镜像同步管理器（支持HTTP/HTTPS、FTP、SFTP、本地、Rsync、Git、S3/OSS/COS、WebDAV）
- **sync_scheduler.py**: 同步调度器，定时执行同步任务
@@ -553,20 +553,17 @@ HTTPS配置:
#### 构建当前平台
项目提供了 `build.py` 脚本，可以将项目打包成当前平台的二进制可执行文件。
项目提供 PyInstaller 打包配置，可以将项目打包成当前平台的二进制可执行文件。
```bash
# 安装 PyInstaller
pip install pyinstaller
# 查看支持的目标平台
python build.py --list
# 构建当前平台（使用仓库内的打包配置）
pyinstaller hyc-download.spec
# 构建当前平台
python build.py
# 清理构建缓存
python build.py --clean
# 使用旧的打包配置（含 aiohttp 隐藏依赖，已弃用）
# pyinstaller pyinstaller.spec
```
**支持的平台（仅当前平台）：**
api/v1.py
+2 -2
View File
@@ -1474,7 +1474,7 @@ class APIv1:
"""API: 获取配置信息"""
handler.send_json_response({
"server_name": self.config.get("server_name", "Mirror Server"),
"version": "2.2",
"version": "2.3",
"base_dir": self.config['base_dir'],
"directory_listing": self.config.get('directory_listing', True),
"max_upload_size": self.config.get('max_upload_size'),
@@ -2252,7 +2252,7 @@ class APIv1:
"""API: 获取镜像站信息"""
info = {
"server_name": self.config.get("server_name", "Mirror Server"),
"version": "2.1",
"version": "2.3",
"uptime": time.time() - self.config.get('start_time', time.time()),
"total_files": sum(1 for _, _, files in os.walk(self.config['base_dir']) for _ in files),
"total_size": self._get_dir_size(self.config['base_dir']),
api/v2.py
+38 -9
View File
@@ -597,11 +597,6 @@ class APIv2(APIv1):
# 获取镜像类型
mirror_type = mirror_config.get('type', 'http')
# 调试
debug_file = '/tmp/pypi_debug.log'
with open(debug_file, 'a') as f:
f.write(f"[V2] mirror_type={mirror_type}, handler_class={get_mirror_handler(mirror_type)}\n")
# 获取处理器类
handler_class = get_mirror_handler(mirror_type)
if not handler_class:
@@ -887,8 +882,17 @@ class APIv2(APIv1):
# v2 端点都没匹配到，尝试调用 APIv1
try:
return super().handle_request(handler, method, path, query_params)
except:
pass
except Exception as e:
# 记录真实错误而非吞掉（避免 v1 内部异常被误报为 404）
if handler._is_debug_enabled('error'):
import traceback
handler._debug_log('error', f"APIv1 处理失败 ({path}): {e}\n{traceback.format_exc()}")
handler.send_json_response({
"error": f"API处理错误: {str(e)}",
"version": "v1",
"path": path
}, 500)
return
# 404 - 未找到端点
handler.send_error(404)
@@ -1283,6 +1287,8 @@ class APIv2(APIv1):
def api_get_download_trend(self, handler, query_params):
"""获取下载趋势（按天统计）"""
days = int(query_params.get('days', [7])[0])
from datetime import timedelta # 供数据库与回退分支共同使用
days = min(max(days, 1), 90) # 限制 1-90 天
trend = []
@@ -1291,7 +1297,6 @@ class APIv2(APIv1):
db = self.get_db()
if db:
try:
from datetime import timedelta
from core.database import DownloadRecord
now = datetime.now()
@@ -1983,7 +1988,11 @@ class APIv2(APIv1):
def api_get_webhook(self, handler, webhook_id):
"""获取webhook详情"""
try:
webhook_id = int(webhook_id)
except (ValueError, TypeError):
handler.send_json_response({"error": "Invalid webhook id"}, 400)
return
if self.db_enabled and self.db:
try:
@@ -2006,7 +2015,11 @@ class APIv2(APIv1):
def api_delete_webhook(self, handler, webhook_id):
"""删除webhook"""
try:
webhook_id = int(webhook_id)
except (ValueError, TypeError):
handler.send_json_response({"error": "Invalid webhook id"}, 400)
return
if self.db_enabled and self.db:
try:
@@ -2031,7 +2044,11 @@ class APIv2(APIv1):
def api_test_webhook(self, handler, webhook_id):
"""测试webhook"""
try:
webhook_id = int(webhook_id)
except (ValueError, TypeError):
handler.send_json_response({"error": "Invalid webhook id"}, 400)
return
if self.db_enabled and self.db:
try:
@@ -2095,7 +2112,11 @@ class APIv2(APIv1):
def api_get_webhook_deliveries(self, handler, webhook_id):
"""获取 webhook 交付历史"""
try:
webhook_id = int(webhook_id)
except (ValueError, TypeError):
handler.send_json_response({"error": "Invalid webhook id"}, 400)
return
# 验证 webhook 存在
if self.db_enabled and self.db:
@@ -2126,7 +2147,11 @@ class APIv2(APIv1):
def api_get_webhook_stats(self, handler, webhook_id):
"""获取 webhook 交付统计"""
try:
webhook_id = int(webhook_id)
except (ValueError, TypeError):
handler.send_json_response({"error": "Invalid webhook id"}, 400)
return
if self.db_enabled and self.db:
try:
@@ -2146,7 +2171,11 @@ class APIv2(APIv1):
def api_update_webhook(self, handler, webhook_id):
"""更新 webhook 配置"""
try:
webhook_id = int(webhook_id)
except (ValueError, TypeError):
handler.send_json_response({"error": "Invalid webhook id"}, 400)
return
content_length = int(handler.headers.get('Content-Length', 0))
if content_length == 0:
@@ -3741,7 +3770,7 @@ class APIv2(APIv1):
handler.send_json_response({
"name": self.config.get('server_name', 'HYC下载站'),
"version": "2.2.0",
"version": "2.3.0",
"uptime_seconds": round(uptime_seconds, 2),
"uptime_formatted": uptime_str,
"api_version": "v2",
core/config.py
+1 -1
View File
@@ -272,7 +272,7 @@ class ConfigManager:
"""返回配置字典（不包含敏感信息）"""
safe_config = {
"server_name": self.config.get("server_name", "Mirror Server"),
"version": "2.2",
"version": "2.3",
"base_dir": self.config['base_dir'],
"directory_listing": self.config.get('directory_listing', True),
"max_upload_size": self.config.get('max_upload_size'),
core/database.py
+26 -9
View File
@@ -535,8 +535,10 @@ class DatabaseManager:
if self.db_type == 'sqlite':
# SQLite 配置
db_path = db_config.get('path', './data/hyc.db')
# 确保数据库目录存在
os.makedirs(os.path.dirname(db_path), exist_ok=True)
# 确保数据库目录存在（路径无目录部分时跳过）
db_dir = os.path.dirname(db_path)
if db_dir:
os.makedirs(db_dir, exist_ok=True)
db_url = f"sqlite:///{db_path}"
self.engine = create_engine(
@@ -544,9 +546,22 @@ class DatabaseManager:
poolclass=QueuePool,
pool_size=5,
max_overflow=10,
pool_recycle=3600
pool_recycle=3600,
connect_args={'timeout': 30} # 忙等待，减少 database is locked
)
# 启用 WAL 提升并发读写；sqlite3 是标准库，始终可用
@self.engine.event.listens_for(self.engine, 'connect')
def _set_sqlite_pragma(dbapi_connection, connection_record):
try:
cursor = dbapi_connection.cursor()
cursor.execute('PRAGMA journal_mode=WAL')
cursor.execute('PRAGMA busy_timeout=5000')
cursor.execute('PRAGMA synchronous=NORMAL')
cursor.close()
except Exception:
pass
elif self.db_type == 'postgresql':
# PostgreSQL 配置
host = db_config.get('host', 'localhost')
@@ -721,13 +736,15 @@ class DatabaseManager:
).limit(limit).all()
def increment_download_count(self, file_id: str) -> bool:
"""增加下载计数（通过 file_id 查找）"""
"""增加下载计数（原子 UPDATE，避免并发读改写丢失更新）"""
with self.session() as session:
record = session.query(FileRecord).filter(
FileRecord.file_id == file_id
).first()
if record:
record.download_count += 1
from sqlalchemy import update
result = session.execute(
update(FileRecord)
.where(FileRecord.file_id == file_id)
.values(download_count=FileRecord.download_count + 1)
)
if result.rowcount > 0:
self._operation_count += 1
return True
return False
docker/Dockerfile
+2 -2
View File
@@ -1,5 +1,5 @@
# ============================================
# HYC下载站 v2.2 - Docker 构建配置
# HYC下载站 v2.3 - Docker 构建配置
# ============================================
# 构建阶段
@@ -24,7 +24,7 @@ RUN pip install --no-cache-dir --prefix=/install -r requirements.txt
FROM python:3.11-slim
LABEL maintainer="HYC Download Station"
LABEL description="HYC下载站 v2.2 - 镜像文件服务器 + 下载加速源"
LABEL description="HYC下载站 v2.3 - 镜像文件服务器 + 下载加速源"
# 环境变量
ENV PYTHONDONTWRITEBYTECODE=1
handlers/http_handler.py
+3 -3
View File
@@ -668,7 +668,7 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
file_path = os.path.join(docs_dir, rel_path)
# 防止目录遍历
if not os.path.realpath(file_path).startswith(os.path.realpath(docs_dir)):
if not os.path.realpath(file_path).startswith(os.path.realpath(docs_dir) + os.sep):
self.send_error(403, "Access denied")
return
@@ -765,7 +765,7 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
file_path = os.path.join(ui_dir, rel_path)
# 防止目录遍历
if not os.path.realpath(file_path).startswith(os.path.realpath(ui_dir)):
if not os.path.realpath(file_path).startswith(os.path.realpath(ui_dir) + os.sep):
self.send_error(403, "Access denied")
return
@@ -1200,7 +1200,7 @@ class MirrorServerHandler(BaseHTTPRequestHandler):
mime_type, _ = mimetypes.guess_type(file_path)
self.send_response(200)
self.send_header("Content-Type", mime_type)
self.send_header("Content-Length", str(file_size))
# 使用 chunked 时不发送 Content-Length（协议不允许同时存在）
self.send_header("Content-Disposition",
f'attachment; filename="{self._safe_disposition_filename(file_path)}"')
self.send_header("Accept-Ranges", "bytes")
main.py
+44 -20
View File
@@ -41,6 +41,13 @@ def signal_handler(signum, _frame):
raise KeyboardInterrupt
def _parse_bool_arg(value):
"""解析布尔命令行参数（--flag true/false/1/0/yes/no）"""
if isinstance(value, bool):
return value
return str(value).strip().lower() in ('1', 'true', 'yes', 'on')
def parse_arguments():
"""解析命令行参数"""
parser = argparse.ArgumentParser(
@@ -97,41 +104,49 @@ def parse_arguments():
# 功能配置
func_group = parser.add_argument_group('功能配置')
func_group.add_argument('--directory-listing', type=bool, default=True,
help='启用目录浏览 (默认: True)')
func_group.add_argument('--enable-stats', type=bool, default=True,
help='启用下载统计 (默认: True)')
func_group.add_argument('--directory-listing', nargs='?', const=True, default=None,
type=_parse_bool_arg,
help='启用目录浏览 (默认: True; 传 false 可禁用)')
func_group.add_argument('--enable-stats', nargs='?', const=True, default=None,
type=_parse_bool_arg,
help='启用下载统计 (默认: True; 传 false 可禁用)')
func_group.add_argument('--show-hash', action='store_true', help='显示文件哈希值')
func_group.add_argument('--ignore-hidden', action='store_true', default=True,
help='忽略隐藏文件')
func_group.add_argument('--ignore-hidden', nargs='?', const=True, default=None,
type=_parse_bool_arg,
help='忽略隐藏文件 (默认: True; 传 false 显示隐藏文件)')
func_group.add_argument('--max-upload-size', default='1G', help='最大上传文件大小')
func_group.add_argument('--api-version', choices=['v1', 'v2'], default='v2',
help='API版本 (默认: v2)')
# 实时通信配置
realtime_group = parser.add_argument_group('实时通信配置')
realtime_group.add_argument('--enable-ws', action='store_true', default=True,
help='启用WebSocket (默认: True)')
realtime_group.add_argument('--enable-sse', action='store_true', default=True,
help='启用SSE (默认: True)')
realtime_group.add_argument('--enable-ws', nargs='?', const=True, default=None,
type=_parse_bool_arg,
help='启用WebSocket (默认: True; 传 false 可禁用)')
realtime_group.add_argument('--enable-sse', nargs='?', const=True, default=None,
type=_parse_bool_arg,
help='启用SSE (默认: True; 传 false 可禁用)')
# 系统监控配置
monitor_group = parser.add_argument_group('系统监控配置')
monitor_group.add_argument('--enable-monitor', action='store_true', default=True,
help='启用系统监控 (默认: True)')
monitor_group.add_argument('--enable-monitor', nargs='?', const=True, default=None,
type=_parse_bool_arg,
help='启用系统监控 (默认: True; 传 false 可禁用)')
monitor_group.add_argument('--monitor-interval', type=int, default=5,
help='监控数据采集间隔(秒) (默认: 5)')
# 同步配置
sync_group = parser.add_argument_group('同步配置')
sync_group.add_argument('--enable-sync', action='store_true', default=True,
help='启用镜像同步 (默认: True)')
sync_group.add_argument('--enable-sync', nargs='?', const=True, default=None,
type=_parse_bool_arg,
help='启用镜像同步 (默认: True; 传 false 可禁用)')
sync_group.add_argument('--sync-config', help='同步配置文件路径')
# 镜像加速源配置
mirror_group = parser.add_argument_group('镜像加速源配置')
mirror_group.add_argument('--enable-mirrors', action='store_true', default=True,
help='启用下载加速源 (默认: True)')
mirror_group.add_argument('--enable-mirrors', nargs='?', const=True, default=None,
type=_parse_bool_arg,
help='启用下载加速源 (默认: True; 传 false 可禁用)')
# 注意: 各个镜像的启用/禁用通过 settings.json 中的 mirrors.xxx.enabled 配置
# 下载限速
@@ -250,19 +265,28 @@ def build_config_from_args(args):
if args.base_dir:
cmd_config['base_dir'] = os.path.abspath(args.base_dir)
# 实时通信
# 实时通信（仅在显式传入时覆盖 settings.json）
if args.enable_ws is not None:
cmd_config['enable_ws'] = args.enable_ws
if args.enable_sse is not None:
cmd_config['enable_sse'] = args.enable_sse
# 系统监控
if args.enable_monitor is not None:
cmd_config['enable_monitor'] = args.enable_monitor
cmd_config['monitor_interval'] = args.monitor_interval
# 同步
if args.enable_sync is not None:
cmd_config['enable_sync'] = args.enable_sync
# 镜像加速源
if args.enable_mirrors is not None:
cmd_config['enable_mirrors'] = args.enable_mirrors
# 设备预设（--preset 实际生效）
if args.preset:
cmd_config['preset'] = args.preset
# 各个镜像的启用/禁用通过 settings.json 中的 mirrors.xxx.enabled 配置
# 下载限速
@@ -334,9 +358,9 @@ def main():
import secrets
# 确定 token 文件路径
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
# 打包环境：使用 _MEIPASS 目录
token_file = os.path.join(sys._MEIPASS, 'auth_token.txt')
if getattr(sys, 'frozen', False):
# 打包环境：使用可执行文件所在目录（_MEIPASS 是临时解压目录，写入会丢失）
token_file = os.path.join(os.path.dirname(os.path.abspath(sys.executable)), 'auth_token.txt')
else:
# 非打包：使用脚本所在目录
token_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'auth_token.txt')
mirrors/__init__.py
+1 -1
View File
@@ -113,7 +113,7 @@ MIRROR_HANDLERS = {
# ======== 容器/云原生 ========
'helm': HttpMirror,
'kubernetes': HttpMirror,
' Quay.io': HttpMirror,
'quay.io': HttpMirror,
'quay': HttpMirror,
'ghcr': HttpMirror,
'gcr': HttpMirror,
mirrors/pypi.py
+8 -21
View File
@@ -35,6 +35,14 @@ class PyPIMirror:
os.makedirs(self.simple_dir, exist_ok=True)
os.makedirs(self.web_dir, exist_ok=True)
def _debug_log(self, msg: str):
"""受 debug 配置控制的调试日志（原硬编码 /tmp/pypi_debug.log）"""
try:
if self.config and self.config.get('debug'):
print(f"[PyPI-Mirror] {msg}")
except Exception:
pass
def handle_request(self, handler, path: str) -> bool:
"""
处理PyPI请求
@@ -155,29 +163,17 @@ class PyPIMirror:
package = package.lower()
# 调试 - 确保函数被调用
debug_file = '/tmp/pypi_debug.log'
with open(debug_file, 'a') as f:
f.write(f"[HANDLE_SIMPLE] START package={package}\n")
# 检查客户端Accept header
accept = handler.headers.get('Accept', '')
wants_json = 'application/vnd.pypi.simple.v1+json' in accept
# 调试
debug_file = '/tmp/pypi_debug.log'
with open(debug_file, 'a') as f:
f.write(f"[SIMPLE_INDEX] package={package}, wants_json={wants_json}, accept={accept[:50]}\n")
# 根据请求格式选择正确的缓存key，统一使用 simple/ 前缀
cache_key = f"simple/{package}"
cached = self._get_cache(cache_key)
# 调试缓存
debug_file = '/tmp/pypi_debug.log'
with open(debug_file, 'a') as f:
f.write(f"[CACHE_CHECK] cache_key={cache_key}, cached={'YES' if cached else 'NO'}\n")
if cached:
# 返回缓存，使用正确的Content-Type
if wants_json:
@@ -213,9 +209,6 @@ class PyPIMirror:
cache_key = f"simple/{package}"
else:
# 转换为HTML格式
debug_file = '/tmp/pypi_debug.log'
with open(debug_file, 'a') as f:
f.write(f"[CONVERT_CALL] Before conversion\n")
try:
proxy_data = self._convert_simple_html(package, data)
with open(debug_file, 'a') as f:
@@ -564,9 +557,6 @@ class PyPIMirror:
import sys
import os
# 写入调试文件
debug_file = '/tmp/pypi_debug.log'
with open(debug_file, 'a') as f:
f.write(f"[PyPI] Converting HTML for package: {package}\n")
# 打印前几个链接用于调试
import re
test_matches = re.findall(r'href="([^"]+)"', html)[:3]
@@ -621,9 +611,6 @@ class PyPIMirror:
def convert_relative_match(match):
"""转换相对路径链接"""
href = match.group(1)
debug_file = '/tmp/pypi_debug.log'
with open(debug_file, 'a') as f:
f.write(f"[CONVERT] Input href: {href[:80]}...\n")
# 提取文件名
filename = href.split('/')[-1].split('#')[0]
# 提取完整的hash路径（从 packages/ 后面的所有部分除了文件名）
requirements.txt
-2
View File
@@ -3,13 +3,11 @@ pyinstaller>=6.0.0
python-multipart>=0.0.6
paramiko>=2.11.0
bcrypt>=4.0.0
cachetools>=5.3.0
requests>=2.31.0
croniter>=2.0.0
# Database (SQLite 内置, 以下为可选)
sqlalchemy>=2.0.0
apscheduler>=3.10.0
# PostgreSQL 驱动
psycopg2-binary>=2.9.0
settings.json
+1 -1
View File
@@ -569,7 +569,7 @@
"shutdown_timeout": 10,
"strategy": "graceful"
},
"version": "2.2",
"version": "2.3",
"sort_by": "name",
"sort_reverse": false,
"calculate_hash": false,