From dfae7496125e417a1f228b6931ec0b8e94b5afda Mon Sep 17 00:00:00 2001 From: HYC Fixer Date: Sun, 30 Aug 2026 12:25:26 +0800 Subject: [PATCH] =?UTF-8?q?P2:=20=E6=B8=85=E7=90=86=E4=B8=8E=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CLI: --preset 实际生效;--directory-listing/--enable-stats/--enable-*/--ignore-hidden 支持 true/false(修复 type=bool 陷阱),未显式传入时不再覆盖 settings.json - 版本号统一 v2.3(settings.json/config.py/v1/v2/Dockerfile) - 移除硬编码 /tmp/pypi_debug.log 写文件;v2 裸 except 改记录错误返回 500 - webhook id 非法输入返回 400;下载趋势 timedelta 导入提升修复 NameError - 打包模式 auth_token 写入 exe 目录(不再写入 _MEIPASS) - README 修正: 线程池架构、pyinstaller 构建方式 - mirrors/__init__.py ' Quay.io' 笔误;chunked 响应去掉 Content-Length;docs/ui realpath 边界 - SQLite WAL+busy_timeout;下载计数原子 UPDATE - requirements 移除未使用的 cachetools/apscheduler --- README.md | 1443 +++++++++--------- api/v1.py | 4 +- api/v2.py | 59 +- core/config.py | 702 ++++----- core/database.py | 3123 +++++++++++++++++++------------------- docker/Dockerfile | 146 +- handlers/http_handler.py | 6 +- main.py | 1272 ++++++++-------- mirrors/__init__.py | 1166 +++++++------- mirrors/pypi.py | 29 +- requirements.txt | 36 +- settings.json | 2 +- 12 files changed, 4020 insertions(+), 3968 deletions(-) diff --git a/README.md b/README.md index 80bf44a..ec2d768 100644 --- a/README.md +++ b/README.md @@ -1,723 +1,720 @@ -# HYC下载站 v2.3 - -模块化重构的Python镜像文件服务器，支持API版本化、实时通信、系统监控、数据库集成等高级功能。 - -## 项目结构 - -``` -mirror_server/ -├── main.py # 主入口文件 -├── requirements.txt # 依赖列表 -├── settings.json # 主配置文件 -├── auth_token.txt # 认证令牌文件 -├── api/ # API模块 -│ ├── __init__.py -│ ├── router.py # API路由器 -│ ├── v1.py # API v1（基础功能） -│ ├── v2.py # API v2（增强功能） -│ ├── admin.py # 管理接口 -│ ├── ws_handler.py # WebSocket处理 -│ ├── sse_handler.py # SSE事件流 -│ └── docs/ # API文档 -├── core/ # 核心模块 -│ ├── __init__.py -│ ├── config.py # 配置管理 -│ ├── config_hotreload.py # 配置热重载 -│ ├── utils.py # 工具函数 -│ ├── server.py # 服务器核心 -│ ├── database.py # 数据库管理 -│ ├── mirror_sync.py # 镜像同步管理 -│ ├── sync_scheduler.py # 同步调度器 -│ ├── sync_engine.py # 同步引擎 -│ ├── cache_manager.py # 缓存管理 -│ ├── cache_prewarm.py # 缓存预热 -│ ├── monitor.py # 系统监控 -│ ├── health_check.py # 健康检查 -│ ├── alerts.py # 告警系统 -│ ├── security.py # 安全模块 -│ ├── api_auth.py # API认证 -│ ├── api_docs.py # API文档 -│ ├── optimization.py # 性能优化 -│ ├── scheduler.py # 任务调度 -│ ├── graceful_restart.py # 优雅重启 -│ └── prometheus.py # Prometheus监控 -├── handlers/ # HTTP请求处理器 -│ ├── __init__.py -│ └── http_handler.py # HTTP请求处理 -├── mirrors/ # 镜像加速源模块 -│ ├── __init__.py -│ ├── http.py # HTTP/HTTPS镜像 -│ ├── docker.py # Docker Hub镜像 -│ ├── apt.py # APT镜像 -│ ├── yum.py # YUM镜像 -│ ├── pypi.py # PyPI镜像 -│ ├── npm.py # npm镜像 -│ └── go.py # Go镜像 -├── scripts/ # 辅助脚本 -├── data/ # 数据目录 -├── docker/ # Docker配置 -└── k8s/ # Kubernetes配置 -``` - -## 安装 - -```bash -pip install -r requirements.txt -``` - -### 依赖 - -- psutil - 系统监控 -- python-multipart - 多部分表单数据处理 -- paramiko - SFTP支持（可选） - -## 使用 - -### 基本使用 - -```bash -# 启动服务器（默认端口8080） -python main.py - -# 指定端口和目录 -python main.py -p 8080 -d ./downloads - -# 启用HTTPS -python main.py --ssl-cert cert.pem --ssl-key key.pem - -# 使用配置文件 -python main.py --config config.json - -# 指定API版本 (默认: v2) -python main.py --api-version v2 - -# 使用settings.json配置文件 -python main.py --settings settings.json - -# 检查系统兼容性 -python main.py --check-compat -``` - -### 认证 - -```bash -# 无认证 -python main.py --auth-type none - -# 基本认证 -python main.py --auth-type basic --auth-user admin --auth-pass password - -# 令牌认证 -python main.py --auth-type token --auth-token your_token_here -``` - -### 功能开关 - -```bash -# 禁用监控/同步/加速源 -python main.py --enable-monitor --enable-sync --enable-mirrors - -# 禁用WebSocket/SSE (适合低端设备) -python main.py --disable-ws --disable-sse - -# 禁用文件哈希计算 -python main.py --disable-hash - -# 启用下载统计 -python main.py --enable-stats --show-hash - -# 启用目录浏览 -python main.py --directory-listing true -``` - -### 镜像加速源 - -```bash -# 启用/禁用所有加速源功能 -python main.py --enable-mirrors # 启用 -python main.py --enable-mirrors=false # 禁用 - -# 各个镜像的启用/禁用通过 settings.json 配置 -# 在 mirrors.xxx.enabled 中设置，例如: -# "docker": { "enabled": true, ... } -# "pypi": { "enabled": true, ... } -``` - -### 镜像同步配置 - -镜像同步支持多种同步类型，通过 `settings.json` 中的 `sync_sources` 配置： - -```json -{ - "mirrors": { - "my-http-mirror": { - "type": "http", - "url": "https://example.com/mirror/", - "target": "downloads/mirror", - "enabled": true, - "auto_sync": true, - "schedule": { "enabled": true, "type": "cron", "cron": "0 4 * * *" } - }, - "my-ftp-mirror": { - "type": "ftp", - "host": "ftp.example.com", - "port": 21, - "username": "anonymous", - "password": "anonymous@example.com", - "remote_path": "/pub", - "target": "downloads/ftp", - "enabled": true - }, - "my-sftp-mirror": { - "type": "sftp", - "host": "sftp.example.com", - "port": 22, - "username": "syncuser", - "password": "password", - "private_key": "/path/to/id_rsa", - "remote_path": "/mirror", - "target": "downloads/sftp" - }, - "my-git-repo": { - "type": "git", - "url": "https://github.com/example/repo.git", - "branch": "main", - "depth": 1, - "target": "downloads/git" - }, - "my-s3-mirror": { - "type": "s3", - "endpoint": "https://s3.amazonaws.com", - "bucket": "my-bucket", - "access_key": "AKIAIOSFODNN7EXAMPLE", - "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "region": "us-east-1", - "prefix": "mirrors/", - "target": "downloads/s3" - }, - "my-oss-mirror": { - "type": "oss", - "bucket": "my-bucket", - "access_key": "your-access-key", - "secret_key": "your-secret-key", - "region": "cn-hangzhou", - "target": "downloads/oss" - }, - "my-cos-mirror": { - "type": "cos", - "bucket": "my-bucket", - "access_key": "your-access-key", - "secret_key": "your-secret-key", - "region": "ap-guangzhou", - "target": "downloads/cos" - }, - "my-rsync-mirror": { - "type": "rsync", - "source": "rsync://rsync.example.com/module/path", - "target": "downloads/rsync", - "exclude": ["*.tmp", "*.log"] - }, - "my-local-mirror": { - "type": "local", - "path": "/mnt/external drive/mirrors", - "target": "downloads/local" - } - } -} -``` - -**支持的同步类型：** - -| 类型 | 说明 | 必需配置 | -|------|------|----------| -| `http` / `https` | HTTP/HTTPS同步 | `url`, `target` | -| `ftp` | FTP同步 | `host`, `target` | -| `sftp` | SFTP同步 | `host`, `username`, `target` | -| `rsync` | Rsync同步 | `source`, `target` | -| `git` | Git仓库克隆/更新 | `url`, `target` | -| `s3` | AWS S3 / MinIO | `bucket`, `access_key`, `secret_key`, `endpoint` | -| `oss` | 阿里云OSS | `bucket`, `access_key`, `secret_key` | -| `cos` | 腾讯云COS | `bucket`, `access_key`, `secret_key` | -| `webdav` | WebDAV同步 | `url`, `username`, `password` | -| `local` | 本地目录同步 | `path`, `target` | - -### 下载限速 - -```bash -# 全局限速 (字节/秒) -python main.py --rate-limit 10485760 # 10MB/s -``` - -### 低端设备优化 - -```bash -# 使用预设配置 -python main.py --preset ultra_low # 超低功耗 -python main.py --preset low # 低配置 -python main.py --preset medium # 中等 -python main.py --preset high # 高性能 -python main.py --preset auto # 自动检测 - -# 自定义参数 -python main.py --memory-limit 256M \ - --workers 2 \ - --chunk-size 64K \ - --buffer-size 128K -``` - -### 调试选项 - -```bash -# 启用所有调试 -python main.py --debug - -# 启用特定调试 -python main.py --debug-http -python main.py --debug-auth -python main.py --debug-api -python main.py --debug-v2 -python main.py --debug-error -python main.py --debug-download - -# 输出到调试文件 -python main.py --debug-log /var/log/hyc-debug.log - -# 指定调试类型 -python main.py --debug-types http auth api v2 error - -# 详细输出 -python main.py -v -python main.py -vv -python main.py -q # 静默模式 -``` - -### 启动参数完整列表 - -``` -服务器配置: - --host HOST 监听地址 (默认: 0.0.0.0) - -p, --port PORT 监听端口 (默认: 8080) - -d, --base-dir DIR 文件存储目录 (默认: ./downloads) - --server-name NAME 服务器名称 (默认: HYC下载站) - -HTTPS配置: - --ssl-cert FILE SSL证书文件路径 - --ssl-key FILE SSL私钥文件路径 - -认证配置: - --auth-type TYPE 认证类型: none/basic/token (默认: none) - --auth-user USER 基本认证用户名 (默认: admin) - --auth-pass PASS 基本认证密码 - --auth-token TOKEN 令牌认证密钥 - -功能配置: - --directory-listing BOOL 启用目录浏览 (默认: True) - --enable-stats BOOL 启用下载统计 (默认: True) - --show-hash 显示文件哈希值 - --ignore-hidden 忽略隐藏文件 (默认: True) - --max-upload-size SIZE 最大上传文件大小 (默认: 1G) - --api-version VERSION API版本: v1/v2 (默认: v2) - -实时通信: - --enable-ws 启用WebSocket (默认: True) - --enable-sse 启用SSE (默认: True) - -系统监控: - --enable-monitor 启用系统监控 (默认: True) - --monitor-interval SEC 监控采集间隔(秒) (默认: 5) - -同步配置: - --enable-sync 启用镜像同步 (默认: True) - --sync-config FILE 同步配置文件路径 - -镜像加速源: - --enable-mirrors 启用下载加速源 (默认: True) - # 各个镜像通过 settings.json 配置 (mirrors.xxx.enabled) - -下载限速: - --rate-limit BYTES 全局下载限速 (默认: 0=不限速) - -低端设备优化: - --preset PRESET 设备预设: ultra_low/low/medium/high/auto (默认: auto) - --memory-limit SIZE 内存限制 (例如: 256M, 512M, 1G) - --workers NUM 工作进程数 (0=自动) - --chunk-size SIZE 文件传输块大小 (默认: 128K) - --buffer-size SIZE 缓冲区大小 (默认: 256K) - --disable-ws 禁用WebSocket - --disable-sse 禁用SSE - --disable-hash 禁用文件哈希计算 - --check-compat 检查系统兼容性后退出 - -日志配置: - --access-log FILE 访问日志文件路径 - -v, --verbose 详细输出 (可叠加: -vvv) - -q, --quiet 静默模式 - -调试选项: - -D, --debug 启用所有调试输出 - --debug-log FILE 调试日志文件路径 - --debug-http 调试HTTP请求 - --debug-auth 调试认证检查 - --debug-api 调试API路由 - --debug-v2 调试V2 API - --debug-error 调试错误堆栈 - --debug-download 调试下载记录 - --debug-types TYPES 指定调试类型列表 - -配置文件: - --settings FILE 默认配置文件路径 (默认: settings.json) - --config FILE 覆盖配置文件路径 (JSON格式) -``` - -## API版本 - -### API v1 - 基础功能 - -- 文件管理：上传、下载、删除、搜索 -- 目录浏览 -- 基础统计 -- 同步管理（HTTP/HTTPS、FTP、SFTP、本地） - -#### 示例API端点 - -``` -GET /api/v1/files # 列出文件 -GET /api/v1/file/{path} # 获取文件信息 -DELETE /api/v1/file/{path} # 删除文件 -POST /api/v1/upload # 上传文件 -GET /api/v1/search?q={term} # 搜索文件 -GET /api/v1/stats # 获取统计 -GET /api/v1/health # 健康检查 -POST /api/v1/sync/start # 开始同步 -``` - -### API v2 - 增强功能 - -继承v1所有功能，额外提供： - -- 增强搜索（模糊匹配、正则表达式、内容搜索） -- 按标签和日期范围搜索 -- 文件元数据管理 -- 文件版本控制 -- 缩略图生成 -- 实时服务器监控 -- Webhook支持 -- 历史数据分析 - -#### 示例API端点 - -``` -GET /api/v2/search/enhanced?q={term} # 增强搜索 -GET /api/v2/search/by-tag?tag={tag} # 按标签搜索 -GET /api/v2/search/by-date?start={date} # 按日期搜索 -GET /api/v2/stats/detailed # 详细统计 -GET /api/v2/stats/trending # 热门文件 -GET /api/v2/file/{path}/metadata # 获取文件元数据 -PUT /api/v2/file/{path}/metadata # 更新文件元数据 -GET /api/v2/file/{path}/versions # 获取文件版本 -POST /api/v2/file/{path}/versions # 创建文件版本 -GET /api/v2/file/{path}/thumbnail?w=200&h=200 # 获取缩略图 -GET /api/v2/monitor/realtime # 实时监控 -GET /api/v2/webhooks # 列出webhooks -POST /api/v2/webhooks # 创建webhook -``` - -## 配置 - -### 配置文件示例 - -参考 `config.example.json` - -### 命令行参数 - -``` -服务器配置: - --host 监听地址 (默认: 0.0.0.0) - -p, --port 监听端口 (默认: 8080) - -d, --base-dir 文件存储目录 (默认: ./downloads) - --server-name 服务器名称 - -HTTPS配置: - --ssl-cert SSL证书文件路径 - --ssl-key SSL私钥文件路径 - -认证配置: - --auth-type 认证类型: none, basic, token - --auth-user 基本认证用户名 - --auth-pass 基本认证密码 - --auth-token 令牌认证密钥 - -功能配置: - --no-directory-listing 禁用目录浏览 - --no-stats 禁用下载统计 - --show-hash 显示文件哈希值 - --max-upload-size 最大上传文件大小 (默认: 1G) - --sync-config 同步配置文件路径 - --api-version API版本: v1, v2 (默认: v1) - -日志配置: - --access-log 访问日志文件路径 - -v, --verbose 详细输出 - -q, --quiet 静默模式 - -配置: - --config 配置文件路径 (JSON格式) -``` - -## 模块说明 - -### core/ - 核心模块 - -- **config.py**: 配置管理和验证，支持多级配置合并 -- **config_hotreload.py**: 配置文件热重载，无需重启生效 -- **utils.py**: 通用工具函数（文件大小格式化、哈希计算等） -- **server.py**: 服务器核心类，基于aiohttp -- **database.py**: 数据库管理，支持SQLite/MySQL/PostgreSQL -- **mirror_sync.py**: 镜像同步管理器（支持HTTP/HTTPS、FTP、SFTP、本地、Rsync、Git、S3/OSS/COS、WebDAV） -- **sync_scheduler.py**: 同步调度器，定时执行同步任务 -- **sync_engine.py**: 同步引擎，核心同步逻辑 -- **cache_manager.py**: 缓存管理，智能缓存策略 -- **cache_prewarm.py**: 缓存预热，启动时预加载热门资源 -- **monitor.py**: 系统监控，CPU/内存/磁盘/网络实时监控 -- **health_check.py**: 健康检查，系统组件状态检测 -- **alerts.py**: 告警系统，异常情况自动告警 -- **security.py**: 安全模块，防护和审计 -- **api_auth.py**: API认证，Token/Basic认证 -- **api_docs.py**: API文档自动生成 -- **optimization.py**: 性能优化，自动检测设备配置 -- **scheduler.py**: 通用任务调度器 -- **graceful_restart.py**: 优雅重启，服务无缝更新 -- **prometheus.py**: Prometheus监控指标导出 - -### api/ - API模块 - -- **router.py**: API路由器，支持版本化 -- **v1.py**: API v1实现（基础功能） -- **v2.py**: API v2实现（增强功能，继承v1） -- **admin.py**: 管理接口，Webhooks管理 -- **ws_handler.py**: WebSocket处理，实时双向通信 -- **sse_handler.py**: SSE事件流，单向实时推送 -- **docs/**: API文档 (index.html, api.md) - -### handlers/ - HTTP请求处理器 - -- **http_handler.py**: HTTP请求处理，包括文件服务、目录浏览等 - -## 特性 - -- ✅ 模块化设计，易于扩展 -- ✅ API版本化支持（v1/v2） -- ✅ 镜像同步（HTTP/HTTPS、FTP、SFTP、Rsync、Git、S3/OSS/COS、WebDAV、本地目录） -- ✅ 文件上传下载（支持断点续传） -- ✅ 目录浏览（镜像站风格） -- ✅ 认证支持（无、基本认证、令牌认证） -- ✅ HTTPS支持 -- ✅ 下载统计 -- ✅ 文件搜索（基础/增强/正则） -- ✅ 批量操作 -- ✅ 压缩/解压缩 -- ✅ 增强搜索（v2） -- ✅ 文件元数据管理（v2） -- ✅ 服务器监控（v2） -- ✅ Webhook支持（v2） -- ✅ 实时通信（WebSocket + SSE） -- ✅ 数据库支持（SQLite/MySQL/PostgreSQL） -- ✅ 缓存系统（智能/手动/全同步） -- ✅ 配置热重载 -- ✅ 优雅重启 -- ✅ Prometheus监控 -- ✅ 告警系统 -- ✅ 设备自动优化 -- ✅ 多镜像加速源（Docker/APT/YUM/PyPI/npm/Go） -- ✅ Web管理界面 - -## 开发 - -### 添加新的API版本 - -1. 在 `api/` 目录下创建 `v3.py` -2. 继承 `APIv2` 或 `APIv1` -3. 实现所需方法 -4. 在 `api/router.py` 中注册新版本 - -### 添加新的同步类型 - -在 `core/mirror_sync.py` 的 `MirrorSyncManager` 类中： -1. 添加 `_sync_{type}` 方法 -2. 实现同步逻辑 -3. 在 `_sync_worker` 中添加路由 - -## 构建和部署 - -### 使用 PyInstaller 打包成二进制文件 - -#### 构建当前平台 - -项目提供了 `build.py` 脚本，可以将项目打包成当前平台的二进制可执行文件。 - -```bash -# 安装 PyInstaller -pip install pyinstaller - -# 查看支持的目标平台 -python build.py --list - -# 构建当前平台 -python build.py - -# 清理构建缓存 -python build.py --clean -``` - -**支持的平台（仅当前平台）：** -- Linux: amd64, arm64, 386 -- Windows: amd64, 386 -- macOS: amd64, arm64 - -构建完成后，二进制文件和发布包会输出到 `dist/` 目录。 - -#### 跨平台构建（GitHub Actions） - -使用 GitHub Actions 可以方便地构建多平台版本（Linux ARM、AMD64 等）： - -```bash -# 1. 初始化 git 仓库 -git init -git add . -git commit -m "Add build workflow" - -# 2. 添加远程仓库（替换为你的仓库地址） -git remote add origin https://github.com/your-username/your-repo.git -git branch -M main -git push -u origin main -``` - -推送完成后： - -1. 访问 GitHub 仓库 -2. 点击 **Actions** 标签 -3. 选择 **"Build Linux ARM Binaries"** -4. 点击 **"Run workflow"** -5. 等待 5-10 分钟后下载构建产物 - -**优势：** -- ✅ 完全免费 -- ✅ 无需本地配置 -- ✅ 支持多平台构建 -- ✅ 自动化持续集成 - -详细说明请参考 [BUILD.md](../BUILD.md) - -#### 快速开始 - -**Linux/macOS:** -```bash -# 给脚本添加执行权限 -chmod +x docker-deploy.sh - -# 构建并运行 -./docker-deploy.sh build -./docker-deploy.sh run - -# 使用 docker-compose -./docker-deploy.sh compose -``` - -**Windows:** -```batch -docker-deploy.bat build -docker-deploy.bat run -``` - -### HTTPS 支持 - -#### 使用 Nginx 反向代理 - -1. 创建 SSL 证书目录并放置证书 -```bash -mkdir -p ssl -# 将 cert.pem 和 key.pem 放入 ssl/ 目录 -``` - -2. 使用带 Nginx 的 docker-compose 启动 -```bash -docker-compose --profile with-nginx up -d -``` - -#### 直接启用 HTTPS - -```bash -python main.py --ssl-cert cert.pem --ssl-key key.pem -``` - -详细说明请参考 [DOCKER.md](DOCKER.md) - -## CI/CD 自动构建 - -项目使用 GitHub Actions 自动构建 Docker 镜像和可执行文件。 - -### GitHub Actions 工作流 - -| 工作流文件 | 触发条件 | 功能 | -|------------|----------|------| -| `.github/workflows/releases.yml` | 打 Tag (`v*`) | 构建并发布二进制可执行文件到 GitHub Releases | -| `.github/workflows/docker-multiarch.yml` | 打 Tag (`v*`) | 构建并推送多架构 Docker 镜像到 Docker Hub | -| `.github/workflows/ci-cd.yml` | push main / PR | 运行测试 | - -### 发布新版本 - -```bash -# 1. 推送所有代码 -git push origin main - -# 2. 创建版本标签 -git tag v1.0.0 - -# 3. 推送到远程 -git push origin v1.0.0 -``` - -推送 Tag 后，GitHub Actions 会自动： -- ✅ 构建 4 个架构的二进制文件（Linux AMD64/ARM64/ARMv7 + Windows AMD64） -- ✅ 构建 3 个架构的 Docker 镜像（AMD64/ARM64/ARMv7） -- ✅ 上传到 GitHub Releases -- ✅ 推送到 Docker Hub 和 GitHub Container Registry - -### GitHub Secrets 配置 - -需要配置以下 Secrets 才能正常工作： - -| Secret 名称 | 值 | 说明 | -|-------------|-----|------| -| `DOCKERHUB_USERNAME` | Docker Hub 用户名 | | -| `DOCKERHUB_TOKEN` | Docker Hub Access Token | | - -### Docker Hub 镜像 - -构建完成后，镜像会自动推送到： -- **Docker Hub**: `hx100cv/hyc-download:latest` -- **GitHub Container Registry**: `ghcr.io/hexikx/mirror_server:latest` - -### GitHub Releases - -每个版本会生成以下文件： - -| 文件名 | 说明 | -|--------|------| -| `hyc-download-linux-amd64` | Linux x64 可执行文件 | -| `hyc-download-linux-arm64` | Linux ARM64 可执行文件 | -| `hyc-download-linux-armv7` | Linux ARMv7 可执行文件 | -| `hyc-download-windows-amd64.exe` | Windows x64 可执行文件 | - -## Stargazers over time - -[![Stargazers over time](https://starchart.cc/HEXIkx/mirror_server.svg?variant=adaptive)](https://starchart.cc/HEXIkx/mirror_server) - -## Contributors - -[![Contributors](https://contrib.rocks/image?repo=HEXIkx/mirror_server)](https://github.com/HEXIkx/mirror_server/graphs/contributors) - -## License - - -[The MIT License (MIT)](https://github.com/HEXIkx/mirror_server/blob/main/LICENSE) +# HYC下载站 v2.3 + +模块化重构的Python镜像文件服务器，支持API版本化、实时通信、系统监控、数据库集成等高级功能。 + +## 项目结构 + +``` +mirror_server/ +├── main.py # 主入口文件 +├── requirements.txt # 依赖列表 +├── settings.json # 主配置文件 +├── auth_token.txt # 认证令牌文件 +├── api/ # API模块 +│ ├── __init__.py +│ ├── router.py # API路由器 +│ ├── v1.py # API v1（基础功能） +│ ├── v2.py # API v2（增强功能） +│ ├── admin.py # 管理接口 +│ ├── ws_handler.py # WebSocket处理 +│ ├── sse_handler.py # SSE事件流 +│ └── docs/ # API文档 +├── core/ # 核心模块 +│ ├── __init__.py +│ ├── config.py # 配置管理 +│ ├── config_hotreload.py # 配置热重载 +│ ├── utils.py # 工具函数 +│ ├── server.py # 服务器核心 +│ ├── database.py # 数据库管理 +│ ├── mirror_sync.py # 镜像同步管理 +│ ├── sync_scheduler.py # 同步调度器 +│ ├── sync_engine.py # 同步引擎 +│ ├── cache_manager.py # 缓存管理 +│ ├── cache_prewarm.py # 缓存预热 +│ ├── monitor.py # 系统监控 +│ ├── health_check.py # 健康检查 +│ ├── alerts.py # 告警系统 +│ ├── security.py # 安全模块 +│ ├── api_auth.py # API认证 +│ ├── api_docs.py # API文档 +│ ├── optimization.py # 性能优化 +│ ├── scheduler.py # 任务调度 +│ ├── graceful_restart.py # 优雅重启 +│ └── prometheus.py # Prometheus监控 +├── handlers/ # HTTP请求处理器 +│ ├── __init__.py +│ └── http_handler.py # HTTP请求处理 +├── mirrors/ # 镜像加速源模块 +│ ├── __init__.py +│ ├── http.py # HTTP/HTTPS镜像 +│ ├── docker.py # Docker Hub镜像 +│ ├── apt.py # APT镜像 +│ ├── yum.py # YUM镜像 +│ ├── pypi.py # PyPI镜像 +│ ├── npm.py # npm镜像 +│ └── go.py # Go镜像 +├── scripts/ # 辅助脚本 +├── data/ # 数据目录 +├── docker/ # Docker配置 +└── k8s/ # Kubernetes配置 +``` + +## 安装 + +```bash +pip install -r requirements.txt +``` + +### 依赖 + +- psutil - 系统监控 +- python-multipart - 多部分表单数据处理 +- paramiko - SFTP支持（可选） + +## 使用 + +### 基本使用 + +```bash +# 启动服务器（默认端口8080） +python main.py + +# 指定端口和目录 +python main.py -p 8080 -d ./downloads + +# 启用HTTPS +python main.py --ssl-cert cert.pem --ssl-key key.pem + +# 使用配置文件 +python main.py --config config.json + +# 指定API版本 (默认: v2) +python main.py --api-version v2 + +# 使用settings.json配置文件 +python main.py --settings settings.json + +# 检查系统兼容性 +python main.py --check-compat +``` + +### 认证 + +```bash +# 无认证 +python main.py --auth-type none + +# 基本认证 +python main.py --auth-type basic --auth-user admin --auth-pass password + +# 令牌认证 +python main.py --auth-type token --auth-token your_token_here +``` + +### 功能开关 + +```bash +# 禁用监控/同步/加速源 +python main.py --enable-monitor --enable-sync --enable-mirrors + +# 禁用WebSocket/SSE (适合低端设备) +python main.py --disable-ws --disable-sse + +# 禁用文件哈希计算 +python main.py --disable-hash + +# 启用下载统计 +python main.py --enable-stats --show-hash + +# 启用目录浏览 +python main.py --directory-listing true +``` + +### 镜像加速源 + +```bash +# 启用/禁用所有加速源功能 +python main.py --enable-mirrors # 启用 +python main.py --enable-mirrors=false # 禁用 + +# 各个镜像的启用/禁用通过 settings.json 配置 +# 在 mirrors.xxx.enabled 中设置，例如: +# "docker": { "enabled": true, ... } +# "pypi": { "enabled": true, ... } +``` + +### 镜像同步配置 + +镜像同步支持多种同步类型，通过 `settings.json` 中的 `sync_sources` 配置： + +```json +{ + "mirrors": { + "my-http-mirror": { + "type": "http", + "url": "https://example.com/mirror/", + "target": "downloads/mirror", + "enabled": true, + "auto_sync": true, + "schedule": { "enabled": true, "type": "cron", "cron": "0 4 * * *" } + }, + "my-ftp-mirror": { + "type": "ftp", + "host": "ftp.example.com", + "port": 21, + "username": "anonymous", + "password": "anonymous@example.com", + "remote_path": "/pub", + "target": "downloads/ftp", + "enabled": true + }, + "my-sftp-mirror": { + "type": "sftp", + "host": "sftp.example.com", + "port": 22, + "username": "syncuser", + "password": "password", + "private_key": "/path/to/id_rsa", + "remote_path": "/mirror", + "target": "downloads/sftp" + }, + "my-git-repo": { + "type": "git", + "url": "https://github.com/example/repo.git", + "branch": "main", + "depth": 1, + "target": "downloads/git" + }, + "my-s3-mirror": { + "type": "s3", + "endpoint": "https://s3.amazonaws.com", + "bucket": "my-bucket", + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + "prefix": "mirrors/", + "target": "downloads/s3" + }, + "my-oss-mirror": { + "type": "oss", + "bucket": "my-bucket", + "access_key": "your-access-key", + "secret_key": "your-secret-key", + "region": "cn-hangzhou", + "target": "downloads/oss" + }, + "my-cos-mirror": { + "type": "cos", + "bucket": "my-bucket", + "access_key": "your-access-key", + "secret_key": "your-secret-key", + "region": "ap-guangzhou", + "target": "downloads/cos" + }, + "my-rsync-mirror": { + "type": "rsync", + "source": "rsync://rsync.example.com/module/path", + "target": "downloads/rsync", + "exclude": ["*.tmp", "*.log"] + }, + "my-local-mirror": { + "type": "local", + "path": "/mnt/external drive/mirrors", + "target": "downloads/local" + } + } +} +``` + +**支持的同步类型：** + +| 类型 | 说明 | 必需配置 | +|------|------|----------| +| `http` / `https` | HTTP/HTTPS同步 | `url`, `target` | +| `ftp` | FTP同步 | `host`, `target` | +| `sftp` | SFTP同步 | `host`, `username`, `target` | +| `rsync` | Rsync同步 | `source`, `target` | +| `git` | Git仓库克隆/更新 | `url`, `target` | +| `s3` | AWS S3 / MinIO | `bucket`, `access_key`, `secret_key`, `endpoint` | +| `oss` | 阿里云OSS | `bucket`, `access_key`, `secret_key` | +| `cos` | 腾讯云COS | `bucket`, `access_key`, `secret_key` | +| `webdav` | WebDAV同步 | `url`, `username`, `password` | +| `local` | 本地目录同步 | `path`, `target` | + +### 下载限速 + +```bash +# 全局限速 (字节/秒) +python main.py --rate-limit 10485760 # 10MB/s +``` + +### 低端设备优化 + +```bash +# 使用预设配置 +python main.py --preset ultra_low # 超低功耗 +python main.py --preset low # 低配置 +python main.py --preset medium # 中等 +python main.py --preset high # 高性能 +python main.py --preset auto # 自动检测 + +# 自定义参数 +python main.py --memory-limit 256M \ + --workers 2 \ + --chunk-size 64K \ + --buffer-size 128K +``` + +### 调试选项 + +```bash +# 启用所有调试 +python main.py --debug + +# 启用特定调试 +python main.py --debug-http +python main.py --debug-auth +python main.py --debug-api +python main.py --debug-v2 +python main.py --debug-error +python main.py --debug-download + +# 输出到调试文件 +python main.py --debug-log /var/log/hyc-debug.log + +# 指定调试类型 +python main.py --debug-types http auth api v2 error + +# 详细输出 +python main.py -v +python main.py -vv +python main.py -q # 静默模式 +``` + +### 启动参数完整列表 + +``` +服务器配置: + --host HOST 监听地址 (默认: 0.0.0.0) + -p, --port PORT 监听端口 (默认: 8080) + -d, --base-dir DIR 文件存储目录 (默认: ./downloads) + --server-name NAME 服务器名称 (默认: HYC下载站) + +HTTPS配置: + --ssl-cert FILE SSL证书文件路径 + --ssl-key FILE SSL私钥文件路径 + +认证配置: + --auth-type TYPE 认证类型: none/basic/token (默认: none) + --auth-user USER 基本认证用户名 (默认: admin) + --auth-pass PASS 基本认证密码 + --auth-token TOKEN 令牌认证密钥 + +功能配置: + --directory-listing BOOL 启用目录浏览 (默认: True) + --enable-stats BOOL 启用下载统计 (默认: True) + --show-hash 显示文件哈希值 + --ignore-hidden 忽略隐藏文件 (默认: True) + --max-upload-size SIZE 最大上传文件大小 (默认: 1G) + --api-version VERSION API版本: v1/v2 (默认: v2) + +实时通信: + --enable-ws 启用WebSocket (默认: True) + --enable-sse 启用SSE (默认: True) + +系统监控: + --enable-monitor 启用系统监控 (默认: True) + --monitor-interval SEC 监控采集间隔(秒) (默认: 5) + +同步配置: + --enable-sync 启用镜像同步 (默认: True) + --sync-config FILE 同步配置文件路径 + +镜像加速源: + --enable-mirrors 启用下载加速源 (默认: True) + # 各个镜像通过 settings.json 配置 (mirrors.xxx.enabled) + +下载限速: + --rate-limit BYTES 全局下载限速 (默认: 0=不限速) + +低端设备优化: + --preset PRESET 设备预设: ultra_low/low/medium/high/auto (默认: auto) + --memory-limit SIZE 内存限制 (例如: 256M, 512M, 1G) + --workers NUM 工作进程数 (0=自动) + --chunk-size SIZE 文件传输块大小 (默认: 128K) + --buffer-size SIZE 缓冲区大小 (默认: 256K) + --disable-ws 禁用WebSocket + --disable-sse 禁用SSE + --disable-hash 禁用文件哈希计算 + --check-compat 检查系统兼容性后退出 + +日志配置: + --access-log FILE 访问日志文件路径 + -v, --verbose 详细输出 (可叠加: -vvv) + -q, --quiet 静默模式 + +调试选项: + -D, --debug 启用所有调试输出 + --debug-log FILE 调试日志文件路径 + --debug-http 调试HTTP请求 + --debug-auth 调试认证检查 + --debug-api 调试API路由 + --debug-v2 调试V2 API + --debug-error 调试错误堆栈 + --debug-download 调试下载记录 + --debug-types TYPES 指定调试类型列表 + +配置文件: + --settings FILE 默认配置文件路径 (默认: settings.json) + --config FILE 覆盖配置文件路径 (JSON格式) +``` + +## API版本 + +### API v1 - 基础功能 + +- 文件管理：上传、下载、删除、搜索 +- 目录浏览 +- 基础统计 +- 同步管理（HTTP/HTTPS、FTP、SFTP、本地） + +#### 示例API端点 + +``` +GET /api/v1/files # 列出文件 +GET /api/v1/file/{path} # 获取文件信息 +DELETE /api/v1/file/{path} # 删除文件 +POST /api/v1/upload # 上传文件 +GET /api/v1/search?q={term} # 搜索文件 +GET /api/v1/stats # 获取统计 +GET /api/v1/health # 健康检查 +POST /api/v1/sync/start # 开始同步 +``` + +### API v2 - 增强功能 + +继承v1所有功能，额外提供： + +- 增强搜索（模糊匹配、正则表达式、内容搜索） +- 按标签和日期范围搜索 +- 文件元数据管理 +- 文件版本控制 +- 缩略图生成 +- 实时服务器监控 +- Webhook支持 +- 历史数据分析 + +#### 示例API端点 + +``` +GET /api/v2/search/enhanced?q={term} # 增强搜索 +GET /api/v2/search/by-tag?tag={tag} # 按标签搜索 +GET /api/v2/search/by-date?start={date} # 按日期搜索 +GET /api/v2/stats/detailed # 详细统计 +GET /api/v2/stats/trending # 热门文件 +GET /api/v2/file/{path}/metadata # 获取文件元数据 +PUT /api/v2/file/{path}/metadata # 更新文件元数据 +GET /api/v2/file/{path}/versions # 获取文件版本 +POST /api/v2/file/{path}/versions # 创建文件版本 +GET /api/v2/file/{path}/thumbnail?w=200&h=200 # 获取缩略图 +GET /api/v2/monitor/realtime # 实时监控 +GET /api/v2/webhooks # 列出webhooks +POST /api/v2/webhooks # 创建webhook +``` + +## 配置 + +### 配置文件示例 + +参考 `config.example.json` + +### 命令行参数 + +``` +服务器配置: + --host 监听地址 (默认: 0.0.0.0) + -p, --port 监听端口 (默认: 8080) + -d, --base-dir 文件存储目录 (默认: ./downloads) + --server-name 服务器名称 + +HTTPS配置: + --ssl-cert SSL证书文件路径 + --ssl-key SSL私钥文件路径 + +认证配置: + --auth-type 认证类型: none, basic, token + --auth-user 基本认证用户名 + --auth-pass 基本认证密码 + --auth-token 令牌认证密钥 + +功能配置: + --no-directory-listing 禁用目录浏览 + --no-stats 禁用下载统计 + --show-hash 显示文件哈希值 + --max-upload-size 最大上传文件大小 (默认: 1G) + --sync-config 同步配置文件路径 + --api-version API版本: v1, v2 (默认: v1) + +日志配置: + --access-log 访问日志文件路径 + -v, --verbose 详细输出 + -q, --quiet 静默模式 + +配置: + --config 配置文件路径 (JSON格式) +``` + +## 模块说明 + +### core/ - 核心模块 + +- **config.py**: 配置管理和验证，支持多级配置合并 +- **config_hotreload.py**: 配置文件热重载，无需重启生效 +- **utils.py**: 通用工具函数（文件大小格式化、哈希计算等） +- **server.py**: 服务器核心类（基于标准库 http.server 的线程池服务器） +- **database.py**: 数据库管理，支持SQLite/MySQL/PostgreSQL +- **mirror_sync.py**: 镜像同步管理器（支持HTTP/HTTPS、FTP、SFTP、本地、Rsync、Git、S3/OSS/COS、WebDAV） +- **sync_scheduler.py**: 同步调度器，定时执行同步任务 +- **sync_engine.py**: 同步引擎，核心同步逻辑 +- **cache_manager.py**: 缓存管理，智能缓存策略 +- **cache_prewarm.py**: 缓存预热，启动时预加载热门资源 +- **monitor.py**: 系统监控，CPU/内存/磁盘/网络实时监控 +- **health_check.py**: 健康检查，系统组件状态检测 +- **alerts.py**: 告警系统，异常情况自动告警 +- **security.py**: 安全模块，防护和审计 +- **api_auth.py**: API认证，Token/Basic认证 +- **api_docs.py**: API文档自动生成 +- **optimization.py**: 性能优化，自动检测设备配置 +- **scheduler.py**: 通用任务调度器 +- **graceful_restart.py**: 优雅重启，服务无缝更新 +- **prometheus.py**: Prometheus监控指标导出 + +### api/ - API模块 + +- **router.py**: API路由器，支持版本化 +- **v1.py**: API v1实现（基础功能） +- **v2.py**: API v2实现（增强功能，继承v1） +- **admin.py**: 管理接口，Webhooks管理 +- **ws_handler.py**: WebSocket处理，实时双向通信 +- **sse_handler.py**: SSE事件流，单向实时推送 +- **docs/**: API文档 (index.html, api.md) + +### handlers/ - HTTP请求处理器 + +- **http_handler.py**: HTTP请求处理，包括文件服务、目录浏览等 + +## 特性 + +- ✅ 模块化设计，易于扩展 +- ✅ API版本化支持（v1/v2） +- ✅ 镜像同步（HTTP/HTTPS、FTP、SFTP、Rsync、Git、S3/OSS/COS、WebDAV、本地目录） +- ✅ 文件上传下载（支持断点续传） +- ✅ 目录浏览（镜像站风格） +- ✅ 认证支持（无、基本认证、令牌认证） +- ✅ HTTPS支持 +- ✅ 下载统计 +- ✅ 文件搜索（基础/增强/正则） +- ✅ 批量操作 +- ✅ 压缩/解压缩 +- ✅ 增强搜索（v2） +- ✅ 文件元数据管理（v2） +- ✅ 服务器监控（v2） +- ✅ Webhook支持（v2） +- ✅ 实时通信（WebSocket + SSE） +- ✅ 数据库支持（SQLite/MySQL/PostgreSQL） +- ✅ 缓存系统（智能/手动/全同步） +- ✅ 配置热重载 +- ✅ 优雅重启 +- ✅ Prometheus监控 +- ✅ 告警系统 +- ✅ 设备自动优化 +- ✅ 多镜像加速源（Docker/APT/YUM/PyPI/npm/Go） +- ✅ Web管理界面 + +## 开发 + +### 添加新的API版本 + +1. 在 `api/` 目录下创建 `v3.py` +2. 继承 `APIv2` 或 `APIv1` +3. 实现所需方法 +4. 在 `api/router.py` 中注册新版本 + +### 添加新的同步类型 + +在 `core/mirror_sync.py` 的 `MirrorSyncManager` 类中： +1. 添加 `_sync_{type}` 方法 +2. 实现同步逻辑 +3. 在 `_sync_worker` 中添加路由 + +## 构建和部署 + +### 使用 PyInstaller 打包成二进制文件 + +#### 构建当前平台 + +项目提供 PyInstaller 打包配置，可以将项目打包成当前平台的二进制可执行文件。 + +```bash +# 安装 PyInstaller +pip install pyinstaller + +# 构建当前平台（使用仓库内的打包配置） +pyinstaller hyc-download.spec + +# 使用旧的打包配置（含 aiohttp 隐藏依赖，已弃用） +# pyinstaller pyinstaller.spec +``` + +**支持的平台（仅当前平台）：** +- Linux: amd64, arm64, 386 +- Windows: amd64, 386 +- macOS: amd64, arm64 + +构建完成后，二进制文件和发布包会输出到 `dist/` 目录。 + +#### 跨平台构建（GitHub Actions） + +使用 GitHub Actions 可以方便地构建多平台版本（Linux ARM、AMD64 等）： + +```bash +# 1. 初始化 git 仓库 +git init +git add . +git commit -m "Add build workflow" + +# 2. 添加远程仓库（替换为你的仓库地址） +git remote add origin https://github.com/your-username/your-repo.git +git branch -M main +git push -u origin main +``` + +推送完成后： + +1. 访问 GitHub 仓库 +2. 点击 **Actions** 标签 +3. 选择 **"Build Linux ARM Binaries"** +4. 点击 **"Run workflow"** +5. 等待 5-10 分钟后下载构建产物 + +**优势：** +- ✅ 完全免费 +- ✅ 无需本地配置 +- ✅ 支持多平台构建 +- ✅ 自动化持续集成 + +详细说明请参考 [BUILD.md](../BUILD.md) + +#### 快速开始 + +**Linux/macOS:** +```bash +# 给脚本添加执行权限 +chmod +x docker-deploy.sh + +# 构建并运行 +./docker-deploy.sh build +./docker-deploy.sh run + +# 使用 docker-compose +./docker-deploy.sh compose +``` + +**Windows:** +```batch +docker-deploy.bat build +docker-deploy.bat run +``` + +### HTTPS 支持 + +#### 使用 Nginx 反向代理 + +1. 创建 SSL 证书目录并放置证书 +```bash +mkdir -p ssl +# 将 cert.pem 和 key.pem 放入 ssl/ 目录 +``` + +2. 使用带 Nginx 的 docker-compose 启动 +```bash +docker-compose --profile with-nginx up -d +``` + +#### 直接启用 HTTPS + +```bash +python main.py --ssl-cert cert.pem --ssl-key key.pem +``` + +详细说明请参考 [DOCKER.md](DOCKER.md) + +## CI/CD 自动构建 + +项目使用 GitHub Actions 自动构建 Docker 镜像和可执行文件。 + +### GitHub Actions 工作流 + +| 工作流文件 | 触发条件 | 功能 | +|------------|----------|------| +| `.github/workflows/releases.yml` | 打 Tag (`v*`) | 构建并发布二进制可执行文件到 GitHub Releases | +| `.github/workflows/docker-multiarch.yml` | 打 Tag (`v*`) | 构建并推送多架构 Docker 镜像到 Docker Hub | +| `.github/workflows/ci-cd.yml` | push main / PR | 运行测试 | + +### 发布新版本 + +```bash +# 1. 推送所有代码 +git push origin main + +# 2. 创建版本标签 +git tag v1.0.0 + +# 3. 推送到远程 +git push origin v1.0.0 +``` + +推送 Tag 后，GitHub Actions 会自动： +- ✅ 构建 4 个架构的二进制文件（Linux AMD64/ARM64/ARMv7 + Windows AMD64） +- ✅ 构建 3 个架构的 Docker 镜像（AMD64/ARM64/ARMv7） +- ✅ 上传到 GitHub Releases +- ✅ 推送到 Docker Hub 和 GitHub Container Registry + +### GitHub Secrets 配置 + +需要配置以下 Secrets 才能正常工作： + +| Secret 名称 | 值 | 说明 | +|-------------|-----|------| +| `DOCKERHUB_USERNAME` | Docker Hub 用户名 | | +| `DOCKERHUB_TOKEN` | Docker Hub Access Token | | + +### Docker Hub 镜像 + +构建完成后，镜像会自动推送到： +- **Docker Hub**: `hx100cv/hyc-download:latest` +- **GitHub Container Registry**: `ghcr.io/hexikx/mirror_server:latest` + +### GitHub Releases + +每个版本会生成以下文件： + +| 文件名 | 说明 | +|--------|------| +| `hyc-download-linux-amd64` | Linux x64 可执行文件 | +| `hyc-download-linux-arm64` | Linux ARM64 可执行文件 | +| `hyc-download-linux-armv7` | Linux ARMv7 可执行文件 | +| `hyc-download-windows-amd64.exe` | Windows x64 可执行文件 | + +## Stargazers over time + +[![Stargazers over time](https://starchart.cc/HEXIkx/mirror_server.svg?variant=adaptive)](https://starchart.cc/HEXIkx/mirror_server) + +## Contributors + +[![Contributors](https://contrib.rocks/image?repo=HEXIkx/mirror_server)](https://github.com/HEXIkx/mirror_server/graphs/contributors) + +## License + + +[The MIT License (MIT)](https://github.com/HEXIkx/mirror_server/blob/main/LICENSE) diff --git a/api/v1.py b/api/v1.py index f0babe7..7baf976 100644 --- a/api/v1.py +++ b/api/v1.py @@ -1474,7 +1474,7 @@ class APIv1: """API: 获取配置信息""" handler.send_json_response({ "server_name": self.config.get("server_name", "Mirror Server"), - "version": "2.2", + "version": "2.3", "base_dir": self.config['base_dir'], "directory_listing": self.config.get('directory_listing', True), "max_upload_size": self.config.get('max_upload_size'), @@ -2252,7 +2252,7 @@ class APIv1: """API: 获取镜像站信息""" info = { "server_name": self.config.get("server_name", "Mirror Server"), - "version": "2.1", + "version": "2.3", "uptime": time.time() - self.config.get('start_time', time.time()), "total_files": sum(1 for _, _, files in os.walk(self.config['base_dir']) for _ in files), "total_size": self._get_dir_size(self.config['base_dir']), diff --git a/api/v2.py b/api/v2.py index 97247d9..6c6b9cb 100644 --- a/api/v2.py +++ b/api/v2.py @@ -597,11 +597,6 @@ class APIv2(APIv1): # 获取镜像类型 mirror_type = mirror_config.get('type', 'http') - # 调试 - debug_file = '/tmp/pypi_debug.log' - with open(debug_file, 'a') as f: - f.write(f"[V2] mirror_type={mirror_type}, handler_class={get_mirror_handler(mirror_type)}\n") - # 获取处理器类 handler_class = get_mirror_handler(mirror_type) if not handler_class: @@ -887,8 +882,17 @@ class APIv2(APIv1): # v2 端点都没匹配到，尝试调用 APIv1 try: return super().handle_request(handler, method, path, query_params) - except: - pass + except Exception as e: + # 记录真实错误而非吞掉（避免 v1 内部异常被误报为 404） + if handler._is_debug_enabled('error'): + import traceback + handler._debug_log('error', f"APIv1 处理失败 ({path}): {e}\n{traceback.format_exc()}") + handler.send_json_response({ + "error": f"API处理错误: {str(e)}", + "version": "v1", + "path": path + }, 500) + return # 404 - 未找到端点 handler.send_error(404) @@ -1283,6 +1287,8 @@ class APIv2(APIv1): def api_get_download_trend(self, handler, query_params): """获取下载趋势（按天统计）""" days = int(query_params.get('days', [7])[0]) + from datetime import timedelta # 供数据库与回退分支共同使用 + days = min(max(days, 1), 90) # 限制 1-90 天 trend = [] @@ -1291,7 +1297,6 @@ class APIv2(APIv1): db = self.get_db() if db: try: - from datetime import timedelta from core.database import DownloadRecord now = datetime.now() @@ -1983,7 +1988,11 @@ class APIv2(APIv1): def api_get_webhook(self, handler, webhook_id): """获取webhook详情""" - webhook_id = int(webhook_id) + try: + webhook_id = int(webhook_id) + except (ValueError, TypeError): + handler.send_json_response({"error": "Invalid webhook id"}, 400) + return if self.db_enabled and self.db: try: @@ -2006,7 +2015,11 @@ class APIv2(APIv1): def api_delete_webhook(self, handler, webhook_id): """删除webhook""" - webhook_id = int(webhook_id) + try: + webhook_id = int(webhook_id) + except (ValueError, TypeError): + handler.send_json_response({"error": "Invalid webhook id"}, 400) + return if self.db_enabled and self.db: try: @@ -2031,7 +2044,11 @@ class APIv2(APIv1): def api_test_webhook(self, handler, webhook_id): """测试webhook""" - webhook_id = int(webhook_id) + try: + webhook_id = int(webhook_id) + except (ValueError, TypeError): + handler.send_json_response({"error": "Invalid webhook id"}, 400) + return if self.db_enabled and self.db: try: @@ -2095,7 +2112,11 @@ class APIv2(APIv1): def api_get_webhook_deliveries(self, handler, webhook_id): """获取 webhook 交付历史""" - webhook_id = int(webhook_id) + try: + webhook_id = int(webhook_id) + except (ValueError, TypeError): + handler.send_json_response({"error": "Invalid webhook id"}, 400) + return # 验证 webhook 存在 if self.db_enabled and self.db: @@ -2126,7 +2147,11 @@ class APIv2(APIv1): def api_get_webhook_stats(self, handler, webhook_id): """获取 webhook 交付统计""" - webhook_id = int(webhook_id) + try: + webhook_id = int(webhook_id) + except (ValueError, TypeError): + handler.send_json_response({"error": "Invalid webhook id"}, 400) + return if self.db_enabled and self.db: try: @@ -2146,7 +2171,11 @@ class APIv2(APIv1): def api_update_webhook(self, handler, webhook_id): """更新 webhook 配置""" - webhook_id = int(webhook_id) + try: + webhook_id = int(webhook_id) + except (ValueError, TypeError): + handler.send_json_response({"error": "Invalid webhook id"}, 400) + return content_length = int(handler.headers.get('Content-Length', 0)) if content_length == 0: @@ -3741,7 +3770,7 @@ class APIv2(APIv1): handler.send_json_response({ "name": self.config.get('server_name', 'HYC下载站'), - "version": "2.2.0", + "version": "2.3.0", "uptime_seconds": round(uptime_seconds, 2), "uptime_formatted": uptime_str, "api_version": "v2", diff --git a/core/config.py b/core/config.py index 0da1d13..d27ca92 100644 --- a/core/config.py +++ b/core/config.py @@ -1,351 +1,351 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -"""配置管理模块""" - -import os -import sys -import json -import hashlib -import time -from typing import Dict, Any, Optional - -from .utils import parse_size - - -def get_resource_path(relative_path: str) -> str: - """获取打包后的资源路径""" - if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): - # 打包后的路径 - return os.path.join(sys._MEIPASS, relative_path) - return os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), relative_path) - - # 打包模式 - if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): - # 外部目录：与 exe 同级 - external_path = os.path.join(os.path.dirname(sys.executable), relative_path) - if os.path.exists(external_path): - return external_path - - # 打包后的资源路径（_MEIPASS） - bundled_path = os.path.join(sys._MEIPASS, relative_path) - if os.path.exists(bundled_path): - return bundled_path - - return external_path - - # 开发模式 - return os.path.join(project_root, relative_path) - - -def deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]: - """ - 深度合并配置 - - 如果 override 中有的键，会完全替换 base 中的值（除非都是 dict） - - 如果都是 dict，则递归合并 - - 不会修改原参数 - - Args: - base: 默认配置（基础配置） - override: 要合并的配置（优先级更高） - - Returns: - 合并后的配置 - """ - result = base.copy() - - for key, value in override.items(): - if key in result and isinstance(result[key], dict) and isinstance(value, dict): - # 两者都是字典，递归合并 - result[key] = deep_merge(result[key], value) - else: - # 直接覆盖 - result[key] = value - - return result - - -def load_json_config(file_path: str) -> Optional[Dict[str, Any]]: - """加载 JSON 配置文件""" - if not os.path.exists(file_path): - return None - - try: - with open(file_path, 'r', encoding='utf-8') as f: - return json.load(f) - except json.JSONDecodeError as e: - print(f"警告: 配置文件 {file_path} JSON 格式错误: {e}") - return None - except Exception as e: - print(f"警告: 无法读取配置文件 {file_path}: {e}") - return None - - -class ConfigManager: - """配置管理器""" - - def __init__(self, config: Dict[str, Any] = None, settings_path: str = None): - """ - 初始化配置管理器 - - Args: - config: 传入的配置（会覆盖默认配置） - settings_path: 默认配置文件路径（可以是位置参数或关键字参数） - """ - # 支持 settings_path 作为位置参数 - if isinstance(config, str): - settings_path = config - config = None - elif settings_path is None: - # 使用打包后的资源路径 - settings_path = get_resource_path('settings.json') - elif settings_path: - settings_path = settings_path - - self._settings_path = settings_path - - # 加载默认配置 - self.default_config = self._load_default_config() - - # 合并传入的配置 - if config: - self.config = self._validate_config(deep_merge(self.default_config, config)) - else: - self.config = self._validate_config(self.default_config.copy()) - - def _load_default_config(self) -> Dict[str, Any]: - """加载默认配置文件""" - default_config = load_json_config(self._settings_path) - if default_config is None: - # 如果找不到默认配置，使用内联最小配置 - default_config = { - 'server_name': 'HYC下载站', - 'host': '0.0.0.0', - 'port': 8080, - 'base_dir': './downloads', - 'api_version': 'v2', - 'directory_listing': True, - 'enable_stats': True, - 'auth_type': 'none', - 'max_upload_size': 1024 * 1024 * 1024, - 'timeout': 30, - 'verbose': 0, - 'enable_range': True, - 'ignore_hidden': True, - 'show_hash': False, - 'calculate_hash': False, - 'max_search_results': 100, - 'enable_ws': True, - 'enable_sse': True, - 'enable_monitor': True, - 'monitor_interval': 5, - 'enable_sync': True, - 'enable_mirrors': True, - 'database': { - 'enabled': True, - 'type': 'sqlite', - 'sqlite': {'path': './data/hyc.db'} - }, - 'mirrors': { - 'docker': {'enabled': True}, - 'apt': {'enabled': True}, - 'yum': {'enabled': True}, - 'pypi': {'enabled': True}, - 'npm': {'enabled': True}, - 'go': {'enabled': True} - }, - 'sync_sources': {}, - 'webhooks': {'enabled': False, 'storage': 'webhooks.json'}, - 'auth_sessions_file': 'auth_sessions.json', - 'auth_session_timeout': 3600, - 'auth_cookie_max_age': 86400 - } - print(f"警告: 未找到默认配置文件 ({self._settings_path})，使用内联默认配置") - return default_config - - @classmethod - def from_settings(cls, custom_config: Dict[str, Any] = None, settings_path: str = None) -> 'ConfigManager': - """ - 从默认配置创建配置管理器 - - Args: - custom_config: 自定义配置，会覆盖默认配置 - settings_path: 默认配置文件路径 - - Returns: - ConfigManager 实例 - """ - return cls(config=custom_config, settings_path=settings_path) - - def _validate_config(self, config: Dict[str, Any]) -> Dict[str, Any]: - """验证和修复配置""" - # 确保必要配置存在 - required = ['base_dir', 'host', 'port'] - for key in required: - if key not in config: - raise ValueError(f"缺少必要配置: {key}") - - # 修复路径配置 - config['base_dir'] = os.path.abspath(config['base_dir']) - - # 设置默认值（不在 _validate_config 中处理，由默认配置提供） - - # 验证认证配置 - auth_type = config.get('auth_type', 'none') - if auth_type == 'basic': - if 'auth_user' not in config: - config['auth_user'] = 'admin' - if 'auth_pass' not in config: - config['auth_pass'] = 'admin123' - elif auth_type == 'token': - # 只在缺失时生成 token，避免每次配置验证/更新都使已签发 token 失效 - if not config.get('auth_token'): - import secrets - config['auth_token'] = secrets.token_hex(32) - - # 验证上传大小配置 - if 'max_upload_size' in config: - try: - if isinstance(config['max_upload_size'], str): - config['max_upload_size'] = parse_size(config['max_upload_size']) - except ValueError as e: - print(f"警告: 无效的上传大小配置: {e}") - config['max_upload_size'] = 1024 * 1024 * 1024 - - # 验证端口范围 - if 'port' in config: - port = config['port'] - if not (1 <= port <= 65535): - raise ValueError(f"无效的端口号: {port}") - - # 验证并创建必要目录 - base_dir = config['base_dir'] - try: - if not os.path.exists(base_dir): - os.makedirs(base_dir, exist_ok=True) - - # 测试写入权限 - test_file = os.path.join(base_dir, '.write_test') - with open(test_file, 'w') as f: - f.write('test') - os.remove(test_file) - - except Exception as e: - raise ValueError(f"基础目录无法访问: {e}") - - # 获取项目根目录（脚本所在目录） - project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - # 在项目根目录创建必要的数据目录 - necessary_dirs = [ - os.path.join(project_root, 'data'), - os.path.join(project_root, 'logs'), - ] - - for dir_path in necessary_dirs: - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path, exist_ok=True) - except Exception as e: - print(f"警告: 无法创建目录 {dir_path}: {e}") - - # 更新配置指向项目根目录 - config['data_dir'] = os.path.join(project_root, 'data') - config['logs_dir'] = os.path.join(project_root, 'logs') - - return config - - def get(self, key: str, default: Any = None) -> Any: - """获取配置项""" - return self.config.get(key, default) - - def update(self, updates: Dict[str, Any]): - """更新配置""" - self.config.update(updates) - self.config = self._validate_config(self.config) - - def get_full_config(self) -> Dict[str, Any]: - """获取完整配置字典""" - return self.config.copy() - - def to_dict(self) -> Dict[str, Any]: - """返回配置字典（不包含敏感信息）""" - safe_config = { - "server_name": self.config.get("server_name", "Mirror Server"), - "version": "2.2", - "base_dir": self.config['base_dir'], - "directory_listing": self.config.get('directory_listing', True), - "max_upload_size": self.config.get('max_upload_size'), - "enable_stats": self.config.get('enable_stats', True), - "auth_type": self.config.get('auth_type', 'none'), - "sort_by": self.config.get('sort_by', 'name'), - "sort_reverse": self.config.get('sort_reverse', False), - "ignore_hidden": self.config.get('ignore_hidden', True), - "enable_range": self.config.get('enable_range', True), - "show_hash": self.config.get('show_hash', False), - "calculate_hash": self.config.get('calculate_hash', False), - "max_search_results": self.config.get('max_search_results', 100), - "api_version": self.config.get('api_version', 'v1'), - "verbose": self.config.get('verbose', 0) - } - return safe_config - - -def load_config_file(config_path: str) -> Dict[str, Any]: - """加载配置文件""" - if not os.path.exists(config_path): - return {} - - try: - with open(config_path, 'r', encoding='utf-8') as f: - return json.load(f) - except Exception as e: - print(f"错误: 无法加载配置文件 {config_path}: {e}") - return {} - - -def load_settings_with_override(settings_path: str, override_path: str = None) -> Dict[str, Any]: - """ - 加载默认配置并合并覆盖配置 - - Args: - settings_path: 默认配置文件路径 - override_path: 覆盖配置文件路径（可选） - - Returns: - 合并后的完整配置 - """ - # 加载默认配置 - default_config = load_json_config(settings_path) or {} - - # 加载覆盖配置 - override_config = {} - if override_path: - override_config = load_json_config(override_path) or {} - - # 深度合并 - return deep_merge(default_config, override_config) - - -def save_config_file(config_path: str, config: Dict[str, Any]) -> bool: - """ - 保存配置文件 - - Args: - config_path: 保存路径 - config: 配置字典 - - Returns: - 是否保存成功 - """ - try: - # 创建目录 - os.makedirs(os.path.dirname(config_path), exist_ok=True) - - with open(config_path, 'w', encoding='utf-8') as f: - json.dump(config, f, ensure_ascii=False, indent=4) - - return True - except Exception as e: - print(f"错误: 无法保存配置文件 {config_path}: {e}") - return False +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""配置管理模块""" + +import os +import sys +import json +import hashlib +import time +from typing import Dict, Any, Optional + +from .utils import parse_size + + +def get_resource_path(relative_path: str) -> str: + """获取打包后的资源路径""" + if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): + # 打包后的路径 + return os.path.join(sys._MEIPASS, relative_path) + return os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), relative_path) + + # 打包模式 + if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): + # 外部目录：与 exe 同级 + external_path = os.path.join(os.path.dirname(sys.executable), relative_path) + if os.path.exists(external_path): + return external_path + + # 打包后的资源路径（_MEIPASS） + bundled_path = os.path.join(sys._MEIPASS, relative_path) + if os.path.exists(bundled_path): + return bundled_path + + return external_path + + # 开发模式 + return os.path.join(project_root, relative_path) + + +def deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]: + """ + 深度合并配置 + - 如果 override 中有的键，会完全替换 base 中的值（除非都是 dict） + - 如果都是 dict，则递归合并 + - 不会修改原参数 + + Args: + base: 默认配置（基础配置） + override: 要合并的配置（优先级更高） + + Returns: + 合并后的配置 + """ + result = base.copy() + + for key, value in override.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + # 两者都是字典，递归合并 + result[key] = deep_merge(result[key], value) + else: + # 直接覆盖 + result[key] = value + + return result + + +def load_json_config(file_path: str) -> Optional[Dict[str, Any]]: + """加载 JSON 配置文件""" + if not os.path.exists(file_path): + return None + + try: + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) + except json.JSONDecodeError as e: + print(f"警告: 配置文件 {file_path} JSON 格式错误: {e}") + return None + except Exception as e: + print(f"警告: 无法读取配置文件 {file_path}: {e}") + return None + + +class ConfigManager: + """配置管理器""" + + def __init__(self, config: Dict[str, Any] = None, settings_path: str = None): + """ + 初始化配置管理器 + + Args: + config: 传入的配置（会覆盖默认配置） + settings_path: 默认配置文件路径（可以是位置参数或关键字参数） + """ + # 支持 settings_path 作为位置参数 + if isinstance(config, str): + settings_path = config + config = None + elif settings_path is None: + # 使用打包后的资源路径 + settings_path = get_resource_path('settings.json') + elif settings_path: + settings_path = settings_path + + self._settings_path = settings_path + + # 加载默认配置 + self.default_config = self._load_default_config() + + # 合并传入的配置 + if config: + self.config = self._validate_config(deep_merge(self.default_config, config)) + else: + self.config = self._validate_config(self.default_config.copy()) + + def _load_default_config(self) -> Dict[str, Any]: + """加载默认配置文件""" + default_config = load_json_config(self._settings_path) + if default_config is None: + # 如果找不到默认配置，使用内联最小配置 + default_config = { + 'server_name': 'HYC下载站', + 'host': '0.0.0.0', + 'port': 8080, + 'base_dir': './downloads', + 'api_version': 'v2', + 'directory_listing': True, + 'enable_stats': True, + 'auth_type': 'none', + 'max_upload_size': 1024 * 1024 * 1024, + 'timeout': 30, + 'verbose': 0, + 'enable_range': True, + 'ignore_hidden': True, + 'show_hash': False, + 'calculate_hash': False, + 'max_search_results': 100, + 'enable_ws': True, + 'enable_sse': True, + 'enable_monitor': True, + 'monitor_interval': 5, + 'enable_sync': True, + 'enable_mirrors': True, + 'database': { + 'enabled': True, + 'type': 'sqlite', + 'sqlite': {'path': './data/hyc.db'} + }, + 'mirrors': { + 'docker': {'enabled': True}, + 'apt': {'enabled': True}, + 'yum': {'enabled': True}, + 'pypi': {'enabled': True}, + 'npm': {'enabled': True}, + 'go': {'enabled': True} + }, + 'sync_sources': {}, + 'webhooks': {'enabled': False, 'storage': 'webhooks.json'}, + 'auth_sessions_file': 'auth_sessions.json', + 'auth_session_timeout': 3600, + 'auth_cookie_max_age': 86400 + } + print(f"警告: 未找到默认配置文件 ({self._settings_path})，使用内联默认配置") + return default_config + + @classmethod + def from_settings(cls, custom_config: Dict[str, Any] = None, settings_path: str = None) -> 'ConfigManager': + """ + 从默认配置创建配置管理器 + + Args: + custom_config: 自定义配置，会覆盖默认配置 + settings_path: 默认配置文件路径 + + Returns: + ConfigManager 实例 + """ + return cls(config=custom_config, settings_path=settings_path) + + def _validate_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """验证和修复配置""" + # 确保必要配置存在 + required = ['base_dir', 'host', 'port'] + for key in required: + if key not in config: + raise ValueError(f"缺少必要配置: {key}") + + # 修复路径配置 + config['base_dir'] = os.path.abspath(config['base_dir']) + + # 设置默认值（不在 _validate_config 中处理，由默认配置提供） + + # 验证认证配置 + auth_type = config.get('auth_type', 'none') + if auth_type == 'basic': + if 'auth_user' not in config: + config['auth_user'] = 'admin' + if 'auth_pass' not in config: + config['auth_pass'] = 'admin123' + elif auth_type == 'token': + # 只在缺失时生成 token，避免每次配置验证/更新都使已签发 token 失效 + if not config.get('auth_token'): + import secrets + config['auth_token'] = secrets.token_hex(32) + + # 验证上传大小配置 + if 'max_upload_size' in config: + try: + if isinstance(config['max_upload_size'], str): + config['max_upload_size'] = parse_size(config['max_upload_size']) + except ValueError as e: + print(f"警告: 无效的上传大小配置: {e}") + config['max_upload_size'] = 1024 * 1024 * 1024 + + # 验证端口范围 + if 'port' in config: + port = config['port'] + if not (1 <= port <= 65535): + raise ValueError(f"无效的端口号: {port}") + + # 验证并创建必要目录 + base_dir = config['base_dir'] + try: + if not os.path.exists(base_dir): + os.makedirs(base_dir, exist_ok=True) + + # 测试写入权限 + test_file = os.path.join(base_dir, '.write_test') + with open(test_file, 'w') as f: + f.write('test') + os.remove(test_file) + + except Exception as e: + raise ValueError(f"基础目录无法访问: {e}") + + # 获取项目根目录（脚本所在目录） + project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # 在项目根目录创建必要的数据目录 + necessary_dirs = [ + os.path.join(project_root, 'data'), + os.path.join(project_root, 'logs'), + ] + + for dir_path in necessary_dirs: + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path, exist_ok=True) + except Exception as e: + print(f"警告: 无法创建目录 {dir_path}: {e}") + + # 更新配置指向项目根目录 + config['data_dir'] = os.path.join(project_root, 'data') + config['logs_dir'] = os.path.join(project_root, 'logs') + + return config + + def get(self, key: str, default: Any = None) -> Any: + """获取配置项""" + return self.config.get(key, default) + + def update(self, updates: Dict[str, Any]): + """更新配置""" + self.config.update(updates) + self.config = self._validate_config(self.config) + + def get_full_config(self) -> Dict[str, Any]: + """获取完整配置字典""" + return self.config.copy() + + def to_dict(self) -> Dict[str, Any]: + """返回配置字典（不包含敏感信息）""" + safe_config = { + "server_name": self.config.get("server_name", "Mirror Server"), + "version": "2.3", + "base_dir": self.config['base_dir'], + "directory_listing": self.config.get('directory_listing', True), + "max_upload_size": self.config.get('max_upload_size'), + "enable_stats": self.config.get('enable_stats', True), + "auth_type": self.config.get('auth_type', 'none'), + "sort_by": self.config.get('sort_by', 'name'), + "sort_reverse": self.config.get('sort_reverse', False), + "ignore_hidden": self.config.get('ignore_hidden', True), + "enable_range": self.config.get('enable_range', True), + "show_hash": self.config.get('show_hash', False), + "calculate_hash": self.config.get('calculate_hash', False), + "max_search_results": self.config.get('max_search_results', 100), + "api_version": self.config.get('api_version', 'v1'), + "verbose": self.config.get('verbose', 0) + } + return safe_config + + +def load_config_file(config_path: str) -> Dict[str, Any]: + """加载配置文件""" + if not os.path.exists(config_path): + return {} + + try: + with open(config_path, 'r', encoding='utf-8') as f: + return json.load(f) + except Exception as e: + print(f"错误: 无法加载配置文件 {config_path}: {e}") + return {} + + +def load_settings_with_override(settings_path: str, override_path: str = None) -> Dict[str, Any]: + """ + 加载默认配置并合并覆盖配置 + + Args: + settings_path: 默认配置文件路径 + override_path: 覆盖配置文件路径（可选） + + Returns: + 合并后的完整配置 + """ + # 加载默认配置 + default_config = load_json_config(settings_path) or {} + + # 加载覆盖配置 + override_config = {} + if override_path: + override_config = load_json_config(override_path) or {} + + # 深度合并 + return deep_merge(default_config, override_config) + + +def save_config_file(config_path: str, config: Dict[str, Any]) -> bool: + """ + 保存配置文件 + + Args: + config_path: 保存路径 + config: 配置字典 + + Returns: + 是否保存成功 + """ + try: + # 创建目录 + os.makedirs(os.path.dirname(config_path), exist_ok=True) + + with open(config_path, 'w', encoding='utf-8') as f: + json.dump(config, f, ensure_ascii=False, indent=4) + + return True + except Exception as e: + print(f"错误: 无法保存配置文件 {config_path}: {e}") + return False diff --git a/core/database.py b/core/database.py index 1ac1447..44d307d 100644 --- a/core/database.py +++ b/core/database.py @@ -1,1553 +1,1570 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -数据库模块 -提供本地文件和数据库的双存储支持 -""" - -import os -import json -import time -import threading -from datetime import datetime -from typing import Dict, List, Optional, Any -from dataclasses import dataclass, field -from contextlib import contextmanager - -from sqlalchemy import create_engine, Column, Integer, String, Float, Boolean, DateTime, Text, BigInteger, Index, text -from sqlalchemy.orm import sessionmaker, declarative_base -from sqlalchemy.pool import QueuePool - -Base = declarative_base() - - -# ==================== 环境变量支持 ==================== - -def load_db_config_from_env() -> dict: - """ - 从环境变量加载数据库配置 - 支持的变量: - DB_TYPE: 数据库类型 (sqlite/postgresql/mysql/external) - DB_PATH: SQLite 数据库路径 - DB_HOST: 数据库主机地址 - DB_PORT: 数据库端口 - DB_NAME: 数据库名 - DB_USER: 数据库用户 - DB_PASS: 数据库密码 - DB_CONN_STR: 完整连接字符串 (用于外部数据库) - DB_TABLE_PREFIX: 表前缀 - """ - config = {} - - db_type = os.environ.get('DB_TYPE', '').lower() - if db_type: - config['type'] = db_type - - # SQLite - db_path = os.environ.get('DB_PATH', '') - if db_path: - config['sqlite'] = {'path': db_path} - - # PostgreSQL / MySQL 通用 - db_host = os.environ.get('DB_HOST', '') - if db_host: - if 'postgresql' in db_type: - config['postgresql'] = { - 'host': db_host, - 'port': int(os.environ.get('DB_PORT', 5432)), - 'database': os.environ.get('DB_NAME', 'hyc'), - 'user': os.environ.get('DB_USER', 'postgres'), - 'password': os.environ.get('DB_PASS', '') - } - elif 'mysql' in db_type: - config['mysql'] = { - 'host': db_host, - 'port': int(os.environ.get('DB_PORT', 3306)), - 'database': os.environ.get('DB_NAME', 'hyc'), - 'user': os.environ.get('DB_USER', 'root'), - 'password': os.environ.get('DB_PASS', '') - } - - # 外部数据库连接字符串 - db_conn_str = os.environ.get('DB_CONN_STR', '') - if db_conn_str: - config['external'] = {'connection_string': db_conn_str} - - # 表前缀 - table_prefix = os.environ.get('DB_TABLE_PREFIX', '') - if table_prefix: - # 添加到对应类型的配置 - for db_key in ['sqlite', 'postgresql', 'mysql', 'external']: - if db_key in config: - config[db_key]['table_prefix'] = table_prefix - - return config - - -def merge_config(file_config: dict, env_config: dict) -> dict: - """合并配置文件和环境变量配置""" - merged = file_config.copy() - - # 如果有环境变量配置，合并 database 部分 - if 'database' in env_config and env_config['database']: - if 'database' not in merged: - merged['database'] = {} - merged['database'].update(env_config['database']) - - return merged - - -# ==================== 数据库模型定义 ==================== - -# 表名前缀（可在初始化时设置） -_TABLE_PREFIX = '' -_SCHEMA_VERSION = 1 # 当前Schema版本 - - -def set_table_prefix(prefix: str): - """设置表前缀""" - global _TABLE_PREFIX - _TABLE_PREFIX = prefix - - -def get_table_name(base_name: str) -> str: - """获取带前缀的表名""" - return f"{_TABLE_PREFIX}{base_name}" - - -class SchemaVersion(Base): - """数据库结构版本表""" - __tablename__ = 'schema_versions' - - id = Column(Integer, primary_key=True, autoincrement=True) - version = Column(Integer, nullable=False, unique=True) - applied_at = Column(Float, default=time.time) - description = Column(String(255), nullable=True) - - -class FileRecord(Base): - """文件记录表""" - __tablename__ = 'files' - - id = Column(Integer, primary_key=True, autoincrement=True) - file_id = Column(String(64), unique=True, nullable=False, index=True) # 文件唯一ID - path = Column(String(1024), nullable=False, index=True) # 文件路径 - name = Column(String(512), nullable=False) # 文件名 - size = Column(BigInteger, default=0) # 文件大小 - hash = Column(String(64), nullable=True) # 文件hash - mime_type = Column(String(128), nullable=True) # MIME类型 - is_dir = Column(Boolean, default=False) # 是否是目录 - created_at = Column(Float, default=time.time) # 创建时间 - updated_at = Column(Float, default=time.time) # 更新时间 - last_accessed = Column(Float, default=time.time) # 最后访问时间 - download_count = Column(Integer, default=0) # 下载次数 - is_deleted = Column(Boolean, default=False, index=True) # 软删除标记 - sync_status = Column(String(32), default='synced') # 同步状态: pending, synced, error - - __table_args__ = ( - Index('idx_files_path_status', 'path', 'is_deleted'), - ) - - def to_dict(self) -> dict: - """转换为字典""" - return { - 'id': self.id, - 'file_id': self.file_id, - 'path': self.path, - 'name': self.name, - 'size': self.size, - 'hash': self.hash, - 'mime_type': self.mime_type, - 'is_dir': self.is_dir, - 'created_at': self.created_at, - 'updated_at': self.updated_at, - 'last_accessed': self.last_accessed, - 'download_count': self.download_count, - 'is_deleted': self.is_deleted, - 'sync_status': self.sync_status - } - - -class SyncRecord(Base): - """同步记录表""" - __tablename__ = 'sync_records' - - id = Column(Integer, primary_key=True, autoincrement=True) - sync_id = Column(String(64), unique=True, nullable=False, index=True) - source_type = Column(String(64), nullable=False) # 同步源类型 - source_name = Column(String(256), nullable=False) # 同步源名称 - status = Column(String(32), default='pending') # pending, running, completed, failed - total_files = Column(Integer, default=0) - synced_files = Column(Integer, default=0) - failed_files = Column(Integer, default=0) - total_size = Column(BigInteger, default=0) - synced_size = Column(BigInteger, default=0) - started_at = Column(Float, default=time.time) - completed_at = Column(Float, nullable=True) - error_message = Column(Text, nullable=True) - - def to_dict(self) -> dict: - return { - 'id': self.id, - 'sync_id': self.sync_id, - 'source_type': self.source_type, - 'source_name': self.source_name, - 'status': self.status, - 'total_files': self.total_files, - 'synced_files': self.synced_files, - 'failed_files': self.failed_files, - 'total_size': self.total_size, - 'synced_size': self.synced_size, - 'started_at': self.started_at, - 'completed_at': self.completed_at, - 'error_message': self.error_message - } - - -class CacheRecord(Base): - """缓存记录表""" - __tablename__ = 'cache_records' - - id = Column(Integer, primary_key=True, autoincrement=True) - cache_key = Column(String(512), unique=True, nullable=False, index=True) - cache_type = Column(String(64), nullable=False) # docker, apt, pypi, etc. - file_path = Column(String(1024), nullable=True) - file_size = Column(BigInteger, default=0) - file_hash = Column(String(64), nullable=True) - hits = Column(Integer, default=0) - created_at = Column(Float, default=time.time) - expires_at = Column(Float, nullable=True) - last_hit = Column(Float, default=time.time) - - def to_dict(self) -> dict: - return { - 'id': self.id, - 'cache_key': self.cache_key, - 'cache_type': self.cache_type, - 'file_path': self.file_path, - 'file_size': self.file_size, - 'file_hash': self.file_hash, - 'hits': self.hits, - 'created_at': self.created_at, - 'expires_at': self.expires_at, - 'last_hit': self.last_hit - } - - -class DownloadRecord(Base): - """下载记录表""" - __tablename__ = 'download_records' - - id = Column(Integer, primary_key=True, autoincrement=True) - file_path = Column(String(1024), nullable=False, index=True) - file_size = Column(BigInteger, default=0) - download_time = Column(Float, default=time.time) - duration = Column(Float, default=0) - client_ip = Column(String(64), nullable=True) - user_agent = Column(String(512), nullable=True) - success = Column(Boolean, default=True) - error_message = Column(Text, nullable=True) - - def to_dict(self) -> dict: - return { - 'id': self.id, - 'file_path': self.file_path, - 'file_size': self.file_size, - 'download_time': self.download_time, - 'duration': self.duration, - 'client_ip': self.client_ip, - 'user_agent': self.user_agent, - 'success': self.success, - 'error_message': self.error_message - } - - -class MonitorHistoryRecord(Base): - """监控历史记录表""" - __tablename__ = 'monitor_history' - - id = Column(Integer, primary_key=True, autoincrement=True) - timestamp = Column(Float, nullable=False, index=True) - cpu_percent = Column(Float, default=0) - memory_percent = Column(Float, default=0) - disk_percent = Column(Float, default=0) - network_rx = Column(BigInteger, default=0) - network_tx = Column(BigInteger, default=0) - active_connections = Column(Integer, default=0) - server_uptime = Column(Float, default=0) - - def to_dict(self) -> dict: - return { - 'id': self.id, - 'timestamp': self.timestamp, - 'cpu_percent': self.cpu_percent, - 'memory_percent': self.memory_percent, - 'disk_percent': self.disk_percent, - 'network_rx': self.network_rx, - 'network_tx': self.network_tx, - 'active_connections': self.active_connections, - 'server_uptime': self.server_uptime - } - - -class WebhookRecord(Base): - """Webhook配置记录表""" - __tablename__ = 'webhooks' - - id = Column(Integer, primary_key=True, autoincrement=True) - name = Column(String(255), nullable=False) - url = Column(String(2048), nullable=False) - events = Column(Text, nullable=True) # JSON 格式存储 - secret = Column(String(255), nullable=True) - enabled = Column(Boolean, default=True) - created_at = Column(Float, default=time.time) - updated_at = Column(Float, default=time.time) - - def to_dict(self) -> dict: - import json - return { - 'id': self.id, - 'name': self.name, - 'url': self.url, - 'events': json.loads(self.events) if self.events else [], - 'secret': self.secret, - 'enabled': self.enabled, - 'created_at': self.created_at, - 'updated_at': self.updated_at - } - - -class WebhookDeliveryRecord(Base): - """Webhook交付记录表""" - __tablename__ = 'webhook_deliveries' - - id = Column(Integer, primary_key=True, autoincrement=True) - webhook_id = Column(Integer, nullable=False, index=True) - event = Column(String(100), nullable=False) - status = Column(String(50), nullable=False) # success, failed, pending - status_code = Column(Integer, nullable=True) - response_body = Column(Text, nullable=True) - error_message = Column(Text, nullable=True) - duration_ms = Column(Float, nullable=True) - created_at = Column(Float, default=time.time, index=True) - retry_count = Column(Integer, default=0) - - def to_dict(self) -> dict: - return { - 'id': self.id, - 'webhook_id': self.webhook_id, - 'event': self.event, - 'status': self.status, - 'status_code': self.status_code, - 'response_body': self.response_body[:500] if self.response_body else None, - 'error_message': self.error_message, - 'duration_ms': self.duration_ms, - 'created_at': self.created_at, - 'retry_count': self.retry_count - } - - -class UserRecord(Base): - """用户账号记录表""" - __tablename__ = 'users' - - id = Column(Integer, primary_key=True, autoincrement=True) - username = Column(String(100), nullable=False, unique=True, index=True) - password_hash = Column(String(255), nullable=False) # bcrypt 加密后的哈希 - token = Column(String(64), nullable=True) # 登录生成的 token - token_expires_at = Column(Float, nullable=True) # token 过期时间 - role = Column(String(50), default='admin') # admin, user - email = Column(String(255), nullable=True) - phone = Column(String(50), nullable=True) - last_login = Column(Float, nullable=True) - login_count = Column(Integer, default=0) - failed_attempts = Column(Integer, default=0) # 登录失败次数 - locked_until = Column(Float, nullable=True) # 锁定直到时间戳 - created_at = Column(Float, default=time.time) - updated_at = Column(Float, default=time.time) - enabled = Column(Boolean, default=True) - - def to_dict(self) -> dict: - return { - 'id': self.id, - 'username': self.username, - 'password_hash': self.password_hash, - 'token': self.token, - 'token_expires_at': self.token_expires_at, - 'role': self.role, - 'email': self.email, - 'phone': self.phone, - 'last_login': self.last_login, - 'login_count': self.login_count, - 'failed_attempts': self.failed_attempts, - 'locked_until': self.locked_until, - 'created_at': self.created_at, - 'updated_at': self.updated_at, - 'enabled': self.enabled - } - - -class LoginLogRecord(Base): - """登录日志记录表""" - __tablename__ = 'login_logs' - - id = Column(Integer, primary_key=True, autoincrement=True) - username = Column(String(100), nullable=False, index=True) - ip_address = Column(String(50), nullable=True) - user_agent = Column(String(500), nullable=True) - status = Column(String(20), nullable=False) # success, failed, locked - reason = Column(String(255), nullable=True) - created_at = Column(Float, default=time.time, index=True) - - def to_dict(self) -> dict: - return { - 'id': self.id, - 'username': self.username, - 'ip_address': self.ip_address, - 'user_agent': self.user_agent, - 'status': self.status, - 'reason': self.reason, - 'created_at': self.created_at - } - - -# ==================== 数据库管理器 ==================== - -class DatabaseManager: - """数据库管理器""" - - _instance = None - _lock = threading.Lock() - - def __new__(cls, config: dict = None): - if cls._instance is None: - with cls._lock: - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance._initialized = False - return cls._instance - - def __init__(self, config: dict = None): - if self._initialized: - return - - self.config = config or {} - self.db_type = self.config.get('type', 'sqlite') # sqlite, postgresql, mysql, external - - # 获取数据库配置 - db_config = self.config.get(self.db_type, {}) - - # 获取表前缀（支持多租户） - self.table_prefix = db_config.get('table_prefix', '') - - # 创建数据库引擎 - self._create_engine(db_config) - - # 设置表前缀（需要在创建表之前） - if self.table_prefix: - set_table_prefix(self.table_prefix) - - # 自动创建表结构 - self._create_tables() - - # 会话工厂 - self.Session = sessionmaker(bind=self.engine) - - self._initialized = True - self._operation_count = 0 - self._last_sync_time = time.time() - self._sync_interval = self.config.get('sync_interval', 60) # 默认60秒同步 - - def _create_tables(self): - """自动创建数据库表结构""" - try: - # 尝试创建所有表 - Base.metadata.create_all(self.engine) - # 对于 MySQL，需要提交事务 - if self.db_type == 'mysql': - from sqlalchemy import text - with self.engine.connect() as conn: - conn.commit() - except Exception as e: - print(f"警告: 创建表结构失败: {e}") - print("将尝试创建数据库...") - self._create_database() - - def _create_database(self): - """创建数据库（如果不存在）""" - # SQLite 不需要预创建数据库 - if self.db_type == 'sqlite': - return - - # 对于 PostgreSQL/MySQL，尝试创建数据库 - try: - if self.db_type == 'postgresql': - # 连接到默认数据库 postgres - from sqlalchemy import text - db_config = self.config.get('postgresql', {}) - host = db_config.get('host', 'localhost') - port = db_config.get('port', 5432) - user = db_config.get('user', 'postgres') - password = db_config.get('password', '') - database = db_config.get('database', 'hyc') - - temp_engine = create_engine( - f"postgresql://{user}:{password}@{host}:{port}/postgres" - ) - with temp_engine.connect() as conn: - # 检查数据库是否存在 - result = conn.execute( - text(f"SELECT 1 FROM pg_database WHERE datname = '{database}'") - ).fetchone() - if not result: - conn.execute(text(f"CREATE DATABASE {database}")) - print(f"已创建数据库: {database}") - temp_engine.dispose() - - elif self.db_type == 'mysql': - from sqlalchemy import text - db_config = self.config.get('mysql', {}) - host = db_config.get('host', 'localhost') - port = db_config.get('port', 3306) - user = db_config.get('user', 'root') - password = db_config.get('password', '') - - temp_engine = create_engine( - f"mysql+pymysql://{user}:{password}@{host}:{port}" - ) - with temp_engine.connect() as conn: - db_config = self.config.get('mysql', {}) - database = db_config.get('database', 'hyc') - try: - conn.execute(text(f"CREATE DATABASE IF NOT EXISTS {database} CHARACTER SET utf8mb4")) - conn.commit() # 提交事务 - print(f"已创建/确认数据库: {database}") - except Exception as e: - print(f"警告: 创建数据库失败: {e}") - temp_engine.dispose() - - except Exception as e: - print(f"警告: 创建数据库失败: {e}") - - def _create_engine(self, db_config: dict): - """创建数据库引擎""" - if self.db_type == 'sqlite': - # SQLite 配置 - db_path = db_config.get('path', './data/hyc.db') - # 确保数据库目录存在 - os.makedirs(os.path.dirname(db_path), exist_ok=True) - - db_url = f"sqlite:///{db_path}" - self.engine = create_engine( - db_url, - poolclass=QueuePool, - pool_size=5, - max_overflow=10, - pool_recycle=3600 - ) - - elif self.db_type == 'postgresql': - # PostgreSQL 配置 - host = db_config.get('host', 'localhost') - port = db_config.get('port', 5432) - database = db_config.get('database', 'hyc') - user = db_config.get('user', 'postgres') - password = db_config.get('password', '') - ssl_mode = db_config.get('ssl_mode', 'prefer') - timeout = db_config.get('connection_timeout', 30) - - db_url = f"postgresql://{user}:{password}@{host}:{port}/{database}?sslmode={ssl_mode}" - self.engine = create_engine( - db_url, - poolclass=QueuePool, - pool_size=db_config.get('pool_size', 5), - max_overflow=db_config.get('max_overflow', 10), - pool_recycle=3600, - connect_args={'connect_timeout': timeout} - ) - - elif self.db_type == 'mysql': - # MySQL 配置 - host = db_config.get('host', 'localhost') - port = db_config.get('port', 3306) - database = db_config.get('database', 'hyc') - user = db_config.get('user', 'root') - password = db_config.get('password', '') - charset = db_config.get('charset', 'utf8mb4') - timeout = db_config.get('connection_timeout', 30) - - db_url = f"mysql+pymysql://{user}:{password}@{host}:{port}/{database}?charset={charset}" - self.engine = create_engine( - db_url, - poolclass=QueuePool, - pool_size=db_config.get('pool_size', 5), - max_overflow=db_config.get('max_overflow', 10), - pool_recycle=3600, - connect_args={'connect_timeout': timeout} - ) - - elif self.db_type == 'external': - # 外部数据库 - 使用完整连接字符串 - connection_string = db_config.get('connection_string', '') - if not connection_string: - raise ValueError("外部数据库配置需要提供 connection_string") - - self.engine = create_engine( - connection_string, - poolclass=QueuePool, - pool_size=db_config.get('pool_size', 5), - max_overflow=db_config.get('max_overflow', 10), - pool_recycle=3600 - ) - - else: - raise ValueError(f"不支持的数据库类型: {self.db_type}. 支持: sqlite, postgresql, mysql, external") - - @contextmanager - def session(self): - """获取数据库会话""" - session = self.Session() - try: - yield session - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() - - # ==================== 文件操作 ==================== - - def add_file(self, file_id: str, path: str, name: str, - size: int = 0, hash: str = None, mime_type: str = None, - is_dir: bool = False, created_at: float = None) -> FileRecord: - """添加文件记录""" - with self.session() as session: - record = FileRecord( - file_id=file_id, - path=path, - name=name, - size=size, - hash=hash, - mime_type=mime_type, - is_dir=is_dir, - created_at=created_at or time.time(), - updated_at=time.time(), - sync_status='pending' - ) - session.add(record) - self._operation_count += 1 - return record - - def update_file(self, file_id: str, **kwargs) -> Optional[FileRecord]: - """更新文件记录""" - with self.session() as session: - record = session.query(FileRecord).filter( - FileRecord.file_id == file_id, - FileRecord.is_deleted == False - ).first() - if record: - for key, value in kwargs.items(): - if hasattr(record, key): - setattr(record, key, value) - record.updated_at = time.time() - record.sync_status = 'pending' - self._operation_count += 1 - return record - - def delete_file(self, file_id: str, hard: bool = False) -> bool: - """删除文件记录""" - with self.session() as session: - record = session.query(FileRecord).filter( - FileRecord.file_id == file_id - ).first() - if record: - if hard: - session.delete(record) - else: - record.is_deleted = True - record.updated_at = time.time() - record.sync_status = 'pending' - self._operation_count += 1 - return True - return False - - def get_file(self, file_id: str) -> Optional[FileRecord]: - """获取文件记录""" - with self.session() as session: - return session.query(FileRecord).filter( - FileRecord.file_id == file_id, - FileRecord.is_deleted == False - ).first() - - def get_file_by_path(self, path: str) -> Optional[FileRecord]: - """根据路径获取文件记录""" - with self.session() as session: - return session.query(FileRecord).filter( - FileRecord.path == path, - FileRecord.is_deleted == False - ).first() - - def list_files(self, path: str = '/', recursive: bool = False, - include_deleted: bool = False, limit: int = 1000, - offset: int = 0) -> List[FileRecord]: - """列出文件记录""" - with self.session() as session: - query = session.query(FileRecord) - - if not include_deleted: - query = query.filter(FileRecord.is_deleted == False) - - if path and path != '/': - if recursive: - query = query.filter(FileRecord.path.startswith(path)) - else: - parent_path = path.rstrip('/') + '/' - query = query.filter( - (FileRecord.path == path) | - (FileRecord.path.startswith(parent_path)) - ) - - return query.order_by(FileRecord.path).offset(offset).limit(limit).all() - - def search_files(self, keyword: str, limit: int = 100) -> List[FileRecord]: - """搜索文件""" - with self.session() as session: - return session.query(FileRecord).filter( - FileRecord.is_deleted == False, - (FileRecord.name.contains(keyword) | - FileRecord.path.contains(keyword)) - ).limit(limit).all() - - def increment_download_count(self, file_id: str) -> bool: - """增加下载计数（通过 file_id 查找）""" - with self.session() as session: - record = session.query(FileRecord).filter( - FileRecord.file_id == file_id - ).first() - if record: - record.download_count += 1 - self._operation_count += 1 - return True - return False - - # ==================== 同步记录操作 ==================== - - def add_sync_record(self, sync_id: str, source_type: str, - source_name: str, **kwargs) -> SyncRecord: - """添加同步记录""" - with self.session() as session: - record = SyncRecord( - sync_id=sync_id, - source_type=source_type, - source_name=source_name, - started_at=time.time(), - **kwargs - ) - session.add(record) - return record - - def update_sync_record(self, sync_id: str, **kwargs) -> Optional[SyncRecord]: - """更新同步记录""" - with self.session() as session: - record = session.query(SyncRecord).filter( - SyncRecord.sync_id == sync_id - ).first() - if record: - for key, value in kwargs.items(): - if hasattr(record, key): - setattr(record, key, value) - return record - return None - - def get_sync_records(self, limit: int = 50) -> List[SyncRecord]: - """获取同步记录""" - with self.session() as session: - return session.query(SyncRecord).order_by( - SyncRecord.started_at.desc() - ).limit(limit).all() - - # ==================== 缓存记录操作 ==================== - - def add_cache_record(self, cache_key: str, cache_type: str, - **kwargs) -> CacheRecord: - """添加缓存记录""" - with self.session() as session: - record = CacheRecord( - cache_key=cache_key, - cache_type=cache_type, - **kwargs - ) - session.add(record) - return record - - def update_cache_record(self, cache_key: str, **kwargs) -> Optional[CacheRecord]: - """更新缓存记录""" - with self.session() as session: - record = session.query(CacheRecord).filter( - CacheRecord.cache_key == cache_key - ).first() - if record: - for key, value in kwargs.items(): - if hasattr(record, key): - setattr(record, key, value) - return record - return None - - def get_cache_record(self, cache_key: str) -> Optional[CacheRecord]: - """获取缓存记录""" - with self.session() as session: - return session.query(CacheRecord).filter( - CacheRecord.cache_key == cache_key - ).first() - - def increment_cache_hits(self, cache_key: str) -> bool: - """增加缓存命中次数""" - with self.session() as session: - record = session.query(CacheRecord).filter( - CacheRecord.cache_key == cache_key - ).first() - if record: - record.hits += 1 - record.last_hit = time.time() - return True - return False - - def list_cache_records(self, cache_type: str = None, - limit: int = 100) -> List[CacheRecord]: - """列出缓存记录""" - with self.session() as session: - query = session.query(CacheRecord) - if cache_type: - query = query.filter(CacheRecord.cache_type == cache_type) - return query.order_by(CacheRecord.hits.desc()).limit(limit).all() - - # ==================== 下载记录操作 ==================== - - def add_download_record(self, file_path: str, file_size: int = 0, - client_ip: str = None, user_agent: str = None, - success: bool = True, error_message: str = None, - duration: float = 0) -> DownloadRecord: - """添加下载记录""" - with self.session() as session: - record = DownloadRecord( - file_path=file_path, - file_size=file_size, - download_time=time.time(), - client_ip=client_ip, - user_agent=user_agent, - success=success, - error_message=error_message, - duration=duration - ) - session.add(record) - self._operation_count += 1 - return record - - def get_download_records(self, file_path: str = None, - limit: int = 100) -> List[DownloadRecord]: - """获取下载记录""" - with self.session() as session: - query = session.query(DownloadRecord) - if file_path: - query = query.filter(DownloadRecord.file_path == file_path) - return query.order_by( - DownloadRecord.download_time.desc() - ).limit(limit).all() - - def get_download_stats(self, days: int = 7) -> dict: - """获取下载统计""" - from sqlalchemy import func - start_time = time.time() - (days * 86400) - - with self.session() as session: - total_downloads = session.query(func.count(DownloadRecord.id)).filter( - DownloadRecord.download_time >= start_time - ).scalar() - - successful_downloads = session.query(func.count(DownloadRecord.id)).filter( - DownloadRecord.download_time >= start_time, - DownloadRecord.success == True - ).scalar() - - total_bytes = session.query(func.sum(DownloadRecord.file_size)).filter( - DownloadRecord.download_time >= start_time, - DownloadRecord.success == True - ).scalar() or 0 - - return { - 'total_downloads': total_downloads, - 'successful_downloads': successful_downloads, - 'failed_downloads': total_downloads - successful_downloads, - 'total_bytes': total_bytes, - 'total_human': self._format_size(total_bytes) - } - - # ==================== 监控历史记录 ==================== - - def add_monitor_record(self, cpu_percent: float = 0, memory_percent: float = 0, - disk_percent: float = 0, network_rx: int = 0, - network_tx: int = 0, active_connections: int = 0, - server_uptime: float = 0) -> Optional[MonitorHistoryRecord]: - """添加监控记录""" - with self.session() as session: - record = MonitorHistoryRecord( - timestamp=time.time(), - cpu_percent=cpu_percent, - memory_percent=memory_percent, - disk_percent=disk_percent, - network_rx=network_rx, - network_tx=network_tx, - active_connections=active_connections, - server_uptime=server_uptime - ) - session.add(record) - self._operation_count += 1 - return record - - def get_monitor_history(self, hours: int = 24) -> List[MonitorHistoryRecord]: - """获取监控历史记录""" - from sqlalchemy import func - cutoff_time = time.time() - (hours * 3600) - - with self.session() as session: - # 按时间聚合，每5分钟一个数据点 - records = session.query( - MonitorHistoryRecord - ).filter( - MonitorHistoryRecord.timestamp >= cutoff_time - ).order_by( - MonitorHistoryRecord.timestamp.desc() - ).all() - - return records - - def get_monitor_stats(self, hours: int = 24) -> dict: - """获取监控统计数据""" - from sqlalchemy import func - cutoff_time = time.time() - (hours * 3600) - - with self.session() as session: - stats = session.query( - func.avg(MonitorHistoryRecord.cpu_percent).label('avg_cpu'), - func.max(MonitorHistoryRecord.cpu_percent).label('max_cpu'), - func.avg(MonitorHistoryRecord.memory_percent).label('avg_memory'), - func.max(MonitorHistoryRecord.memory_percent).label('max_memory'), - func.avg(MonitorHistoryRecord.disk_percent).label('avg_disk'), - func.max(MonitorHistoryRecord.disk_percent).label('max_disk'), - func.sum(MonitorHistoryRecord.network_rx).label('total_rx'), - func.sum(MonitorHistoryRecord.network_tx).label('total_tx') - ).filter( - MonitorHistoryRecord.timestamp >= cutoff_time - ).first() - - return { - 'avg_cpu': round(stats.avg_cpu, 1) if stats.avg_cpu else 0, - 'max_cpu': round(stats.max_cpu, 1) if stats.max_cpu else 0, - 'avg_memory': round(stats.avg_memory, 1) if stats.avg_memory else 0, - 'max_memory': round(stats.max_memory, 1) if stats.max_memory else 0, - 'avg_disk': round(stats.avg_disk, 1) if stats.avg_disk else 0, - 'max_disk': round(stats.max_disk, 1) if stats.max_disk else 0, - 'total_rx': stats.total_rx or 0, - 'total_tx': stats.total_tx or 0 - } - - # ==================== Webhook 管理 ==================== - - def get_webhooks(self) -> List[WebhookRecord]: - """获取所有 webhook 配置""" - import json - with self.session() as session: - return session.query(WebhookRecord).order_by( - WebhookRecord.created_at.desc() - ).all() - - def get_webhook(self, webhook_id: int) -> Optional[WebhookRecord]: - """获取单个 webhook 配置""" - with self.session() as session: - return session.query(WebhookRecord).filter( - WebhookRecord.id == webhook_id - ).first() - - def add_webhook(self, name: str, url: str, events: List[str] = None, - secret: str = None, enabled: bool = True) -> WebhookRecord: - """添加 webhook 配置""" - import json - with self.session() as session: - record = WebhookRecord( - name=name, - url=url, - events=json.dumps(events or []), - secret=secret, - enabled=enabled, - created_at=time.time(), - updated_at=time.time() - ) - session.add(record) - return record - - def update_webhook(self, webhook_id: int, **kwargs) -> Optional[WebhookRecord]: - """更新 webhook 配置""" - import json - with self.session() as session: - record = session.query(WebhookRecord).filter( - WebhookRecord.id == webhook_id - ).first() - if not record: - return None - - if 'name' in kwargs: - record.name = kwargs['name'] - if 'url' in kwargs: - record.url = kwargs['url'] - if 'events' in kwargs: - record.events = json.dumps(kwargs['events']) - if 'secret' in kwargs: - record.secret = kwargs['secret'] - if 'enabled' in kwargs: - record.enabled = kwargs['enabled'] - - record.updated_at = time.time() - return record - - def delete_webhook(self, webhook_id: int) -> bool: - """删除 webhook 配置""" - with self.session() as session: - result = session.query(WebhookRecord).filter( - WebhookRecord.id == webhook_id - ).delete() - return result > 0 - - # ==================== Webhook 交付记录 ==================== - - def add_webhook_delivery( - self, - webhook_id: int, - event: str, - status: str, - status_code: int = None, - response_body: str = None, - error_message: str = None, - duration_ms: float = None, - retry_count: int = 0 - ) -> WebhookDeliveryRecord: - """添加 webhook 交付记录""" - with self.session() as session: - record = WebhookDeliveryRecord( - webhook_id=webhook_id, - event=event, - status=status, - status_code=status_code, - response_body=response_body, - error_message=error_message, - duration_ms=duration_ms, - created_at=time.time(), - retry_count=retry_count - ) - session.add(record) - session.commit() - return record - - def get_webhook_deliveries( - self, - webhook_id: int = None, - status: str = None, - limit: int = 50 - ) -> List[WebhookDeliveryRecord]: - """获取 webhook 交付记录""" - with self.session() as session: - query = session.query(WebhookDeliveryRecord) - - if webhook_id: - query = query.filter(WebhookDeliveryRecord.webhook_id == webhook_id) - if status: - query = query.filter(WebhookDeliveryRecord.status == status) - - return query.order_by( - WebhookDeliveryRecord.created_at.desc() - ).limit(limit).all() - - def get_webhook_stats(self, webhook_id: int = None) -> Dict: - """获取 webhook 交付统计""" - with self.session() as session: - query = session.query(WebhookDeliveryRecord) - if webhook_id: - query = query.filter(WebhookDeliveryRecord.webhook_id == webhook_id) - - deliveries = query.all() - - total = len(deliveries) - success = sum(1 for d in deliveries if d.status == 'success') - failed = sum(1 for d in deliveries if d.status == 'failed') - - avg_duration = 0 - if deliveries: - durations = [d.duration_ms for d in deliveries if d.duration_ms] - if durations: - avg_duration = sum(durations) / len(durations) - - return { - 'total_deliveries': total, - 'success': success, - 'failed': failed, - 'pending': sum(1 for d in deliveries if d.status == 'pending'), - 'success_rate': (success / total * 100) if total > 0 else 0, - 'avg_duration_ms': round(avg_duration, 2) - } - - def cleanup_webhook_deliveries(self, older_than: int = 604800) -> int: - """ - 清理旧的 webhook 交付记录 - - Args: - older_than: 清理多少秒之前的记录，默认 7 天 (604800 秒) - - Returns: - 删除的记录数量 - """ - with self.session() as session: - cutoff = time.time() - older_than - result = session.query(WebhookDeliveryRecord).filter( - WebhookDeliveryRecord.created_at < cutoff - ).delete() - session.commit() - return result - - @staticmethod - def _format_size(size: int) -> str: - """格式化文件大小""" - for unit in ['B', 'KB', 'MB', 'GB', 'TB']: - if size < 1024: - return f"{size:.2f} {unit}" - size /= 1024 - return f"{size:.2f} PB" - - # ==================== 统计和同步 ==================== - - def get_stats(self) -> dict: - """获取数据库统计""" - with self.session() as session: - return { - 'total_files': session.query(FileRecord).filter( - FileRecord.is_deleted == False - ).count(), - 'deleted_files': session.query(FileRecord).filter( - FileRecord.is_deleted == True - ).count(), - 'total_sync_records': session.query(SyncRecord).count(), - 'total_cache_records': session.query(CacheRecord).count(), - 'total_download_records': session.query(DownloadRecord).count(), - 'pending_operations': self._operation_count - } - - def get_download_stats(self, limit: int = 10000) -> dict: - """获取下载统计数据（返回字典，避免会话关闭后访问对象问题）""" - with self.session() as session: - stats = {} - # 直接查询需要的字段，在会话关闭前提取数据 - results = session.query(FileRecord.path, FileRecord.download_count).filter( - FileRecord.is_deleted == False, - FileRecord.download_count > 0 - ).limit(limit).all() - for path, count in results: - stats[path] = count - return stats - - def get_pending_operations(self) -> int: - """获取待同步操作数""" - return self._operation_count - - def reset_pending_count(self): - """重置待同步计数""" - self._operation_count = 0 - self._last_sync_time = time.time() - - def cleanup_expired_cache(self) -> int: - """清理过期缓存记录""" - with self.session() as session: - expired = session.query(CacheRecord).filter( - CacheRecord.expires_at != None, - CacheRecord.expires_at < time.time() - ).delete() - return expired - - # ==================== 用户管理 ==================== - - def create_user(self, username: str, password_hash: str, role: str = 'admin', email: str = None) -> dict: - """创建用户""" - with self.session() as session: - # 检查用户名是否已存在 - existing = session.query(UserRecord).filter_by(username=username).first() - if existing: - return {'success': False, 'error': '用户名已存在'} - - user = UserRecord( - username=username, - password_hash=password_hash, - role=role, - email=email, - enabled=True - ) - session.add(user) - session.commit() - return {'success': True, 'user_id': user.id} - - def get_user(self, username: str) -> dict: - """获取用户信息""" - with self.session() as session: - user = session.query(UserRecord).filter_by(username=username).first() - if user: - return user.to_dict() - return None - - def get_user_by_id(self, user_id: int) -> dict: - """通过ID获取用户信息""" - with self.session() as session: - user = session.query(UserRecord).filter_by(id=user_id).first() - if user: - return user.to_dict() - return None - - def update_password(self, username: str, new_password_hash: str) -> bool: - """更新用户密码""" - with self.session() as session: - user = session.query(UserRecord).filter_by(username=username).first() - if user: - user.password_hash = new_password_hash - user.updated_at = time.time() - session.commit() - return True - return False - - def verify_user(self, username: str, password: str) -> dict: - """验证用户登录（password 为明文密码，用 bcrypt 校验）""" - with self.session() as session: - user = session.query(UserRecord).filter_by(username=username).first() - - if not user: - return {'valid': False, 'reason': '用户不存在'} - - # 检查是否被锁定 - if user.locked_until and user.locked_until > time.time(): - return {'valid': False, 'reason': '账号已被锁定', 'locked_until': user.locked_until} - - # 检查是否启用 - if not user.enabled: - return {'valid': False, 'reason': '账号已被禁用'} - - # 验证密码（bcrypt.checkpw，恒定时间比较） - if self.verify_password(password, user.password_hash): - # 登录成功 - user.last_login = time.time() - user.login_count = (user.login_count or 0) + 1 - user.failed_attempts = 0 - user.locked_until = None - session.commit() - return {'valid': True, 'user': user.to_dict()} - else: - # 登录失败 - user.failed_attempts = (user.failed_attempts or 0) + 1 - - # 连续失败5次锁定10分钟 - if user.failed_attempts >= 5: - user.locked_until = time.time() + 600 # 10分钟 - session.commit() - return {'valid': False, 'reason': '密码错误次数过多，账号已锁定10分钟'} - - session.commit() - return {'valid': False, 'reason': '用户名或密码错误'} - - def add_login_log(self, username: str, ip_address: str, status: str, reason: str = None, user_agent: str = None): - """添加登录日志""" - with self.session() as session: - log = LoginLogRecord( - username=username, - ip_address=ip_address, - user_agent=user_agent, - status=status, - reason=reason - ) - session.add(log) - session.commit() - - def get_login_logs(self, limit: int = 100, username: str = None) -> list: - """获取登录日志""" - with self.session() as session: - query = session.query(LoginLogRecord).order_by(LoginLogRecord.created_at.desc()) - if username: - query = query.filter(LoginLogRecord.username == username) - logs = query.limit(limit).all() - return [log.to_dict() for log in logs] - - def init_default_user(self, username: str, password: str): - """初始化默认用户（如果不存在）""" - # 使用 bcrypt 加密密码 - password_hash = self.hash_password(password) - - user = self.get_user(username) - if not user: - self.create_user(username, password_hash, 'admin') - print(f"[数据库] 已创建默认用户: {username}") - - def hash_password(self, password: str) -> str: - """使用 bcrypt 加密密码（回退: 自描述格式 PBKDF2-HMAC-SHA256）""" - try: - import bcrypt - salt = bcrypt.gensalt(rounds=12) - return bcrypt.hashpw(password.encode('utf-8'), salt).decode('utf-8') - except ImportError: - # 如果 bcrypt 不可用，回退到 PBKDF2-HMAC-SHA256 - # 格式: $pbkdf2-sha256$$$ - import hashlib - import os as _os - import base64 - iterations = 100_000 - salt = _os.urandom(16) - dk = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, iterations) - return (f"$pbkdf2-sha256${iterations}${base64.b64encode(salt).decode()}" - f"${base64.b64encode(dk).decode()}") - - def verify_password(self, password: str, password_hash: str) -> bool: - """验证密码是否正确""" - try: - import bcrypt - return bcrypt.checkpw(password.encode('utf-8'), password_hash.encode('utf-8')) - except ImportError: - # 回退: 只接受自描述 PBKDF2 格式，绝不接受明文 - import hashlib - import base64 - import hmac - if not password_hash.startswith('$pbkdf2-sha256$'): - return False - try: - _, algo, iters_s, salt_b64, hash_b64 = password_hash.split('$') - iterations = int(iters_s) - salt = base64.b64decode(salt_b64) - expected = base64.b64decode(hash_b64) - dk = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, iterations) - return hmac.compare_digest(dk, expected) - except Exception: - return False - - def get_user_by_token(self, token: str) -> dict: - """通过 token 获取用户信息""" - if not token: - return None - with self.session() as session: - from sqlalchemy import or_ - # 兼容旧格式：config_auth - if token == 'config_auth': - return None - user = session.query(UserRecord).filter( - UserRecord.token == token, - UserRecord.enabled == True - ).first() - if user: - return user.to_dict() - return None - - def update_user_token(self, username: str, token: str, expires_at: float = None) -> bool: - """更新用户的 token""" - with self.session() as session: - user = session.query(UserRecord).filter_by(username=username).first() - if user: - user.token = token - user.token_expires_at = expires_at - user.updated_at = time.time() - session.commit() - return True - return False - - def clear_user_token(self, username: str) -> bool: - """清除用户的 token""" - with self.session() as session: - user = session.query(UserRecord).filter_by(username=username).first() - if user: - user.token = None - user.token_expires_at = None - user.updated_at = time.time() - session.commit() - return True - return False - - # ==================== 数据库迁移和健康检查 ==================== - - def check_schema_version(self) -> bool: - """检查并更新数据库结构版本""" - try: - # 确保所有表都存在（包括新增的表） - Base.metadata.create_all(self.engine) - - with self.session() as session: - # 检查版本表是否存在 - from sqlalchemy import inspect - inspector = inspect(self.engine) - tables = inspector.get_table_names() - - if 'schema_versions' not in tables: - # 首次运行，创建版本表并记录当前版本 - version = SchemaVersion(version=_SCHEMA_VERSION, description="Initial schema") - session.add(version) - return True - - # 检查当前版本 - current = session.query(SchemaVersion).filter( - SchemaVersion.version == _SCHEMA_VERSION - ).first() - - if not current: - # 需要迁移 - return self._run_migrations(session) - - return True - - except Exception as e: - print(f"检查数据库版本失败: {e}") - return False - - def _run_migrations(self, session) -> bool: - """运行数据库迁移""" - try: - # 获取当前数据库中的最高版本 - latest = session.query(SchemaVersion).order_by( - SchemaVersion.version.desc() - ).first() - - current_version = latest.version if latest else 0 - - # 按版本运行迁移 - migrations = [ - (1, self._migrate_to_v1), - # 未来版本添加在这里 - # (2, self._migrate_to_v2), - ] - - for version, migration_func in migrations: - if version > current_version: - print(f"运行数据库迁移到版本 {version}...") - migration_func(session) - - # 记录迁移 - new_version = SchemaVersion( - version=version, - description=migration_func.__doc__ or f"Migration to v{version}" - ) - session.add(new_version) - - return True - - except Exception as e: - print(f"数据库迁移失败: {e}") - session.rollback() - return False - - def _migrate_to_v1(self, session): - """v1 迁移 - 初始Schema""" - # 初始Schema无需额外操作 - pass - - def health_check(self) -> dict: - """数据库健康检查""" - try: - with self.session() as session: - # 测试连接 - session.execute(text("SELECT 1")) - - # 获取表信息 - from sqlalchemy import inspect - inspector = inspect(self.engine) - tables = inspector.get_table_names() - - return { - 'healthy': True, - 'db_type': self.db_type, - 'tables': tables, - 'table_count': len(tables), - 'schema_version': _SCHEMA_VERSION - } - - except Exception as e: - return { - 'healthy': False, - 'error': str(e), - 'db_type': self.db_type - } - - def get_table_info(self) -> dict: - """获取所有表的信息""" - try: - from sqlalchemy import inspect - inspector = inspect(self.engine) - - info = {} - for table_name in inspector.get_table_names(): - try: - columns = inspector.get_columns(table_name) - indexes = inspector.get_indexes(table_name) - info[table_name] = { - 'columns': len(columns), - 'indexes': len(indexes), - 'column_names': [c['name'] for c in columns] - } - except Exception: - info[table_name] = {'error': '无法读取表信息'} - - return info - - except Exception as e: - return {'error': str(e)} - - def vacuum(self): - """清理数据库 - 仅 SQLite 支持""" - if self.db_type == 'sqlite': - from sqlalchemy import text - with self.session() as session: - session.execute(text("VACUUM")) - session.commit() - - def recreate_tables(self): - """重建所有表（危险操作！会清空数据）""" - print("警告: 即将重建所有数据库表，这将删除所有数据！") - confirm = input("输入 'yes' 确认: ") - if confirm != 'yes': - print("操作已取消") - return - - with self.session() as session: - # 删除所有表 - Base.metadata.drop_all(self.engine) - session.commit() - - # 重新创建 - self._create_tables() - print("数据库表已重建") - - -# ==================== 数据库工具函数 ==================== - -__all__ = [ - 'DatabaseManager', - 'FileRecord', - 'SyncRecord', - 'CacheRecord', - 'DownloadRecord', - 'SchemaVersion', - 'get_db', - 'init_database', - 'load_db_config_from_env', - 'merge_config', - 'set_table_prefix', - 'get_table_name', -] - - -def get_db(config: dict = None) -> DatabaseManager: - """获取数据库单例""" - return DatabaseManager(config) - - -def init_database(config: dict) -> DatabaseManager: - """初始化数据库""" - db = get_db(config) - return db +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +数据库模块 +提供本地文件和数据库的双存储支持 +""" + +import os +import json +import time +import threading +from datetime import datetime +from typing import Dict, List, Optional, Any +from dataclasses import dataclass, field +from contextlib import contextmanager + +from sqlalchemy import create_engine, Column, Integer, String, Float, Boolean, DateTime, Text, BigInteger, Index, text +from sqlalchemy.orm import sessionmaker, declarative_base +from sqlalchemy.pool import QueuePool + +Base = declarative_base() + + +# ==================== 环境变量支持 ==================== + +def load_db_config_from_env() -> dict: + """ + 从环境变量加载数据库配置 + 支持的变量: + DB_TYPE: 数据库类型 (sqlite/postgresql/mysql/external) + DB_PATH: SQLite 数据库路径 + DB_HOST: 数据库主机地址 + DB_PORT: 数据库端口 + DB_NAME: 数据库名 + DB_USER: 数据库用户 + DB_PASS: 数据库密码 + DB_CONN_STR: 完整连接字符串 (用于外部数据库) + DB_TABLE_PREFIX: 表前缀 + """ + config = {} + + db_type = os.environ.get('DB_TYPE', '').lower() + if db_type: + config['type'] = db_type + + # SQLite + db_path = os.environ.get('DB_PATH', '') + if db_path: + config['sqlite'] = {'path': db_path} + + # PostgreSQL / MySQL 通用 + db_host = os.environ.get('DB_HOST', '') + if db_host: + if 'postgresql' in db_type: + config['postgresql'] = { + 'host': db_host, + 'port': int(os.environ.get('DB_PORT', 5432)), + 'database': os.environ.get('DB_NAME', 'hyc'), + 'user': os.environ.get('DB_USER', 'postgres'), + 'password': os.environ.get('DB_PASS', '') + } + elif 'mysql' in db_type: + config['mysql'] = { + 'host': db_host, + 'port': int(os.environ.get('DB_PORT', 3306)), + 'database': os.environ.get('DB_NAME', 'hyc'), + 'user': os.environ.get('DB_USER', 'root'), + 'password': os.environ.get('DB_PASS', '') + } + + # 外部数据库连接字符串 + db_conn_str = os.environ.get('DB_CONN_STR', '') + if db_conn_str: + config['external'] = {'connection_string': db_conn_str} + + # 表前缀 + table_prefix = os.environ.get('DB_TABLE_PREFIX', '') + if table_prefix: + # 添加到对应类型的配置 + for db_key in ['sqlite', 'postgresql', 'mysql', 'external']: + if db_key in config: + config[db_key]['table_prefix'] = table_prefix + + return config + + +def merge_config(file_config: dict, env_config: dict) -> dict: + """合并配置文件和环境变量配置""" + merged = file_config.copy() + + # 如果有环境变量配置，合并 database 部分 + if 'database' in env_config and env_config['database']: + if 'database' not in merged: + merged['database'] = {} + merged['database'].update(env_config['database']) + + return merged + + +# ==================== 数据库模型定义 ==================== + +# 表名前缀（可在初始化时设置） +_TABLE_PREFIX = '' +_SCHEMA_VERSION = 1 # 当前Schema版本 + + +def set_table_prefix(prefix: str): + """设置表前缀""" + global _TABLE_PREFIX + _TABLE_PREFIX = prefix + + +def get_table_name(base_name: str) -> str: + """获取带前缀的表名""" + return f"{_TABLE_PREFIX}{base_name}" + + +class SchemaVersion(Base): + """数据库结构版本表""" + __tablename__ = 'schema_versions' + + id = Column(Integer, primary_key=True, autoincrement=True) + version = Column(Integer, nullable=False, unique=True) + applied_at = Column(Float, default=time.time) + description = Column(String(255), nullable=True) + + +class FileRecord(Base): + """文件记录表""" + __tablename__ = 'files' + + id = Column(Integer, primary_key=True, autoincrement=True) + file_id = Column(String(64), unique=True, nullable=False, index=True) # 文件唯一ID + path = Column(String(1024), nullable=False, index=True) # 文件路径 + name = Column(String(512), nullable=False) # 文件名 + size = Column(BigInteger, default=0) # 文件大小 + hash = Column(String(64), nullable=True) # 文件hash + mime_type = Column(String(128), nullable=True) # MIME类型 + is_dir = Column(Boolean, default=False) # 是否是目录 + created_at = Column(Float, default=time.time) # 创建时间 + updated_at = Column(Float, default=time.time) # 更新时间 + last_accessed = Column(Float, default=time.time) # 最后访问时间 + download_count = Column(Integer, default=0) # 下载次数 + is_deleted = Column(Boolean, default=False, index=True) # 软删除标记 + sync_status = Column(String(32), default='synced') # 同步状态: pending, synced, error + + __table_args__ = ( + Index('idx_files_path_status', 'path', 'is_deleted'), + ) + + def to_dict(self) -> dict: + """转换为字典""" + return { + 'id': self.id, + 'file_id': self.file_id, + 'path': self.path, + 'name': self.name, + 'size': self.size, + 'hash': self.hash, + 'mime_type': self.mime_type, + 'is_dir': self.is_dir, + 'created_at': self.created_at, + 'updated_at': self.updated_at, + 'last_accessed': self.last_accessed, + 'download_count': self.download_count, + 'is_deleted': self.is_deleted, + 'sync_status': self.sync_status + } + + +class SyncRecord(Base): + """同步记录表""" + __tablename__ = 'sync_records' + + id = Column(Integer, primary_key=True, autoincrement=True) + sync_id = Column(String(64), unique=True, nullable=False, index=True) + source_type = Column(String(64), nullable=False) # 同步源类型 + source_name = Column(String(256), nullable=False) # 同步源名称 + status = Column(String(32), default='pending') # pending, running, completed, failed + total_files = Column(Integer, default=0) + synced_files = Column(Integer, default=0) + failed_files = Column(Integer, default=0) + total_size = Column(BigInteger, default=0) + synced_size = Column(BigInteger, default=0) + started_at = Column(Float, default=time.time) + completed_at = Column(Float, nullable=True) + error_message = Column(Text, nullable=True) + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'sync_id': self.sync_id, + 'source_type': self.source_type, + 'source_name': self.source_name, + 'status': self.status, + 'total_files': self.total_files, + 'synced_files': self.synced_files, + 'failed_files': self.failed_files, + 'total_size': self.total_size, + 'synced_size': self.synced_size, + 'started_at': self.started_at, + 'completed_at': self.completed_at, + 'error_message': self.error_message + } + + +class CacheRecord(Base): + """缓存记录表""" + __tablename__ = 'cache_records' + + id = Column(Integer, primary_key=True, autoincrement=True) + cache_key = Column(String(512), unique=True, nullable=False, index=True) + cache_type = Column(String(64), nullable=False) # docker, apt, pypi, etc. + file_path = Column(String(1024), nullable=True) + file_size = Column(BigInteger, default=0) + file_hash = Column(String(64), nullable=True) + hits = Column(Integer, default=0) + created_at = Column(Float, default=time.time) + expires_at = Column(Float, nullable=True) + last_hit = Column(Float, default=time.time) + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'cache_key': self.cache_key, + 'cache_type': self.cache_type, + 'file_path': self.file_path, + 'file_size': self.file_size, + 'file_hash': self.file_hash, + 'hits': self.hits, + 'created_at': self.created_at, + 'expires_at': self.expires_at, + 'last_hit': self.last_hit + } + + +class DownloadRecord(Base): + """下载记录表""" + __tablename__ = 'download_records' + + id = Column(Integer, primary_key=True, autoincrement=True) + file_path = Column(String(1024), nullable=False, index=True) + file_size = Column(BigInteger, default=0) + download_time = Column(Float, default=time.time) + duration = Column(Float, default=0) + client_ip = Column(String(64), nullable=True) + user_agent = Column(String(512), nullable=True) + success = Column(Boolean, default=True) + error_message = Column(Text, nullable=True) + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'file_path': self.file_path, + 'file_size': self.file_size, + 'download_time': self.download_time, + 'duration': self.duration, + 'client_ip': self.client_ip, + 'user_agent': self.user_agent, + 'success': self.success, + 'error_message': self.error_message + } + + +class MonitorHistoryRecord(Base): + """监控历史记录表""" + __tablename__ = 'monitor_history' + + id = Column(Integer, primary_key=True, autoincrement=True) + timestamp = Column(Float, nullable=False, index=True) + cpu_percent = Column(Float, default=0) + memory_percent = Column(Float, default=0) + disk_percent = Column(Float, default=0) + network_rx = Column(BigInteger, default=0) + network_tx = Column(BigInteger, default=0) + active_connections = Column(Integer, default=0) + server_uptime = Column(Float, default=0) + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'timestamp': self.timestamp, + 'cpu_percent': self.cpu_percent, + 'memory_percent': self.memory_percent, + 'disk_percent': self.disk_percent, + 'network_rx': self.network_rx, + 'network_tx': self.network_tx, + 'active_connections': self.active_connections, + 'server_uptime': self.server_uptime + } + + +class WebhookRecord(Base): + """Webhook配置记录表""" + __tablename__ = 'webhooks' + + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(String(255), nullable=False) + url = Column(String(2048), nullable=False) + events = Column(Text, nullable=True) # JSON 格式存储 + secret = Column(String(255), nullable=True) + enabled = Column(Boolean, default=True) + created_at = Column(Float, default=time.time) + updated_at = Column(Float, default=time.time) + + def to_dict(self) -> dict: + import json + return { + 'id': self.id, + 'name': self.name, + 'url': self.url, + 'events': json.loads(self.events) if self.events else [], + 'secret': self.secret, + 'enabled': self.enabled, + 'created_at': self.created_at, + 'updated_at': self.updated_at + } + + +class WebhookDeliveryRecord(Base): + """Webhook交付记录表""" + __tablename__ = 'webhook_deliveries' + + id = Column(Integer, primary_key=True, autoincrement=True) + webhook_id = Column(Integer, nullable=False, index=True) + event = Column(String(100), nullable=False) + status = Column(String(50), nullable=False) # success, failed, pending + status_code = Column(Integer, nullable=True) + response_body = Column(Text, nullable=True) + error_message = Column(Text, nullable=True) + duration_ms = Column(Float, nullable=True) + created_at = Column(Float, default=time.time, index=True) + retry_count = Column(Integer, default=0) + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'webhook_id': self.webhook_id, + 'event': self.event, + 'status': self.status, + 'status_code': self.status_code, + 'response_body': self.response_body[:500] if self.response_body else None, + 'error_message': self.error_message, + 'duration_ms': self.duration_ms, + 'created_at': self.created_at, + 'retry_count': self.retry_count + } + + +class UserRecord(Base): + """用户账号记录表""" + __tablename__ = 'users' + + id = Column(Integer, primary_key=True, autoincrement=True) + username = Column(String(100), nullable=False, unique=True, index=True) + password_hash = Column(String(255), nullable=False) # bcrypt 加密后的哈希 + token = Column(String(64), nullable=True) # 登录生成的 token + token_expires_at = Column(Float, nullable=True) # token 过期时间 + role = Column(String(50), default='admin') # admin, user + email = Column(String(255), nullable=True) + phone = Column(String(50), nullable=True) + last_login = Column(Float, nullable=True) + login_count = Column(Integer, default=0) + failed_attempts = Column(Integer, default=0) # 登录失败次数 + locked_until = Column(Float, nullable=True) # 锁定直到时间戳 + created_at = Column(Float, default=time.time) + updated_at = Column(Float, default=time.time) + enabled = Column(Boolean, default=True) + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'username': self.username, + 'password_hash': self.password_hash, + 'token': self.token, + 'token_expires_at': self.token_expires_at, + 'role': self.role, + 'email': self.email, + 'phone': self.phone, + 'last_login': self.last_login, + 'login_count': self.login_count, + 'failed_attempts': self.failed_attempts, + 'locked_until': self.locked_until, + 'created_at': self.created_at, + 'updated_at': self.updated_at, + 'enabled': self.enabled + } + + +class LoginLogRecord(Base): + """登录日志记录表""" + __tablename__ = 'login_logs' + + id = Column(Integer, primary_key=True, autoincrement=True) + username = Column(String(100), nullable=False, index=True) + ip_address = Column(String(50), nullable=True) + user_agent = Column(String(500), nullable=True) + status = Column(String(20), nullable=False) # success, failed, locked + reason = Column(String(255), nullable=True) + created_at = Column(Float, default=time.time, index=True) + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'username': self.username, + 'ip_address': self.ip_address, + 'user_agent': self.user_agent, + 'status': self.status, + 'reason': self.reason, + 'created_at': self.created_at + } + + +# ==================== 数据库管理器 ==================== + +class DatabaseManager: + """数据库管理器""" + + _instance = None + _lock = threading.Lock() + + def __new__(cls, config: dict = None): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self, config: dict = None): + if self._initialized: + return + + self.config = config or {} + self.db_type = self.config.get('type', 'sqlite') # sqlite, postgresql, mysql, external + + # 获取数据库配置 + db_config = self.config.get(self.db_type, {}) + + # 获取表前缀（支持多租户） + self.table_prefix = db_config.get('table_prefix', '') + + # 创建数据库引擎 + self._create_engine(db_config) + + # 设置表前缀（需要在创建表之前） + if self.table_prefix: + set_table_prefix(self.table_prefix) + + # 自动创建表结构 + self._create_tables() + + # 会话工厂 + self.Session = sessionmaker(bind=self.engine) + + self._initialized = True + self._operation_count = 0 + self._last_sync_time = time.time() + self._sync_interval = self.config.get('sync_interval', 60) # 默认60秒同步 + + def _create_tables(self): + """自动创建数据库表结构""" + try: + # 尝试创建所有表 + Base.metadata.create_all(self.engine) + # 对于 MySQL，需要提交事务 + if self.db_type == 'mysql': + from sqlalchemy import text + with self.engine.connect() as conn: + conn.commit() + except Exception as e: + print(f"警告: 创建表结构失败: {e}") + print("将尝试创建数据库...") + self._create_database() + + def _create_database(self): + """创建数据库（如果不存在）""" + # SQLite 不需要预创建数据库 + if self.db_type == 'sqlite': + return + + # 对于 PostgreSQL/MySQL，尝试创建数据库 + try: + if self.db_type == 'postgresql': + # 连接到默认数据库 postgres + from sqlalchemy import text + db_config = self.config.get('postgresql', {}) + host = db_config.get('host', 'localhost') + port = db_config.get('port', 5432) + user = db_config.get('user', 'postgres') + password = db_config.get('password', '') + database = db_config.get('database', 'hyc') + + temp_engine = create_engine( + f"postgresql://{user}:{password}@{host}:{port}/postgres" + ) + with temp_engine.connect() as conn: + # 检查数据库是否存在 + result = conn.execute( + text(f"SELECT 1 FROM pg_database WHERE datname = '{database}'") + ).fetchone() + if not result: + conn.execute(text(f"CREATE DATABASE {database}")) + print(f"已创建数据库: {database}") + temp_engine.dispose() + + elif self.db_type == 'mysql': + from sqlalchemy import text + db_config = self.config.get('mysql', {}) + host = db_config.get('host', 'localhost') + port = db_config.get('port', 3306) + user = db_config.get('user', 'root') + password = db_config.get('password', '') + + temp_engine = create_engine( + f"mysql+pymysql://{user}:{password}@{host}:{port}" + ) + with temp_engine.connect() as conn: + db_config = self.config.get('mysql', {}) + database = db_config.get('database', 'hyc') + try: + conn.execute(text(f"CREATE DATABASE IF NOT EXISTS {database} CHARACTER SET utf8mb4")) + conn.commit() # 提交事务 + print(f"已创建/确认数据库: {database}") + except Exception as e: + print(f"警告: 创建数据库失败: {e}") + temp_engine.dispose() + + except Exception as e: + print(f"警告: 创建数据库失败: {e}") + + def _create_engine(self, db_config: dict): + """创建数据库引擎""" + if self.db_type == 'sqlite': + # SQLite 配置 + db_path = db_config.get('path', './data/hyc.db') + # 确保数据库目录存在（路径无目录部分时跳过） + db_dir = os.path.dirname(db_path) + if db_dir: + os.makedirs(db_dir, exist_ok=True) + + db_url = f"sqlite:///{db_path}" + self.engine = create_engine( + db_url, + poolclass=QueuePool, + pool_size=5, + max_overflow=10, + pool_recycle=3600, + connect_args={'timeout': 30} # 忙等待，减少 database is locked + ) + + # 启用 WAL 提升并发读写；sqlite3 是标准库，始终可用 + @self.engine.event.listens_for(self.engine, 'connect') + def _set_sqlite_pragma(dbapi_connection, connection_record): + try: + cursor = dbapi_connection.cursor() + cursor.execute('PRAGMA journal_mode=WAL') + cursor.execute('PRAGMA busy_timeout=5000') + cursor.execute('PRAGMA synchronous=NORMAL') + cursor.close() + except Exception: + pass + + elif self.db_type == 'postgresql': + # PostgreSQL 配置 + host = db_config.get('host', 'localhost') + port = db_config.get('port', 5432) + database = db_config.get('database', 'hyc') + user = db_config.get('user', 'postgres') + password = db_config.get('password', '') + ssl_mode = db_config.get('ssl_mode', 'prefer') + timeout = db_config.get('connection_timeout', 30) + + db_url = f"postgresql://{user}:{password}@{host}:{port}/{database}?sslmode={ssl_mode}" + self.engine = create_engine( + db_url, + poolclass=QueuePool, + pool_size=db_config.get('pool_size', 5), + max_overflow=db_config.get('max_overflow', 10), + pool_recycle=3600, + connect_args={'connect_timeout': timeout} + ) + + elif self.db_type == 'mysql': + # MySQL 配置 + host = db_config.get('host', 'localhost') + port = db_config.get('port', 3306) + database = db_config.get('database', 'hyc') + user = db_config.get('user', 'root') + password = db_config.get('password', '') + charset = db_config.get('charset', 'utf8mb4') + timeout = db_config.get('connection_timeout', 30) + + db_url = f"mysql+pymysql://{user}:{password}@{host}:{port}/{database}?charset={charset}" + self.engine = create_engine( + db_url, + poolclass=QueuePool, + pool_size=db_config.get('pool_size', 5), + max_overflow=db_config.get('max_overflow', 10), + pool_recycle=3600, + connect_args={'connect_timeout': timeout} + ) + + elif self.db_type == 'external': + # 外部数据库 - 使用完整连接字符串 + connection_string = db_config.get('connection_string', '') + if not connection_string: + raise ValueError("外部数据库配置需要提供 connection_string") + + self.engine = create_engine( + connection_string, + poolclass=QueuePool, + pool_size=db_config.get('pool_size', 5), + max_overflow=db_config.get('max_overflow', 10), + pool_recycle=3600 + ) + + else: + raise ValueError(f"不支持的数据库类型: {self.db_type}. 支持: sqlite, postgresql, mysql, external") + + @contextmanager + def session(self): + """获取数据库会话""" + session = self.Session() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + # ==================== 文件操作 ==================== + + def add_file(self, file_id: str, path: str, name: str, + size: int = 0, hash: str = None, mime_type: str = None, + is_dir: bool = False, created_at: float = None) -> FileRecord: + """添加文件记录""" + with self.session() as session: + record = FileRecord( + file_id=file_id, + path=path, + name=name, + size=size, + hash=hash, + mime_type=mime_type, + is_dir=is_dir, + created_at=created_at or time.time(), + updated_at=time.time(), + sync_status='pending' + ) + session.add(record) + self._operation_count += 1 + return record + + def update_file(self, file_id: str, **kwargs) -> Optional[FileRecord]: + """更新文件记录""" + with self.session() as session: + record = session.query(FileRecord).filter( + FileRecord.file_id == file_id, + FileRecord.is_deleted == False + ).first() + if record: + for key, value in kwargs.items(): + if hasattr(record, key): + setattr(record, key, value) + record.updated_at = time.time() + record.sync_status = 'pending' + self._operation_count += 1 + return record + + def delete_file(self, file_id: str, hard: bool = False) -> bool: + """删除文件记录""" + with self.session() as session: + record = session.query(FileRecord).filter( + FileRecord.file_id == file_id + ).first() + if record: + if hard: + session.delete(record) + else: + record.is_deleted = True + record.updated_at = time.time() + record.sync_status = 'pending' + self._operation_count += 1 + return True + return False + + def get_file(self, file_id: str) -> Optional[FileRecord]: + """获取文件记录""" + with self.session() as session: + return session.query(FileRecord).filter( + FileRecord.file_id == file_id, + FileRecord.is_deleted == False + ).first() + + def get_file_by_path(self, path: str) -> Optional[FileRecord]: + """根据路径获取文件记录""" + with self.session() as session: + return session.query(FileRecord).filter( + FileRecord.path == path, + FileRecord.is_deleted == False + ).first() + + def list_files(self, path: str = '/', recursive: bool = False, + include_deleted: bool = False, limit: int = 1000, + offset: int = 0) -> List[FileRecord]: + """列出文件记录""" + with self.session() as session: + query = session.query(FileRecord) + + if not include_deleted: + query = query.filter(FileRecord.is_deleted == False) + + if path and path != '/': + if recursive: + query = query.filter(FileRecord.path.startswith(path)) + else: + parent_path = path.rstrip('/') + '/' + query = query.filter( + (FileRecord.path == path) | + (FileRecord.path.startswith(parent_path)) + ) + + return query.order_by(FileRecord.path).offset(offset).limit(limit).all() + + def search_files(self, keyword: str, limit: int = 100) -> List[FileRecord]: + """搜索文件""" + with self.session() as session: + return session.query(FileRecord).filter( + FileRecord.is_deleted == False, + (FileRecord.name.contains(keyword) | + FileRecord.path.contains(keyword)) + ).limit(limit).all() + + def increment_download_count(self, file_id: str) -> bool: + """增加下载计数（原子 UPDATE，避免并发读改写丢失更新）""" + with self.session() as session: + from sqlalchemy import update + result = session.execute( + update(FileRecord) + .where(FileRecord.file_id == file_id) + .values(download_count=FileRecord.download_count + 1) + ) + if result.rowcount > 0: + self._operation_count += 1 + return True + return False + + # ==================== 同步记录操作 ==================== + + def add_sync_record(self, sync_id: str, source_type: str, + source_name: str, **kwargs) -> SyncRecord: + """添加同步记录""" + with self.session() as session: + record = SyncRecord( + sync_id=sync_id, + source_type=source_type, + source_name=source_name, + started_at=time.time(), + **kwargs + ) + session.add(record) + return record + + def update_sync_record(self, sync_id: str, **kwargs) -> Optional[SyncRecord]: + """更新同步记录""" + with self.session() as session: + record = session.query(SyncRecord).filter( + SyncRecord.sync_id == sync_id + ).first() + if record: + for key, value in kwargs.items(): + if hasattr(record, key): + setattr(record, key, value) + return record + return None + + def get_sync_records(self, limit: int = 50) -> List[SyncRecord]: + """获取同步记录""" + with self.session() as session: + return session.query(SyncRecord).order_by( + SyncRecord.started_at.desc() + ).limit(limit).all() + + # ==================== 缓存记录操作 ==================== + + def add_cache_record(self, cache_key: str, cache_type: str, + **kwargs) -> CacheRecord: + """添加缓存记录""" + with self.session() as session: + record = CacheRecord( + cache_key=cache_key, + cache_type=cache_type, + **kwargs + ) + session.add(record) + return record + + def update_cache_record(self, cache_key: str, **kwargs) -> Optional[CacheRecord]: + """更新缓存记录""" + with self.session() as session: + record = session.query(CacheRecord).filter( + CacheRecord.cache_key == cache_key + ).first() + if record: + for key, value in kwargs.items(): + if hasattr(record, key): + setattr(record, key, value) + return record + return None + + def get_cache_record(self, cache_key: str) -> Optional[CacheRecord]: + """获取缓存记录""" + with self.session() as session: + return session.query(CacheRecord).filter( + CacheRecord.cache_key == cache_key + ).first() + + def increment_cache_hits(self, cache_key: str) -> bool: + """增加缓存命中次数""" + with self.session() as session: + record = session.query(CacheRecord).filter( + CacheRecord.cache_key == cache_key + ).first() + if record: + record.hits += 1 + record.last_hit = time.time() + return True + return False + + def list_cache_records(self, cache_type: str = None, + limit: int = 100) -> List[CacheRecord]: + """列出缓存记录""" + with self.session() as session: + query = session.query(CacheRecord) + if cache_type: + query = query.filter(CacheRecord.cache_type == cache_type) + return query.order_by(CacheRecord.hits.desc()).limit(limit).all() + + # ==================== 下载记录操作 ==================== + + def add_download_record(self, file_path: str, file_size: int = 0, + client_ip: str = None, user_agent: str = None, + success: bool = True, error_message: str = None, + duration: float = 0) -> DownloadRecord: + """添加下载记录""" + with self.session() as session: + record = DownloadRecord( + file_path=file_path, + file_size=file_size, + download_time=time.time(), + client_ip=client_ip, + user_agent=user_agent, + success=success, + error_message=error_message, + duration=duration + ) + session.add(record) + self._operation_count += 1 + return record + + def get_download_records(self, file_path: str = None, + limit: int = 100) -> List[DownloadRecord]: + """获取下载记录""" + with self.session() as session: + query = session.query(DownloadRecord) + if file_path: + query = query.filter(DownloadRecord.file_path == file_path) + return query.order_by( + DownloadRecord.download_time.desc() + ).limit(limit).all() + + def get_download_stats(self, days: int = 7) -> dict: + """获取下载统计""" + from sqlalchemy import func + start_time = time.time() - (days * 86400) + + with self.session() as session: + total_downloads = session.query(func.count(DownloadRecord.id)).filter( + DownloadRecord.download_time >= start_time + ).scalar() + + successful_downloads = session.query(func.count(DownloadRecord.id)).filter( + DownloadRecord.download_time >= start_time, + DownloadRecord.success == True + ).scalar() + + total_bytes = session.query(func.sum(DownloadRecord.file_size)).filter( + DownloadRecord.download_time >= start_time, + DownloadRecord.success == True + ).scalar() or 0 + + return { + 'total_downloads': total_downloads, + 'successful_downloads': successful_downloads, + 'failed_downloads': total_downloads - successful_downloads, + 'total_bytes': total_bytes, + 'total_human': self._format_size(total_bytes) + } + + # ==================== 监控历史记录 ==================== + + def add_monitor_record(self, cpu_percent: float = 0, memory_percent: float = 0, + disk_percent: float = 0, network_rx: int = 0, + network_tx: int = 0, active_connections: int = 0, + server_uptime: float = 0) -> Optional[MonitorHistoryRecord]: + """添加监控记录""" + with self.session() as session: + record = MonitorHistoryRecord( + timestamp=time.time(), + cpu_percent=cpu_percent, + memory_percent=memory_percent, + disk_percent=disk_percent, + network_rx=network_rx, + network_tx=network_tx, + active_connections=active_connections, + server_uptime=server_uptime + ) + session.add(record) + self._operation_count += 1 + return record + + def get_monitor_history(self, hours: int = 24) -> List[MonitorHistoryRecord]: + """获取监控历史记录""" + from sqlalchemy import func + cutoff_time = time.time() - (hours * 3600) + + with self.session() as session: + # 按时间聚合，每5分钟一个数据点 + records = session.query( + MonitorHistoryRecord + ).filter( + MonitorHistoryRecord.timestamp >= cutoff_time + ).order_by( + MonitorHistoryRecord.timestamp.desc() + ).all() + + return records + + def get_monitor_stats(self, hours: int = 24) -> dict: + """获取监控统计数据""" + from sqlalchemy import func + cutoff_time = time.time() - (hours * 3600) + + with self.session() as session: + stats = session.query( + func.avg(MonitorHistoryRecord.cpu_percent).label('avg_cpu'), + func.max(MonitorHistoryRecord.cpu_percent).label('max_cpu'), + func.avg(MonitorHistoryRecord.memory_percent).label('avg_memory'), + func.max(MonitorHistoryRecord.memory_percent).label('max_memory'), + func.avg(MonitorHistoryRecord.disk_percent).label('avg_disk'), + func.max(MonitorHistoryRecord.disk_percent).label('max_disk'), + func.sum(MonitorHistoryRecord.network_rx).label('total_rx'), + func.sum(MonitorHistoryRecord.network_tx).label('total_tx') + ).filter( + MonitorHistoryRecord.timestamp >= cutoff_time + ).first() + + return { + 'avg_cpu': round(stats.avg_cpu, 1) if stats.avg_cpu else 0, + 'max_cpu': round(stats.max_cpu, 1) if stats.max_cpu else 0, + 'avg_memory': round(stats.avg_memory, 1) if stats.avg_memory else 0, + 'max_memory': round(stats.max_memory, 1) if stats.max_memory else 0, + 'avg_disk': round(stats.avg_disk, 1) if stats.avg_disk else 0, + 'max_disk': round(stats.max_disk, 1) if stats.max_disk else 0, + 'total_rx': stats.total_rx or 0, + 'total_tx': stats.total_tx or 0 + } + + # ==================== Webhook 管理 ==================== + + def get_webhooks(self) -> List[WebhookRecord]: + """获取所有 webhook 配置""" + import json + with self.session() as session: + return session.query(WebhookRecord).order_by( + WebhookRecord.created_at.desc() + ).all() + + def get_webhook(self, webhook_id: int) -> Optional[WebhookRecord]: + """获取单个 webhook 配置""" + with self.session() as session: + return session.query(WebhookRecord).filter( + WebhookRecord.id == webhook_id + ).first() + + def add_webhook(self, name: str, url: str, events: List[str] = None, + secret: str = None, enabled: bool = True) -> WebhookRecord: + """添加 webhook 配置""" + import json + with self.session() as session: + record = WebhookRecord( + name=name, + url=url, + events=json.dumps(events or []), + secret=secret, + enabled=enabled, + created_at=time.time(), + updated_at=time.time() + ) + session.add(record) + return record + + def update_webhook(self, webhook_id: int, **kwargs) -> Optional[WebhookRecord]: + """更新 webhook 配置""" + import json + with self.session() as session: + record = session.query(WebhookRecord).filter( + WebhookRecord.id == webhook_id + ).first() + if not record: + return None + + if 'name' in kwargs: + record.name = kwargs['name'] + if 'url' in kwargs: + record.url = kwargs['url'] + if 'events' in kwargs: + record.events = json.dumps(kwargs['events']) + if 'secret' in kwargs: + record.secret = kwargs['secret'] + if 'enabled' in kwargs: + record.enabled = kwargs['enabled'] + + record.updated_at = time.time() + return record + + def delete_webhook(self, webhook_id: int) -> bool: + """删除 webhook 配置""" + with self.session() as session: + result = session.query(WebhookRecord).filter( + WebhookRecord.id == webhook_id + ).delete() + return result > 0 + + # ==================== Webhook 交付记录 ==================== + + def add_webhook_delivery( + self, + webhook_id: int, + event: str, + status: str, + status_code: int = None, + response_body: str = None, + error_message: str = None, + duration_ms: float = None, + retry_count: int = 0 + ) -> WebhookDeliveryRecord: + """添加 webhook 交付记录""" + with self.session() as session: + record = WebhookDeliveryRecord( + webhook_id=webhook_id, + event=event, + status=status, + status_code=status_code, + response_body=response_body, + error_message=error_message, + duration_ms=duration_ms, + created_at=time.time(), + retry_count=retry_count + ) + session.add(record) + session.commit() + return record + + def get_webhook_deliveries( + self, + webhook_id: int = None, + status: str = None, + limit: int = 50 + ) -> List[WebhookDeliveryRecord]: + """获取 webhook 交付记录""" + with self.session() as session: + query = session.query(WebhookDeliveryRecord) + + if webhook_id: + query = query.filter(WebhookDeliveryRecord.webhook_id == webhook_id) + if status: + query = query.filter(WebhookDeliveryRecord.status == status) + + return query.order_by( + WebhookDeliveryRecord.created_at.desc() + ).limit(limit).all() + + def get_webhook_stats(self, webhook_id: int = None) -> Dict: + """获取 webhook 交付统计""" + with self.session() as session: + query = session.query(WebhookDeliveryRecord) + if webhook_id: + query = query.filter(WebhookDeliveryRecord.webhook_id == webhook_id) + + deliveries = query.all() + + total = len(deliveries) + success = sum(1 for d in deliveries if d.status == 'success') + failed = sum(1 for d in deliveries if d.status == 'failed') + + avg_duration = 0 + if deliveries: + durations = [d.duration_ms for d in deliveries if d.duration_ms] + if durations: + avg_duration = sum(durations) / len(durations) + + return { + 'total_deliveries': total, + 'success': success, + 'failed': failed, + 'pending': sum(1 for d in deliveries if d.status == 'pending'), + 'success_rate': (success / total * 100) if total > 0 else 0, + 'avg_duration_ms': round(avg_duration, 2) + } + + def cleanup_webhook_deliveries(self, older_than: int = 604800) -> int: + """ + 清理旧的 webhook 交付记录 + + Args: + older_than: 清理多少秒之前的记录，默认 7 天 (604800 秒) + + Returns: + 删除的记录数量 + """ + with self.session() as session: + cutoff = time.time() - older_than + result = session.query(WebhookDeliveryRecord).filter( + WebhookDeliveryRecord.created_at < cutoff + ).delete() + session.commit() + return result + + @staticmethod + def _format_size(size: int) -> str: + """格式化文件大小""" + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size < 1024: + return f"{size:.2f} {unit}" + size /= 1024 + return f"{size:.2f} PB" + + # ==================== 统计和同步 ==================== + + def get_stats(self) -> dict: + """获取数据库统计""" + with self.session() as session: + return { + 'total_files': session.query(FileRecord).filter( + FileRecord.is_deleted == False + ).count(), + 'deleted_files': session.query(FileRecord).filter( + FileRecord.is_deleted == True + ).count(), + 'total_sync_records': session.query(SyncRecord).count(), + 'total_cache_records': session.query(CacheRecord).count(), + 'total_download_records': session.query(DownloadRecord).count(), + 'pending_operations': self._operation_count + } + + def get_download_stats(self, limit: int = 10000) -> dict: + """获取下载统计数据（返回字典，避免会话关闭后访问对象问题）""" + with self.session() as session: + stats = {} + # 直接查询需要的字段，在会话关闭前提取数据 + results = session.query(FileRecord.path, FileRecord.download_count).filter( + FileRecord.is_deleted == False, + FileRecord.download_count > 0 + ).limit(limit).all() + for path, count in results: + stats[path] = count + return stats + + def get_pending_operations(self) -> int: + """获取待同步操作数""" + return self._operation_count + + def reset_pending_count(self): + """重置待同步计数""" + self._operation_count = 0 + self._last_sync_time = time.time() + + def cleanup_expired_cache(self) -> int: + """清理过期缓存记录""" + with self.session() as session: + expired = session.query(CacheRecord).filter( + CacheRecord.expires_at != None, + CacheRecord.expires_at < time.time() + ).delete() + return expired + + # ==================== 用户管理 ==================== + + def create_user(self, username: str, password_hash: str, role: str = 'admin', email: str = None) -> dict: + """创建用户""" + with self.session() as session: + # 检查用户名是否已存在 + existing = session.query(UserRecord).filter_by(username=username).first() + if existing: + return {'success': False, 'error': '用户名已存在'} + + user = UserRecord( + username=username, + password_hash=password_hash, + role=role, + email=email, + enabled=True + ) + session.add(user) + session.commit() + return {'success': True, 'user_id': user.id} + + def get_user(self, username: str) -> dict: + """获取用户信息""" + with self.session() as session: + user = session.query(UserRecord).filter_by(username=username).first() + if user: + return user.to_dict() + return None + + def get_user_by_id(self, user_id: int) -> dict: + """通过ID获取用户信息""" + with self.session() as session: + user = session.query(UserRecord).filter_by(id=user_id).first() + if user: + return user.to_dict() + return None + + def update_password(self, username: str, new_password_hash: str) -> bool: + """更新用户密码""" + with self.session() as session: + user = session.query(UserRecord).filter_by(username=username).first() + if user: + user.password_hash = new_password_hash + user.updated_at = time.time() + session.commit() + return True + return False + + def verify_user(self, username: str, password: str) -> dict: + """验证用户登录（password 为明文密码，用 bcrypt 校验）""" + with self.session() as session: + user = session.query(UserRecord).filter_by(username=username).first() + + if not user: + return {'valid': False, 'reason': '用户不存在'} + + # 检查是否被锁定 + if user.locked_until and user.locked_until > time.time(): + return {'valid': False, 'reason': '账号已被锁定', 'locked_until': user.locked_until} + + # 检查是否启用 + if not user.enabled: + return {'valid': False, 'reason': '账号已被禁用'} + + # 验证密码（bcrypt.checkpw，恒定时间比较） + if self.verify_password(password, user.password_hash): + # 登录成功 + user.last_login = time.time() + user.login_count = (user.login_count or 0) + 1 + user.failed_attempts = 0 + user.locked_until = None + session.commit() + return {'valid': True, 'user': user.to_dict()} + else: + # 登录失败 + user.failed_attempts = (user.failed_attempts or 0) + 1 + + # 连续失败5次锁定10分钟 + if user.failed_attempts >= 5: + user.locked_until = time.time() + 600 # 10分钟 + session.commit() + return {'valid': False, 'reason': '密码错误次数过多，账号已锁定10分钟'} + + session.commit() + return {'valid': False, 'reason': '用户名或密码错误'} + + def add_login_log(self, username: str, ip_address: str, status: str, reason: str = None, user_agent: str = None): + """添加登录日志""" + with self.session() as session: + log = LoginLogRecord( + username=username, + ip_address=ip_address, + user_agent=user_agent, + status=status, + reason=reason + ) + session.add(log) + session.commit() + + def get_login_logs(self, limit: int = 100, username: str = None) -> list: + """获取登录日志""" + with self.session() as session: + query = session.query(LoginLogRecord).order_by(LoginLogRecord.created_at.desc()) + if username: + query = query.filter(LoginLogRecord.username == username) + logs = query.limit(limit).all() + return [log.to_dict() for log in logs] + + def init_default_user(self, username: str, password: str): + """初始化默认用户（如果不存在）""" + # 使用 bcrypt 加密密码 + password_hash = self.hash_password(password) + + user = self.get_user(username) + if not user: + self.create_user(username, password_hash, 'admin') + print(f"[数据库] 已创建默认用户: {username}") + + def hash_password(self, password: str) -> str: + """使用 bcrypt 加密密码（回退: 自描述格式 PBKDF2-HMAC-SHA256）""" + try: + import bcrypt + salt = bcrypt.gensalt(rounds=12) + return bcrypt.hashpw(password.encode('utf-8'), salt).decode('utf-8') + except ImportError: + # 如果 bcrypt 不可用，回退到 PBKDF2-HMAC-SHA256 + # 格式: $pbkdf2-sha256$$$ + import hashlib + import os as _os + import base64 + iterations = 100_000 + salt = _os.urandom(16) + dk = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, iterations) + return (f"$pbkdf2-sha256${iterations}${base64.b64encode(salt).decode()}" + f"${base64.b64encode(dk).decode()}") + + def verify_password(self, password: str, password_hash: str) -> bool: + """验证密码是否正确""" + try: + import bcrypt + return bcrypt.checkpw(password.encode('utf-8'), password_hash.encode('utf-8')) + except ImportError: + # 回退: 只接受自描述 PBKDF2 格式，绝不接受明文 + import hashlib + import base64 + import hmac + if not password_hash.startswith('$pbkdf2-sha256$'): + return False + try: + _, algo, iters_s, salt_b64, hash_b64 = password_hash.split('$') + iterations = int(iters_s) + salt = base64.b64decode(salt_b64) + expected = base64.b64decode(hash_b64) + dk = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, iterations) + return hmac.compare_digest(dk, expected) + except Exception: + return False + + def get_user_by_token(self, token: str) -> dict: + """通过 token 获取用户信息""" + if not token: + return None + with self.session() as session: + from sqlalchemy import or_ + # 兼容旧格式：config_auth + if token == 'config_auth': + return None + user = session.query(UserRecord).filter( + UserRecord.token == token, + UserRecord.enabled == True + ).first() + if user: + return user.to_dict() + return None + + def update_user_token(self, username: str, token: str, expires_at: float = None) -> bool: + """更新用户的 token""" + with self.session() as session: + user = session.query(UserRecord).filter_by(username=username).first() + if user: + user.token = token + user.token_expires_at = expires_at + user.updated_at = time.time() + session.commit() + return True + return False + + def clear_user_token(self, username: str) -> bool: + """清除用户的 token""" + with self.session() as session: + user = session.query(UserRecord).filter_by(username=username).first() + if user: + user.token = None + user.token_expires_at = None + user.updated_at = time.time() + session.commit() + return True + return False + + # ==================== 数据库迁移和健康检查 ==================== + + def check_schema_version(self) -> bool: + """检查并更新数据库结构版本""" + try: + # 确保所有表都存在（包括新增的表） + Base.metadata.create_all(self.engine) + + with self.session() as session: + # 检查版本表是否存在 + from sqlalchemy import inspect + inspector = inspect(self.engine) + tables = inspector.get_table_names() + + if 'schema_versions' not in tables: + # 首次运行，创建版本表并记录当前版本 + version = SchemaVersion(version=_SCHEMA_VERSION, description="Initial schema") + session.add(version) + return True + + # 检查当前版本 + current = session.query(SchemaVersion).filter( + SchemaVersion.version == _SCHEMA_VERSION + ).first() + + if not current: + # 需要迁移 + return self._run_migrations(session) + + return True + + except Exception as e: + print(f"检查数据库版本失败: {e}") + return False + + def _run_migrations(self, session) -> bool: + """运行数据库迁移""" + try: + # 获取当前数据库中的最高版本 + latest = session.query(SchemaVersion).order_by( + SchemaVersion.version.desc() + ).first() + + current_version = latest.version if latest else 0 + + # 按版本运行迁移 + migrations = [ + (1, self._migrate_to_v1), + # 未来版本添加在这里 + # (2, self._migrate_to_v2), + ] + + for version, migration_func in migrations: + if version > current_version: + print(f"运行数据库迁移到版本 {version}...") + migration_func(session) + + # 记录迁移 + new_version = SchemaVersion( + version=version, + description=migration_func.__doc__ or f"Migration to v{version}" + ) + session.add(new_version) + + return True + + except Exception as e: + print(f"数据库迁移失败: {e}") + session.rollback() + return False + + def _migrate_to_v1(self, session): + """v1 迁移 - 初始Schema""" + # 初始Schema无需额外操作 + pass + + def health_check(self) -> dict: + """数据库健康检查""" + try: + with self.session() as session: + # 测试连接 + session.execute(text("SELECT 1")) + + # 获取表信息 + from sqlalchemy import inspect + inspector = inspect(self.engine) + tables = inspector.get_table_names() + + return { + 'healthy': True, + 'db_type': self.db_type, + 'tables': tables, + 'table_count': len(tables), + 'schema_version': _SCHEMA_VERSION + } + + except Exception as e: + return { + 'healthy': False, + 'error': str(e), + 'db_type': self.db_type + } + + def get_table_info(self) -> dict: + """获取所有表的信息""" + try: + from sqlalchemy import inspect + inspector = inspect(self.engine) + + info = {} + for table_name in inspector.get_table_names(): + try: + columns = inspector.get_columns(table_name) + indexes = inspector.get_indexes(table_name) + info[table_name] = { + 'columns': len(columns), + 'indexes': len(indexes), + 'column_names': [c['name'] for c in columns] + } + except Exception: + info[table_name] = {'error': '无法读取表信息'} + + return info + + except Exception as e: + return {'error': str(e)} + + def vacuum(self): + """清理数据库 - 仅 SQLite 支持""" + if self.db_type == 'sqlite': + from sqlalchemy import text + with self.session() as session: + session.execute(text("VACUUM")) + session.commit() + + def recreate_tables(self): + """重建所有表（危险操作！会清空数据）""" + print("警告: 即将重建所有数据库表，这将删除所有数据！") + confirm = input("输入 'yes' 确认: ") + if confirm != 'yes': + print("操作已取消") + return + + with self.session() as session: + # 删除所有表 + Base.metadata.drop_all(self.engine) + session.commit() + + # 重新创建 + self._create_tables() + print("数据库表已重建") + + +# ==================== 数据库工具函数 ==================== + +__all__ = [ + 'DatabaseManager', + 'FileRecord', + 'SyncRecord', + 'CacheRecord', + 'DownloadRecord', + 'SchemaVersion', + 'get_db', + 'init_database', + 'load_db_config_from_env', + 'merge_config', + 'set_table_prefix', + 'get_table_name', +] + + +def get_db(config: dict = None) -> DatabaseManager: + """获取数据库单例""" + return DatabaseManager(config) + + +def init_database(config: dict) -> DatabaseManager: + """初始化数据库""" + db = get_db(config) + return db diff --git a/docker/Dockerfile b/docker/Dockerfile index d2ea471..95eb406 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,73 +1,73 @@ -# ============================================ -# HYC下载站 v2.2 - Docker 构建配置 -# ============================================ - -# 构建阶段 -FROM python:3.11-slim AS builder - -WORKDIR /app - -# 安装构建依赖 -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - musl-dev \ - libffi-dev \ - libssl-dev \ - && rm -rf /var/lib/apt/lists/* - -# 安装 Python 依赖 -COPY requirements.txt . -RUN pip install --no-cache-dir --prefix=/install -r requirements.txt - -# ============================================ -# 运行阶段 -FROM python:3.11-slim - -LABEL maintainer="HYC Download Station" -LABEL description="HYC下载站 v2.2 - 镜像文件服务器 + 下载加速源" - -# 环境变量 -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -ENV HYC_HOME=/app - -# 创建工作目录 -WORKDIR ${HYC_HOME} - -# 复制 Python 和依赖 -COPY --from=builder /install /usr/local - -# 复制应用代码 -COPY main.py . -COPY settings.json . -COPY requirements.txt . -COPY core/ ./core/ -COPY api/ ./api/ -COPY handlers/ ./handlers/ -COPY mirrors/ ./mirrors/ -COPY scripts/ ./scripts/ - -# 创建数据目录 -RUN mkdir -p \ - /data \ - /downloads \ - && chown -R nobody:nogroup /data /downloads - -# 设置工作用户 -USER nobody - -# 挂载卷 -VOLUME ["/data", "/downloads"] - -# 暴露端口 -EXPOSE 8080 - -# 健康检查 -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/api/v1/health')" || exit 1 - -# 启动命令 -CMD ["python", "main.py", \ - "--host", "0.0.0.0", \ - "--port", "8080", \ - "--base-dir", "/downloads"] +# ============================================ +# HYC下载站 v2.3 - Docker 构建配置 +# ============================================ + +# 构建阶段 +FROM python:3.11-slim AS builder + +WORKDIR /app + +# 安装构建依赖 +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + musl-dev \ + libffi-dev \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +# 安装 Python 依赖 +COPY requirements.txt . +RUN pip install --no-cache-dir --prefix=/install -r requirements.txt + +# ============================================ +# 运行阶段 +FROM python:3.11-slim + +LABEL maintainer="HYC Download Station" +LABEL description="HYC下载站 v2.3 - 镜像文件服务器 + 下载加速源" + +# 环境变量 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV HYC_HOME=/app + +# 创建工作目录 +WORKDIR ${HYC_HOME} + +# 复制 Python 和依赖 +COPY --from=builder /install /usr/local + +# 复制应用代码 +COPY main.py . +COPY settings.json . +COPY requirements.txt . +COPY core/ ./core/ +COPY api/ ./api/ +COPY handlers/ ./handlers/ +COPY mirrors/ ./mirrors/ +COPY scripts/ ./scripts/ + +# 创建数据目录 +RUN mkdir -p \ + /data \ + /downloads \ + && chown -R nobody:nogroup /data /downloads + +# 设置工作用户 +USER nobody + +# 挂载卷 +VOLUME ["/data", "/downloads"] + +# 暴露端口 +EXPOSE 8080 + +# 健康检查 +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/api/v1/health')" || exit 1 + +# 启动命令 +CMD ["python", "main.py", \ + "--host", "0.0.0.0", \ + "--port", "8080", \ + "--base-dir", "/downloads"] diff --git a/handlers/http_handler.py b/handlers/http_handler.py index 1da2952..798aa17 100644 --- a/handlers/http_handler.py +++ b/handlers/http_handler.py @@ -668,7 +668,7 @@ class MirrorServerHandler(BaseHTTPRequestHandler): file_path = os.path.join(docs_dir, rel_path) # 防止目录遍历 - if not os.path.realpath(file_path).startswith(os.path.realpath(docs_dir)): + if not os.path.realpath(file_path).startswith(os.path.realpath(docs_dir) + os.sep): self.send_error(403, "Access denied") return @@ -765,7 +765,7 @@ class MirrorServerHandler(BaseHTTPRequestHandler): file_path = os.path.join(ui_dir, rel_path) # 防止目录遍历 - if not os.path.realpath(file_path).startswith(os.path.realpath(ui_dir)): + if not os.path.realpath(file_path).startswith(os.path.realpath(ui_dir) + os.sep): self.send_error(403, "Access denied") return @@ -1200,7 +1200,7 @@ class MirrorServerHandler(BaseHTTPRequestHandler): mime_type, _ = mimetypes.guess_type(file_path) self.send_response(200) self.send_header("Content-Type", mime_type) - self.send_header("Content-Length", str(file_size)) + # 使用 chunked 时不发送 Content-Length（协议不允许同时存在） self.send_header("Content-Disposition", f'attachment; filename="{self._safe_disposition_filename(file_path)}"') self.send_header("Accept-Ranges", "bytes") diff --git a/main.py b/main.py index d634c23..ec8e62d 100644 --- a/main.py +++ b/main.py @@ -1,624 +1,648 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -HYC下载站 v2.3 - 完整增强版 -主入口文件 -支持镜像同步、下载加速源、系统监控、实时通信等 -""" - -import os -import sys -import signal -import argparse -import time - -# PyInstaller 资源路径处理 -def get_resource_path(relative_path): - """获取打包后的资源路径""" - if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): - # 打包后的路径 - return os.path.join(sys._MEIPASS, relative_path) - return os.path.join(os.path.dirname(os.path.abspath(__file__)), relative_path) - -# 添加项目根目录到Python路径 -base_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, base_dir) - -from core.config import ConfigManager, load_config_file, deep_merge, load_settings_with_override -from core.server import MirrorServer -from core.utils import parse_size -from core.database import init_database, load_db_config_from_env, merge_config -from core.sync_scheduler import init_database_sync -from core.optimization import ( - MemoryManager, LowMemoryConfig, ArchitectureDetector, check_compatibility -) - - -def signal_handler(signum, _frame): - """处理退出信号（优雅退出：触发 KeyboardInterrupt 走正常关闭流程）""" - print(f"\n收到信号 {signum}，正在优雅关闭服务器...") - raise KeyboardInterrupt - - -def parse_arguments(): - """解析命令行参数""" - parser = argparse.ArgumentParser( - description='HYC下载站 v2.3 - 镜像文件服务器 + 下载加速源', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=''' -示例: - # 基本启动 - python main.py -p 8080 -d ./downloads - - # 使用配置文件 - python main.py --settings settings.json --config custom.json - - # 认证配置 - python main.py --auth-type token --auth-token your_token_here - - # 功能开关 - python main.py --enable-monitor --enable-sync --enable-mirrors - python main.py --disable-ws --disable-sse # 适合低端设备 - - # 镜像加速源 (通过 settings.json 配置各个镜像的启用/禁用) - python main.py --enable-mirrors - - # 低端设备优化 - python main.py --preset ultra_low - python main.py --memory-limit 256M --workers 2 - - # 调试 - python main.py --debug - python main.py --debug-types http auth api v2 error - - # 检查系统兼容性 - python main.py --check-compat - ''' - ) - - # 服务器配置 - parser.add_argument('--host', default='0.0.0.0', help='监听地址 (默认: 0.0.0.0)') - parser.add_argument('-p', '--port', type=int, default=8080, help='监听端口 (默认: 8080)') - parser.add_argument('-d', '--base-dir', default='./downloads', help='文件存储目录 (默认: ./downloads)') - parser.add_argument('--server-name', default='HYC下载站', help='服务器名称') - - # HTTPS配置 - parser.add_argument('--ssl-cert', help='SSL证书文件路径') - parser.add_argument('--ssl-key', help='SSL私钥文件路径') - - # 认证配置 - auth_group = parser.add_argument_group('认证配置') - auth_group.add_argument('--auth-type', choices=['none', 'basic', 'token'], default=None, - help='认证类型: none(无), basic(基本认证), token(令牌认证)') - auth_group.add_argument('--auth-user', default='admin', help='基本认证用户名') - auth_group.add_argument('--auth-pass', help='基本认证密码') - auth_group.add_argument('--auth-token', help='令牌认证密钥') - - # 功能配置 - func_group = parser.add_argument_group('功能配置') - func_group.add_argument('--directory-listing', type=bool, default=True, - help='启用目录浏览 (默认: True)') - func_group.add_argument('--enable-stats', type=bool, default=True, - help='启用下载统计 (默认: True)') - func_group.add_argument('--show-hash', action='store_true', help='显示文件哈希值') - func_group.add_argument('--ignore-hidden', action='store_true', default=True, - help='忽略隐藏文件') - func_group.add_argument('--max-upload-size', default='1G', help='最大上传文件大小') - func_group.add_argument('--api-version', choices=['v1', 'v2'], default='v2', - help='API版本 (默认: v2)') - - # 实时通信配置 - realtime_group = parser.add_argument_group('实时通信配置') - realtime_group.add_argument('--enable-ws', action='store_true', default=True, - help='启用WebSocket (默认: True)') - realtime_group.add_argument('--enable-sse', action='store_true', default=True, - help='启用SSE (默认: True)') - - # 系统监控配置 - monitor_group = parser.add_argument_group('系统监控配置') - monitor_group.add_argument('--enable-monitor', action='store_true', default=True, - help='启用系统监控 (默认: True)') - monitor_group.add_argument('--monitor-interval', type=int, default=5, - help='监控数据采集间隔(秒) (默认: 5)') - - # 同步配置 - sync_group = parser.add_argument_group('同步配置') - sync_group.add_argument('--enable-sync', action='store_true', default=True, - help='启用镜像同步 (默认: True)') - sync_group.add_argument('--sync-config', help='同步配置文件路径') - - # 镜像加速源配置 - mirror_group = parser.add_argument_group('镜像加速源配置') - mirror_group.add_argument('--enable-mirrors', action='store_true', default=True, - help='启用下载加速源 (默认: True)') - # 注意: 各个镜像的启用/禁用通过 settings.json 中的 mirrors.xxx.enabled 配置 - - # 下载限速 - rate_group = parser.add_argument_group('下载限速配置') - rate_group.add_argument('--rate-limit', type=int, default=0, - help='全局下载限速(字节/秒) (默认: 0=不限速)') - - # 低端设备优化配置 - optimize_group = parser.add_argument_group('低端设备优化配置') - optimize_group.add_argument('--preset', choices=['ultra_low', 'low', 'medium', 'high', 'auto'], - default='auto', help='设备预设 (默认: auto)') - optimize_group.add_argument('--memory-limit', default='512M', - help='内存限制 (例如: 256M, 512M, 1G)') - optimize_group.add_argument('--workers', type=int, default=0, - help='工作进程数 (0=自动)') - optimize_group.add_argument('--chunk-size', default='128K', - help='文件传输块大小 (默认: 128K)') - optimize_group.add_argument('--buffer-size', default='256K', - help='缓冲区大小 (默认: 256K)') - optimize_group.add_argument('--disable-ws', action='store_true', - help='禁用WebSocket (低端设备)') - optimize_group.add_argument('--disable-sse', action='store_true', - help='禁用SSE (低端设备)') - optimize_group.add_argument('--disable-hash', action='store_true', - help='禁用文件哈希计算 (低端设备)') - optimize_group.add_argument('--check-compat', action='store_true', - help='检查系统兼容性后退出') - - # 日志配置 - log_group = parser.add_argument_group('日志配置') - log_group.add_argument('--access-log', help='访问日志文件路径') - log_group.add_argument('--verbose', '-v', action='count', default=0, help='详细输出') - log_group.add_argument('--quiet', '-q', action='store_true', help='静默模式') - - # 细粒度调试开关 - debug_group = parser.add_argument_group('调试选项') - debug_group.add_argument('--debug', '-D', action='store_true', - help='启用所有调试输出') - debug_group.add_argument('--debug-log', dest='debug_log_file', - help='调试日志文件路径 (debug 输出将写入此文件)') - debug_group.add_argument('--debug-http', action='store_true', - help='调试 HTTP 请求') - debug_group.add_argument('--debug-auth', action='store_true', - help='调试 认证检查') - debug_group.add_argument('--debug-api', action='store_true', - help='调试 API 路由') - debug_group.add_argument('--debug-v2', action='store_true', - help='调试 V2 API') - debug_group.add_argument('--debug-error', action='store_true', - help='调试 错误堆栈') - debug_group.add_argument('--debug-download', action='store_true', - help='调试 下载记录') - debug_group.add_argument('--debug-types', '--debug-list', - nargs='+', metavar='TYPE', - choices=['http', 'auth', 'api', 'v2', 'error', 'download'], - help='指定调试类型列表 (http auth api v2 error download)') - - # 配置文件 - parser.add_argument('--settings', '--default-config', dest='settings', - help='默认配置文件路径 (settings.json)') - parser.add_argument('--config', help='覆盖配置文件路径 (JSON格式，会覆盖默认配置)') - - return parser.parse_args() - - -def build_config_from_args(args): - """从命令行参数构建配置""" - cmd_config = {} - - # 服务器基本配置 - basic_args = ['host', 'port', 'server_name', 'ssl_cert', 'ssl_key', - 'auth_type', 'auth_user', 'auth_pass', 'auth_token', - 'access_log', 'verbose', - 'api_version', 'directory_listing', - 'enable_stats', 'show_hash', 'ignore_hidden', 'max_upload_size'] - - for arg_name in basic_args: - arg_value = getattr(args, arg_name, None) - if arg_value is not None: - cmd_config[arg_name] = arg_value - - # Debug 配置处理 - # 优先级: --debug > --debug-types > individual --debug-xxx - debug_types = [] - if args.debug_types: - # 用户指定了类型列表 - debug_types = list(args.debug_types) - cmd_config['debug'] = debug_types - elif args.debug_http or args.debug_auth or args.debug_api or args.debug_v2 or args.debug_error or args.debug_download: - # 用户指定了单个类型 - if args.debug_http: - debug_types.append('http') - if args.debug_auth: - debug_types.append('auth') - if args.debug_api: - debug_types.append('api') - if args.debug_v2: - debug_types.append('v2') - if args.debug_error: - debug_types.append('error') - if args.debug_download: - debug_types.append('download') - cmd_config['debug'] = debug_types - elif args.debug: - # 开启所有 - cmd_config['debug'] = True - else: - # 未开启任何 debug - cmd_config['debug'] = False - - # Debug 日志文件 - if args.debug_log_file: - cmd_config['debug_log_file'] = args.debug_log_file - - # 路径处理 - if args.base_dir: - cmd_config['base_dir'] = os.path.abspath(args.base_dir) - - # 实时通信 - cmd_config['enable_ws'] = args.enable_ws - cmd_config['enable_sse'] = args.enable_sse - - # 系统监控 - cmd_config['enable_monitor'] = args.enable_monitor - cmd_config['monitor_interval'] = args.monitor_interval - - # 同步 - cmd_config['enable_sync'] = args.enable_sync - - # 镜像加速源 - cmd_config['enable_mirrors'] = args.enable_mirrors - # 各个镜像的启用/禁用通过 settings.json 中的 mirrors.xxx.enabled 配置 - - # 下载限速 - cmd_config['rate_limit'] = args.rate_limit - - # 静默模式 - if args.quiet: - cmd_config['verbose'] = -1 - - return cmd_config - - -def main(): - """主入口函数""" - args = parse_arguments() - - # 确定默认配置文件路径 - project_root = os.path.dirname(os.path.abspath(__file__)) - settings_path = args.settings or os.path.join(project_root, 'settings.json') - - # 打印配置来源信息 - print(f"[配置加载]") - print(f" 默认配置: {settings_path}") - - # 加载配置（优先级从低到高：默认配置 -> 覆盖配置 -> 环境变量 -> 命令行参数） - # 使用深度合并，只替换覆盖配置中有的字段 - - # 1. 加载默认配置 (settings.json) - from core.config import load_json_config - default_config = load_json_config(settings_path) or {} - print(f" 默认配置加载: {'成功' if default_config else '使用内联默认'}") - - # 2. 从覆盖配置文件加载 - override_config = {} - if args.config: - override_config = load_json_config(args.config) or {} - print(f" 覆盖配置: {args.config} ({len(override_config)} 个顶层键)") - else: - print(f" 覆盖配置: 未指定") - - # 3. 深度合并默认配置和覆盖配置 - config = deep_merge(default_config, override_config) - - # 4. 从环境变量加载数据库配置 - env_db_config = load_db_config_from_env() - if env_db_config: - config = deep_merge(config, env_db_config) - - # 5. 同步配置 - if args.sync_config: - sync_config = load_json_config(args.sync_config) - if sync_config: - if 'sync_sources' in sync_config: - config['sync_sources'] = deep_merge( - config.get('sync_sources', {}), - sync_config['sync_sources'] - ) - if 'mirrors' in sync_config: - config['mirrors'] = deep_merge( - config.get('mirrors', {}), - sync_config['mirrors'] - ) - - # 6. 命令行参数覆盖（最高优先级） - cmd_config = build_config_from_args(args) - config = deep_merge(config, cmd_config) - - # 7. 处理 auth_token（优先从文件读取，否则生成新的） - import secrets - - # 确定 token 文件路径 - if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): - # 打包环境：使用 _MEIPASS 目录 - token_file = os.path.join(sys._MEIPASS, 'auth_token.txt') - else: - # 非打包：使用脚本所在目录 - token_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'auth_token.txt') - - # 尝试从文件读取 - existing_token = None - if os.path.exists(token_file): - try: - with open(token_file, 'r') as f: - existing_token = f.read().strip() - if existing_token: - config['auth_token'] = existing_token - print(f" 已从文件加载 auth_token: {config['auth_token'][:16]}...") - except Exception as e: - print(f" 警告: 读取 auth_token 失败: {e}") - - # 如果没有现有 token，生成新的 - if not existing_token: - config['auth_token'] = secrets.token_hex(32) - print(f" 已生成新的 auth_token: {config['auth_token'][:16]}...") - - # 保存新的 token 到文件 - try: - with open(token_file, 'w') as f: - f.write(config['auth_token']) - print(f" 已保存 auth_token 到: {token_file}") - except Exception as e: - print(f" 警告: 保存 auth_token 失败: {e}") - - print(f" 最终配置: {len(config)} 个顶层配置项") - - # ==================== 系统兼容性检查 ==================== - if args.check_compat: - print("\n[系统兼容性检查]") - compat = check_compatibility() - print(f"\n兼容状态: {'✓ 通过' if compat['compatible'] else '✗ 存在问题'}") - - arch = ArchitectureDetector.get_architecture() - print(f"\n架构信息:") - print(f" - 平台: {arch['platform']}") - print(f" - 机器: {arch['machine']}") - print(f" - 架构: {arch['architecture']}") - - recommended = ArchitectureDetector.get_recommended_config() - print(f"\n推荐配置:") - for k, v in recommended.items(): - print(f" - {k}: {v}") - - if compat['warnings']: - print(f"\n警告:") - for w in compat['warnings']: - print(f" ⚠ {w}") - - if compat['errors']: - print(f"\n错误:") - for e in compat['errors']: - print(f" ✗ {e}") - - sys.exit(0 if compat['compatible'] else 1) - - # ==================== 低端设备优化 ==================== - print("\n[设备检测与优化]") - - # 检测架构 - arch = ArchitectureDetector.get_architecture() - - # 应用低端设备配置 - preset = config.get('preset', 'auto') - low_mem_config = LowMemoryConfig(preset) - - # 获取设备信息 - device_info = low_mem_config.get_device_info() - - # 显示设备信息 - if device_info['total_ram_mb'] > 0: - print(f" [硬件配置]") - print(f" - 总内存: {device_info['total_ram_mb']:.0f} MB") - print(f" - 可用内存: {device_info['available_ram_mb']:.0f} MB ({100-device_info['percent_used']:.1f}% 可用)") - print(f" - CPU核心: {device_info['cpu_count']} 核心") - print(f" - 系统架构: {arch['machine']} ({arch['architecture']})") - else: - print(f" - 系统架构: {arch['machine']} ({arch['architecture']})") - - print(f"\n [性能优化]") - print(f" - 优化模式: {preset}") - - # 应用配置 - if preset != 'auto': - status = low_mem_config.get_status() - print(f" - 预设方案: {status.get('description', preset)}") - - config = low_mem_config.apply_to_config(config) - - # 覆盖命令行参数 - if args.memory_limit: - config['memory_limit'] = parse_size(args.memory_limit) - if args.workers > 0: - config['workers'] = args.workers - if args.chunk_size: - config['chunk_size'] = parse_size(args.chunk_size) - if args.buffer_size: - config['buffer_size'] = parse_size(args.buffer_size) - - # 禁用可选功能 - if args.disable_ws: - config['enable_ws'] = False - if args.disable_sse: - config['enable_sse'] = False - if args.disable_hash: - config['calculate_hash'] = False - - # 启动内存管理器 - memory_manager = MemoryManager({ - 'enabled': True, - 'memory_limit': config.get('memory_limit', 512 * 1024 * 1024), - 'gc_interval': config.get('gc_interval', 300), # 定时GC间隔 - 'enable_scheduled_gc': config.get('enable_scheduled_gc', True) - }) - memory_manager.start() - config['_memory_manager'] = memory_manager - - mem_status = memory_manager.get_status() - print(f" - 内存限制: {mem_status['memory_limit_mb']} MB") - print(f" - 工作进程: {config.get('workers', 1)}") - print(f" - 传输块: {config.get('chunk_size', 128 * 1024) // 1024} KB") - print(f" - GC间隔: {config.get('gc_interval', 300)}秒") - print(f" - 缓存大小: {config.get('max_cache_size', 0) // (1024*1024)} MB") - - # 设置信号处理 - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - # 打印启动信息 - print() - print("╔" + "═" * 58 + "╗") - print("║" + " " * 22 + "HYC下载站 v2.3" + " " * 22 + "║") - print("╚" + "═" * 58 + "╝") - - print() - print("【服务器配置】") - print(f" ▶ 监听地址: {config.get('host')}:{config.get('port')}") - print(f" ▶ 文件目录: {os.path.abspath(config.get('base_dir', './downloads'))}") - print(f" ▶ API版本: {config.get('api_version', 'v2')}") - print(f" ▶ 认证方式: {config.get('auth_type', 'none')}") - max_upload = config.get('max_upload_size', '1G') - if isinstance(max_upload, str): - max_upload = parse_size(max_upload) - else: - max_upload = int(max_upload) - print(f" ▶ 最大上传: {max_upload // (1024*1024)} MB") - - # 数据库状态 - db_config = config.get('database', {}) - db_enabled = db_config.get('enabled', True) - - print() - print("【功能模块】") - print(f" {'●' if config.get('enable_monitor') else '○'} 系统监控", end='') - print(f" {'●' if config.get('enable_sync') else '○'} 镜像同步", end='') - print(f" {'●' if config.get('enable_mirrors') else '○'} 加速源") - print(f" {'●' if config.get('enable_ws') else '○'} WebSocket", end='') - print(f" {'●' if config.get('enable_sse') else '○'} SSE", end='') - print(f" {'●' if db_enabled else '○'} 数据库") - - if db_enabled: - print(f" └── 类型: {db_config.get('type', 'sqlite')} | 同步间隔: {db_config.get('sync_interval', 60)}s") - - print() - print("【性能参数】") - print(f" ▶ 最大线程: {config.get('max_workers', 10)}") - print(f" ▶ 工作进程: {config.get('workers', 1)}") - print(f" ▶ 内存限制: {config.get('memory_limit', 512*1024*1024) // (1024*1024)} MB") - print(f" ▶ 连接超时: {config.get('timeout', 30)}s") - - print() - print("=" * 60) - - # 初始化数据库 - db = None - scheduler = None - if db_enabled: - print("\n[初始化数据库...]") - try: - db = init_database(config) - - # 健康检查 - health = db.health_check() - if health.get('healthy'): - print(f" ✓ 数据库连接成功 ({db_config.get('type', 'sqlite')})") - print(f" ✓ 表数量: {health.get('table_count', 0)}") - else: - print(f" ✗ 数据库健康检查失败: {health.get('error')}") - - # 检查并更新Schema - if db.check_schema_version(): - print(f" ✓ 数据库结构版本检查通过") - - # 获取统计 - db_stats = db.get_stats() - print(f"\n 数据库统计:") - print(f" - 文件记录: {db_stats['total_files']}") - print(f" - 同步记录: {db_stats['total_sync_records']}") - print(f" - 缓存记录: {db_stats['total_cache_records']}") - print(f" - 下载记录: {db_stats['total_download_records']}") - - # 将数据库实例添加到配置中 - config['_db_instance'] = db - - except Exception as e: - print(f"\n ✗ 数据库初始化失败: {e}") - print(" ⚠ 服务器将继续运行，但不使用数据库功能") - - # 初始化同步调度器（不使用数据库） - if config.get('enable_sync'): - _, scheduler, _ = init_database_sync(config) - scheduler.start() - print(" - 同步调度器已启动") - - # ==================== 优雅关闭处理 ==================== - import atexit - - def cleanup(): - """服务器关闭时清理资源""" - print("\n正在关闭服务器...") - - # 关闭数据库连接池 - if db: - try: - db.engine.dispose() - print("✓ 数据库连接已关闭") - except Exception as e: - print(f"✗ 关闭数据库连接时出错: {e}") - - # 停止同步调度器 - if scheduler: - try: - scheduler.stop() - print("✓ 同步调度器已停止") - except Exception as e: - print(f"✗ 停止同步调度器时出错: {e}") - - print("服务器已关闭") - - # 注册关闭处理函数 - atexit.register(cleanup) - - # 设置服务器启动时间（用于计算运行时间） - config['start_time'] = time.time() - - # 默认凭据警告 - try: - if config.get('auth_type') == 'basic' and config.get('auth_pass') in (None, '', 'admin123'): - print("\n" + "!" * 60) - print("! 安全警告: 正在使用默认/空认证密码 (admin/admin123)") - print("! 请立即修改 settings.json 中的 auth_pass 或使用 --auth-pass 指定") - print("!" * 60) - if config.get('auth_type') == 'token' and config.get('auth_token') in (None, '', 'admin_token_123456'): - print("\n" + "!" * 60) - print("! 安全警告: 正在使用默认/空认证 token") - print("! 请立即修改 settings.json 中的 auth_token 或使用 --auth-token 指定") - print("!" * 60) - except Exception: - pass - - # 创建并启动服务器 - try: - server = MirrorServer(config) - if server.start(): - server.serve_forever() - else: - print("服务器启动失败") - sys.exit(1) - except KeyboardInterrupt: - # 信号处理已触发优雅关闭（server.stop + atexit cleanup 已执行） - print("\n服务器已正常退出") - sys.exit(0) - except Exception as e: - print(f"错误: {e}") - import traceback - traceback.print_exc() - sys.exit(1) - - -if __name__ == "__main__": - main() +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +HYC下载站 v2.3 - 完整增强版 +主入口文件 +支持镜像同步、下载加速源、系统监控、实时通信等 +""" + +import os +import sys +import signal +import argparse +import time + +# PyInstaller 资源路径处理 +def get_resource_path(relative_path): + """获取打包后的资源路径""" + if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): + # 打包后的路径 + return os.path.join(sys._MEIPASS, relative_path) + return os.path.join(os.path.dirname(os.path.abspath(__file__)), relative_path) + +# 添加项目根目录到Python路径 +base_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, base_dir) + +from core.config import ConfigManager, load_config_file, deep_merge, load_settings_with_override +from core.server import MirrorServer +from core.utils import parse_size +from core.database import init_database, load_db_config_from_env, merge_config +from core.sync_scheduler import init_database_sync +from core.optimization import ( + MemoryManager, LowMemoryConfig, ArchitectureDetector, check_compatibility +) + + +def signal_handler(signum, _frame): + """处理退出信号（优雅退出：触发 KeyboardInterrupt 走正常关闭流程）""" + print(f"\n收到信号 {signum}，正在优雅关闭服务器...") + raise KeyboardInterrupt + + +def _parse_bool_arg(value): + """解析布尔命令行参数（--flag true/false/1/0/yes/no）""" + if isinstance(value, bool): + return value + return str(value).strip().lower() in ('1', 'true', 'yes', 'on') + + +def parse_arguments(): + """解析命令行参数""" + parser = argparse.ArgumentParser( + description='HYC下载站 v2.3 - 镜像文件服务器 + 下载加速源', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +示例: + # 基本启动 + python main.py -p 8080 -d ./downloads + + # 使用配置文件 + python main.py --settings settings.json --config custom.json + + # 认证配置 + python main.py --auth-type token --auth-token your_token_here + + # 功能开关 + python main.py --enable-monitor --enable-sync --enable-mirrors + python main.py --disable-ws --disable-sse # 适合低端设备 + + # 镜像加速源 (通过 settings.json 配置各个镜像的启用/禁用) + python main.py --enable-mirrors + + # 低端设备优化 + python main.py --preset ultra_low + python main.py --memory-limit 256M --workers 2 + + # 调试 + python main.py --debug + python main.py --debug-types http auth api v2 error + + # 检查系统兼容性 + python main.py --check-compat + ''' + ) + + # 服务器配置 + parser.add_argument('--host', default='0.0.0.0', help='监听地址 (默认: 0.0.0.0)') + parser.add_argument('-p', '--port', type=int, default=8080, help='监听端口 (默认: 8080)') + parser.add_argument('-d', '--base-dir', default='./downloads', help='文件存储目录 (默认: ./downloads)') + parser.add_argument('--server-name', default='HYC下载站', help='服务器名称') + + # HTTPS配置 + parser.add_argument('--ssl-cert', help='SSL证书文件路径') + parser.add_argument('--ssl-key', help='SSL私钥文件路径') + + # 认证配置 + auth_group = parser.add_argument_group('认证配置') + auth_group.add_argument('--auth-type', choices=['none', 'basic', 'token'], default=None, + help='认证类型: none(无), basic(基本认证), token(令牌认证)') + auth_group.add_argument('--auth-user', default='admin', help='基本认证用户名') + auth_group.add_argument('--auth-pass', help='基本认证密码') + auth_group.add_argument('--auth-token', help='令牌认证密钥') + + # 功能配置 + func_group = parser.add_argument_group('功能配置') + func_group.add_argument('--directory-listing', nargs='?', const=True, default=None, + type=_parse_bool_arg, + help='启用目录浏览 (默认: True; 传 false 可禁用)') + func_group.add_argument('--enable-stats', nargs='?', const=True, default=None, + type=_parse_bool_arg, + help='启用下载统计 (默认: True; 传 false 可禁用)') + func_group.add_argument('--show-hash', action='store_true', help='显示文件哈希值') + func_group.add_argument('--ignore-hidden', nargs='?', const=True, default=None, + type=_parse_bool_arg, + help='忽略隐藏文件 (默认: True; 传 false 显示隐藏文件)') + func_group.add_argument('--max-upload-size', default='1G', help='最大上传文件大小') + func_group.add_argument('--api-version', choices=['v1', 'v2'], default='v2', + help='API版本 (默认: v2)') + + # 实时通信配置 + realtime_group = parser.add_argument_group('实时通信配置') + realtime_group.add_argument('--enable-ws', nargs='?', const=True, default=None, + type=_parse_bool_arg, + help='启用WebSocket (默认: True; 传 false 可禁用)') + realtime_group.add_argument('--enable-sse', nargs='?', const=True, default=None, + type=_parse_bool_arg, + help='启用SSE (默认: True; 传 false 可禁用)') + + # 系统监控配置 + monitor_group = parser.add_argument_group('系统监控配置') + monitor_group.add_argument('--enable-monitor', nargs='?', const=True, default=None, + type=_parse_bool_arg, + help='启用系统监控 (默认: True; 传 false 可禁用)') + monitor_group.add_argument('--monitor-interval', type=int, default=5, + help='监控数据采集间隔(秒) (默认: 5)') + + # 同步配置 + sync_group = parser.add_argument_group('同步配置') + sync_group.add_argument('--enable-sync', nargs='?', const=True, default=None, + type=_parse_bool_arg, + help='启用镜像同步 (默认: True; 传 false 可禁用)') + sync_group.add_argument('--sync-config', help='同步配置文件路径') + + # 镜像加速源配置 + mirror_group = parser.add_argument_group('镜像加速源配置') + mirror_group.add_argument('--enable-mirrors', nargs='?', const=True, default=None, + type=_parse_bool_arg, + help='启用下载加速源 (默认: True; 传 false 可禁用)') + # 注意: 各个镜像的启用/禁用通过 settings.json 中的 mirrors.xxx.enabled 配置 + + # 下载限速 + rate_group = parser.add_argument_group('下载限速配置') + rate_group.add_argument('--rate-limit', type=int, default=0, + help='全局下载限速(字节/秒) (默认: 0=不限速)') + + # 低端设备优化配置 + optimize_group = parser.add_argument_group('低端设备优化配置') + optimize_group.add_argument('--preset', choices=['ultra_low', 'low', 'medium', 'high', 'auto'], + default='auto', help='设备预设 (默认: auto)') + optimize_group.add_argument('--memory-limit', default='512M', + help='内存限制 (例如: 256M, 512M, 1G)') + optimize_group.add_argument('--workers', type=int, default=0, + help='工作进程数 (0=自动)') + optimize_group.add_argument('--chunk-size', default='128K', + help='文件传输块大小 (默认: 128K)') + optimize_group.add_argument('--buffer-size', default='256K', + help='缓冲区大小 (默认: 256K)') + optimize_group.add_argument('--disable-ws', action='store_true', + help='禁用WebSocket (低端设备)') + optimize_group.add_argument('--disable-sse', action='store_true', + help='禁用SSE (低端设备)') + optimize_group.add_argument('--disable-hash', action='store_true', + help='禁用文件哈希计算 (低端设备)') + optimize_group.add_argument('--check-compat', action='store_true', + help='检查系统兼容性后退出') + + # 日志配置 + log_group = parser.add_argument_group('日志配置') + log_group.add_argument('--access-log', help='访问日志文件路径') + log_group.add_argument('--verbose', '-v', action='count', default=0, help='详细输出') + log_group.add_argument('--quiet', '-q', action='store_true', help='静默模式') + + # 细粒度调试开关 + debug_group = parser.add_argument_group('调试选项') + debug_group.add_argument('--debug', '-D', action='store_true', + help='启用所有调试输出') + debug_group.add_argument('--debug-log', dest='debug_log_file', + help='调试日志文件路径 (debug 输出将写入此文件)') + debug_group.add_argument('--debug-http', action='store_true', + help='调试 HTTP 请求') + debug_group.add_argument('--debug-auth', action='store_true', + help='调试 认证检查') + debug_group.add_argument('--debug-api', action='store_true', + help='调试 API 路由') + debug_group.add_argument('--debug-v2', action='store_true', + help='调试 V2 API') + debug_group.add_argument('--debug-error', action='store_true', + help='调试 错误堆栈') + debug_group.add_argument('--debug-download', action='store_true', + help='调试 下载记录') + debug_group.add_argument('--debug-types', '--debug-list', + nargs='+', metavar='TYPE', + choices=['http', 'auth', 'api', 'v2', 'error', 'download'], + help='指定调试类型列表 (http auth api v2 error download)') + + # 配置文件 + parser.add_argument('--settings', '--default-config', dest='settings', + help='默认配置文件路径 (settings.json)') + parser.add_argument('--config', help='覆盖配置文件路径 (JSON格式，会覆盖默认配置)') + + return parser.parse_args() + + +def build_config_from_args(args): + """从命令行参数构建配置""" + cmd_config = {} + + # 服务器基本配置 + basic_args = ['host', 'port', 'server_name', 'ssl_cert', 'ssl_key', + 'auth_type', 'auth_user', 'auth_pass', 'auth_token', + 'access_log', 'verbose', + 'api_version', 'directory_listing', + 'enable_stats', 'show_hash', 'ignore_hidden', 'max_upload_size'] + + for arg_name in basic_args: + arg_value = getattr(args, arg_name, None) + if arg_value is not None: + cmd_config[arg_name] = arg_value + + # Debug 配置处理 + # 优先级: --debug > --debug-types > individual --debug-xxx + debug_types = [] + if args.debug_types: + # 用户指定了类型列表 + debug_types = list(args.debug_types) + cmd_config['debug'] = debug_types + elif args.debug_http or args.debug_auth or args.debug_api or args.debug_v2 or args.debug_error or args.debug_download: + # 用户指定了单个类型 + if args.debug_http: + debug_types.append('http') + if args.debug_auth: + debug_types.append('auth') + if args.debug_api: + debug_types.append('api') + if args.debug_v2: + debug_types.append('v2') + if args.debug_error: + debug_types.append('error') + if args.debug_download: + debug_types.append('download') + cmd_config['debug'] = debug_types + elif args.debug: + # 开启所有 + cmd_config['debug'] = True + else: + # 未开启任何 debug + cmd_config['debug'] = False + + # Debug 日志文件 + if args.debug_log_file: + cmd_config['debug_log_file'] = args.debug_log_file + + # 路径处理 + if args.base_dir: + cmd_config['base_dir'] = os.path.abspath(args.base_dir) + + # 实时通信（仅在显式传入时覆盖 settings.json） + if args.enable_ws is not None: + cmd_config['enable_ws'] = args.enable_ws + if args.enable_sse is not None: + cmd_config['enable_sse'] = args.enable_sse + + # 系统监控 + if args.enable_monitor is not None: + cmd_config['enable_monitor'] = args.enable_monitor + cmd_config['monitor_interval'] = args.monitor_interval + + # 同步 + if args.enable_sync is not None: + cmd_config['enable_sync'] = args.enable_sync + + # 镜像加速源 + if args.enable_mirrors is not None: + cmd_config['enable_mirrors'] = args.enable_mirrors + + # 设备预设（--preset 实际生效） + if args.preset: + cmd_config['preset'] = args.preset + # 各个镜像的启用/禁用通过 settings.json 中的 mirrors.xxx.enabled 配置 + + # 下载限速 + cmd_config['rate_limit'] = args.rate_limit + + # 静默模式 + if args.quiet: + cmd_config['verbose'] = -1 + + return cmd_config + + +def main(): + """主入口函数""" + args = parse_arguments() + + # 确定默认配置文件路径 + project_root = os.path.dirname(os.path.abspath(__file__)) + settings_path = args.settings or os.path.join(project_root, 'settings.json') + + # 打印配置来源信息 + print(f"[配置加载]") + print(f" 默认配置: {settings_path}") + + # 加载配置（优先级从低到高：默认配置 -> 覆盖配置 -> 环境变量 -> 命令行参数） + # 使用深度合并，只替换覆盖配置中有的字段 + + # 1. 加载默认配置 (settings.json) + from core.config import load_json_config + default_config = load_json_config(settings_path) or {} + print(f" 默认配置加载: {'成功' if default_config else '使用内联默认'}") + + # 2. 从覆盖配置文件加载 + override_config = {} + if args.config: + override_config = load_json_config(args.config) or {} + print(f" 覆盖配置: {args.config} ({len(override_config)} 个顶层键)") + else: + print(f" 覆盖配置: 未指定") + + # 3. 深度合并默认配置和覆盖配置 + config = deep_merge(default_config, override_config) + + # 4. 从环境变量加载数据库配置 + env_db_config = load_db_config_from_env() + if env_db_config: + config = deep_merge(config, env_db_config) + + # 5. 同步配置 + if args.sync_config: + sync_config = load_json_config(args.sync_config) + if sync_config: + if 'sync_sources' in sync_config: + config['sync_sources'] = deep_merge( + config.get('sync_sources', {}), + sync_config['sync_sources'] + ) + if 'mirrors' in sync_config: + config['mirrors'] = deep_merge( + config.get('mirrors', {}), + sync_config['mirrors'] + ) + + # 6. 命令行参数覆盖（最高优先级） + cmd_config = build_config_from_args(args) + config = deep_merge(config, cmd_config) + + # 7. 处理 auth_token（优先从文件读取，否则生成新的） + import secrets + + # 确定 token 文件路径 + if getattr(sys, 'frozen', False): + # 打包环境：使用可执行文件所在目录（_MEIPASS 是临时解压目录，写入会丢失） + token_file = os.path.join(os.path.dirname(os.path.abspath(sys.executable)), 'auth_token.txt') + else: + # 非打包：使用脚本所在目录 + token_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'auth_token.txt') + + # 尝试从文件读取 + existing_token = None + if os.path.exists(token_file): + try: + with open(token_file, 'r') as f: + existing_token = f.read().strip() + if existing_token: + config['auth_token'] = existing_token + print(f" 已从文件加载 auth_token: {config['auth_token'][:16]}...") + except Exception as e: + print(f" 警告: 读取 auth_token 失败: {e}") + + # 如果没有现有 token，生成新的 + if not existing_token: + config['auth_token'] = secrets.token_hex(32) + print(f" 已生成新的 auth_token: {config['auth_token'][:16]}...") + + # 保存新的 token 到文件 + try: + with open(token_file, 'w') as f: + f.write(config['auth_token']) + print(f" 已保存 auth_token 到: {token_file}") + except Exception as e: + print(f" 警告: 保存 auth_token 失败: {e}") + + print(f" 最终配置: {len(config)} 个顶层配置项") + + # ==================== 系统兼容性检查 ==================== + if args.check_compat: + print("\n[系统兼容性检查]") + compat = check_compatibility() + print(f"\n兼容状态: {'✓ 通过' if compat['compatible'] else '✗ 存在问题'}") + + arch = ArchitectureDetector.get_architecture() + print(f"\n架构信息:") + print(f" - 平台: {arch['platform']}") + print(f" - 机器: {arch['machine']}") + print(f" - 架构: {arch['architecture']}") + + recommended = ArchitectureDetector.get_recommended_config() + print(f"\n推荐配置:") + for k, v in recommended.items(): + print(f" - {k}: {v}") + + if compat['warnings']: + print(f"\n警告:") + for w in compat['warnings']: + print(f" ⚠ {w}") + + if compat['errors']: + print(f"\n错误:") + for e in compat['errors']: + print(f" ✗ {e}") + + sys.exit(0 if compat['compatible'] else 1) + + # ==================== 低端设备优化 ==================== + print("\n[设备检测与优化]") + + # 检测架构 + arch = ArchitectureDetector.get_architecture() + + # 应用低端设备配置 + preset = config.get('preset', 'auto') + low_mem_config = LowMemoryConfig(preset) + + # 获取设备信息 + device_info = low_mem_config.get_device_info() + + # 显示设备信息 + if device_info['total_ram_mb'] > 0: + print(f" [硬件配置]") + print(f" - 总内存: {device_info['total_ram_mb']:.0f} MB") + print(f" - 可用内存: {device_info['available_ram_mb']:.0f} MB ({100-device_info['percent_used']:.1f}% 可用)") + print(f" - CPU核心: {device_info['cpu_count']} 核心") + print(f" - 系统架构: {arch['machine']} ({arch['architecture']})") + else: + print(f" - 系统架构: {arch['machine']} ({arch['architecture']})") + + print(f"\n [性能优化]") + print(f" - 优化模式: {preset}") + + # 应用配置 + if preset != 'auto': + status = low_mem_config.get_status() + print(f" - 预设方案: {status.get('description', preset)}") + + config = low_mem_config.apply_to_config(config) + + # 覆盖命令行参数 + if args.memory_limit: + config['memory_limit'] = parse_size(args.memory_limit) + if args.workers > 0: + config['workers'] = args.workers + if args.chunk_size: + config['chunk_size'] = parse_size(args.chunk_size) + if args.buffer_size: + config['buffer_size'] = parse_size(args.buffer_size) + + # 禁用可选功能 + if args.disable_ws: + config['enable_ws'] = False + if args.disable_sse: + config['enable_sse'] = False + if args.disable_hash: + config['calculate_hash'] = False + + # 启动内存管理器 + memory_manager = MemoryManager({ + 'enabled': True, + 'memory_limit': config.get('memory_limit', 512 * 1024 * 1024), + 'gc_interval': config.get('gc_interval', 300), # 定时GC间隔 + 'enable_scheduled_gc': config.get('enable_scheduled_gc', True) + }) + memory_manager.start() + config['_memory_manager'] = memory_manager + + mem_status = memory_manager.get_status() + print(f" - 内存限制: {mem_status['memory_limit_mb']} MB") + print(f" - 工作进程: {config.get('workers', 1)}") + print(f" - 传输块: {config.get('chunk_size', 128 * 1024) // 1024} KB") + print(f" - GC间隔: {config.get('gc_interval', 300)}秒") + print(f" - 缓存大小: {config.get('max_cache_size', 0) // (1024*1024)} MB") + + # 设置信号处理 + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # 打印启动信息 + print() + print("╔" + "═" * 58 + "╗") + print("║" + " " * 22 + "HYC下载站 v2.3" + " " * 22 + "║") + print("╚" + "═" * 58 + "╝") + + print() + print("【服务器配置】") + print(f" ▶ 监听地址: {config.get('host')}:{config.get('port')}") + print(f" ▶ 文件目录: {os.path.abspath(config.get('base_dir', './downloads'))}") + print(f" ▶ API版本: {config.get('api_version', 'v2')}") + print(f" ▶ 认证方式: {config.get('auth_type', 'none')}") + max_upload = config.get('max_upload_size', '1G') + if isinstance(max_upload, str): + max_upload = parse_size(max_upload) + else: + max_upload = int(max_upload) + print(f" ▶ 最大上传: {max_upload // (1024*1024)} MB") + + # 数据库状态 + db_config = config.get('database', {}) + db_enabled = db_config.get('enabled', True) + + print() + print("【功能模块】") + print(f" {'●' if config.get('enable_monitor') else '○'} 系统监控", end='') + print(f" {'●' if config.get('enable_sync') else '○'} 镜像同步", end='') + print(f" {'●' if config.get('enable_mirrors') else '○'} 加速源") + print(f" {'●' if config.get('enable_ws') else '○'} WebSocket", end='') + print(f" {'●' if config.get('enable_sse') else '○'} SSE", end='') + print(f" {'●' if db_enabled else '○'} 数据库") + + if db_enabled: + print(f" └── 类型: {db_config.get('type', 'sqlite')} | 同步间隔: {db_config.get('sync_interval', 60)}s") + + print() + print("【性能参数】") + print(f" ▶ 最大线程: {config.get('max_workers', 10)}") + print(f" ▶ 工作进程: {config.get('workers', 1)}") + print(f" ▶ 内存限制: {config.get('memory_limit', 512*1024*1024) // (1024*1024)} MB") + print(f" ▶ 连接超时: {config.get('timeout', 30)}s") + + print() + print("=" * 60) + + # 初始化数据库 + db = None + scheduler = None + if db_enabled: + print("\n[初始化数据库...]") + try: + db = init_database(config) + + # 健康检查 + health = db.health_check() + if health.get('healthy'): + print(f" ✓ 数据库连接成功 ({db_config.get('type', 'sqlite')})") + print(f" ✓ 表数量: {health.get('table_count', 0)}") + else: + print(f" ✗ 数据库健康检查失败: {health.get('error')}") + + # 检查并更新Schema + if db.check_schema_version(): + print(f" ✓ 数据库结构版本检查通过") + + # 获取统计 + db_stats = db.get_stats() + print(f"\n 数据库统计:") + print(f" - 文件记录: {db_stats['total_files']}") + print(f" - 同步记录: {db_stats['total_sync_records']}") + print(f" - 缓存记录: {db_stats['total_cache_records']}") + print(f" - 下载记录: {db_stats['total_download_records']}") + + # 将数据库实例添加到配置中 + config['_db_instance'] = db + + except Exception as e: + print(f"\n ✗ 数据库初始化失败: {e}") + print(" ⚠ 服务器将继续运行，但不使用数据库功能") + + # 初始化同步调度器（不使用数据库） + if config.get('enable_sync'): + _, scheduler, _ = init_database_sync(config) + scheduler.start() + print(" - 同步调度器已启动") + + # ==================== 优雅关闭处理 ==================== + import atexit + + def cleanup(): + """服务器关闭时清理资源""" + print("\n正在关闭服务器...") + + # 关闭数据库连接池 + if db: + try: + db.engine.dispose() + print("✓ 数据库连接已关闭") + except Exception as e: + print(f"✗ 关闭数据库连接时出错: {e}") + + # 停止同步调度器 + if scheduler: + try: + scheduler.stop() + print("✓ 同步调度器已停止") + except Exception as e: + print(f"✗ 停止同步调度器时出错: {e}") + + print("服务器已关闭") + + # 注册关闭处理函数 + atexit.register(cleanup) + + # 设置服务器启动时间（用于计算运行时间） + config['start_time'] = time.time() + + # 默认凭据警告 + try: + if config.get('auth_type') == 'basic' and config.get('auth_pass') in (None, '', 'admin123'): + print("\n" + "!" * 60) + print("! 安全警告: 正在使用默认/空认证密码 (admin/admin123)") + print("! 请立即修改 settings.json 中的 auth_pass 或使用 --auth-pass 指定") + print("!" * 60) + if config.get('auth_type') == 'token' and config.get('auth_token') in (None, '', 'admin_token_123456'): + print("\n" + "!" * 60) + print("! 安全警告: 正在使用默认/空认证 token") + print("! 请立即修改 settings.json 中的 auth_token 或使用 --auth-token 指定") + print("!" * 60) + except Exception: + pass + + # 创建并启动服务器 + try: + server = MirrorServer(config) + if server.start(): + server.serve_forever() + else: + print("服务器启动失败") + sys.exit(1) + except KeyboardInterrupt: + # 信号处理已触发优雅关闭（server.stop + atexit cleanup 已执行） + print("\n服务器已正常退出") + sys.exit(0) + except Exception as e: + print(f"错误: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/mirrors/__init__.py b/mirrors/__init__.py index 0b19be9..b9d099d 100644 --- a/mirrors/__init__.py +++ b/mirrors/__init__.py @@ -1,583 +1,583 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -镜像源处理器包 -支持 Docker、APT、YUM、PyPI、npm、Go、Maven、Gradle 等镜像源代理 -""" - -from .docker import DockerMirror -from .apt import APTMirror -from .yum import YUMMirror -from .pypi import PyPIMirror -from .npm import NpmMirror -from .go import GoProxy -from .http import HttpMirror - -# 镜像处理器映射 -# 特殊类型使用专用处理器，其他使用通用HTTP处理器 -MIRROR_HANDLERS = { - # 专用处理器 - 'docker': DockerMirror, - 'apt': APTMirror, - 'yum': YUMMirror, - 'pypi': PyPIMirror, - 'npm': NpmMirror, - 'go': GoProxy, - - # ======== 包管理器 ======== - # Python - 'pip': HttpMirror, - 'pipenv': HttpMirror, - 'poetry': HttpMirror, - 'conda': HttpMirror, - 'anaconda': HttpMirror, - 'bandersnatch': HttpMirror, - 'twine': HttpMirror, - - # Node.js - 'yarn': HttpMirror, - 'pnpm': HttpMirror, - 'bower': HttpMirror, - - # Java - 'maven': HttpMirror, - 'gradle': HttpMirror, - - # .NET - 'nuget': HttpMirror, - - # Ruby - 'gem': HttpMirror, - 'rubygems': HttpMirror, - - # Rust - 'cargo': HttpMirror, - 'rustup': HttpMirror, - - # PHP - 'composer': HttpMirror, - 'packagist': HttpMirror, - - # Swift/Apple - 'cocoapods': HttpMirror, - - # C/C++ - 'conan': HttpMirror, - 'vcpkg': HttpMirror, - - # Dart/Flutter - 'pub': HttpMirror, - 'dart': HttpMirror, - 'flutter': HttpMirror, - - # Haskell - 'hackage': HttpMirror, - 'stackage': HttpMirror, - - # OCaml - 'opam': HttpMirror, - - # D语言 - 'dub': HttpMirror, - - # Nim - 'nimble': HttpMirror, - - # V语言 - 'v': HttpMirror, - - # Julia - 'julia': HttpMirror, - - # Lua - 'lua': HttpMirror, - 'luarocks': HttpMirror, - - # Elm - 'elm': HttpMirror, - - # Perl - 'cpan': HttpMirror, - 'cpanm': HttpMirror, - - # R - 'cran': HttpMirror, - - # LaTeX - 'ctan': HttpMirror, - - # Haskell - 'haskell': HttpMirror, - - # ======== 容器/云原生 ======== - 'helm': HttpMirror, - 'kubernetes': HttpMirror, - ' Quay.io': HttpMirror, - 'quay': HttpMirror, - 'ghcr': HttpMirror, - 'gcr': HttpMirror, - 'harbor': HttpMirror, - - # ======== 开发工具 ======== - 'jetbrains': HttpMirror, - 'vscode': HttpMirror, - 'cuda': HttpMirror, - - # ======== 语言运行时 ======== - 'node': HttpMirror, - 'python': HttpMirror, - 'ruby': HttpMirror, - 'php': HttpMirror, - 'java': HttpMirror, - 'dotnet': HttpMirror, - - # ======== Linux 发行版 ======== - 'alpine': HttpMirror, - 'arch': HttpMirror, - 'aur': HttpMirror, - 'centos': HttpMirror, - 'debian': HttpMirror, - 'fedora': HttpMirror, - 'gentoo': HttpMirror, - 'opensuse': HttpMirror, - 'void': HttpMirror, - 'freebsd': HttpMirror, - 'netbsd': HttpMirror, - 'openbsd': HttpMirror, - 'rocky': HttpMirror, - 'alma': HttpMirror, - 'kali': HttpMirror, - 'ubuntu': HttpMirror, - 'mint': HttpMirror, - 'manjaro': HttpMirror, - 'slackware': HttpMirror, - 'mageia': HttpMirror, - 'openmandriva': HttpMirror, - - # ======== 系统包管理器 ======== - 'homebrew': HttpMirror, - 'brew': HttpMirror, - 'chocolatey': HttpMirror, - 'snap': HttpMirror, - 'flatpak': HttpMirror, - 'appimage': HttpMirror, - 'winget': HttpMirror, - 'scoop': HttpMirror, - - # ======== 数据库/运维 ======== - 'postgresql': HttpMirror, - 'mysql': HttpMirror, - 'mariadb': HttpMirror, - 'mongodb': HttpMirror, - 'redis': HttpMirror, - 'influxdata': HttpMirror, - 'grafana': HttpMirror, - 'prometheus': HttpMirror, - 'elastic': HttpMirror, - 'bitnami': HttpMirror, - - # ======== 代码托管/源码 ======== - 'git': HttpMirror, - 'github': HttpMirror, - 'gitlab': HttpMirror, - 'bitbucket': HttpMirror, - 'sourceforge': HttpMirror, - - # ======== 其他 ======== - 'pacman': HttpMirror, - 'nix': HttpMirror, - 'guix': HttpMirror, - 'termux': HttpMirror, - 'msys2': HttpMirror, - 'google-fonts': HttpMirror, - 'aurora': HttpMirror, - 'cloudflare': HttpMirror, - 'fastly': HttpMirror, - - # ======== 自定义 ======== - 'http': HttpMirror, - 'custom': HttpMirror, -} - - -def get_mirror_handler(mirror_type: str): - """获取镜像处理器类""" - return MIRROR_HANDLERS.get(mirror_type) - - -# 各镜像类型的默认上游URL -DEFAULT_UPSTREAM_URLS = { - # ======== 包管理器 ======== - # Python - 'pip': 'https://pypi.org/simple', - 'pipenv': 'https://pypi.org/simple', - 'poetry': 'https://pypi.org/simple', - 'conda': 'https://repo.anaconda.com', - 'anaconda': 'https://repo.anaconda.com', - 'bandersnatch': 'https://pypi.org/simple', - 'twine': 'https://pypi.org/simple', - - # Node.js - 'yarn': 'https://registry.yarnpkg.com', - 'pnpm': 'https://registry.npmjs.org', - 'bower': 'https://registry.bower.io', - - # Java - 'maven': 'https://repo1.maven.org/maven2', - 'gradle': 'https://services.gradle.org/distributions', - - # .NET - 'nuget': 'https://api.nuget.org/v3', - - # Ruby - 'gem': 'https://rubygems.org', - 'rubygems': 'https://rubygems.org', - - # Rust - 'cargo': 'https://crates.io', - 'rustup': 'https://static.rust-lang.org', - - # PHP - 'composer': 'https://repo.packagist.org', - 'packagist': 'https://repo.packagist.org', - - # Swift/Apple - 'cocoapods': 'https://cdn.cocoapods.org', - - # C/C++ - 'conan': 'https://center.conan.io', - 'vcpkg': 'https://github.com/microsoft/vcpkg/archive/refs/heads/main.tar.gz', - - # Dart/Flutter - 'pub': 'https://pub.dev', - 'dart': 'https://storage.googleapis.com/flutter_infra_release', - 'flutter': 'https://storage.flutter-io.cn', - - # Haskell - 'hackage': 'https://hackage.haskell.org', - 'stackage': 'https://www.stackage.org', - - # OCaml - 'opam': 'https://opam.ocaml.org', - - # D语言 - 'dub': 'https://code.dlang.org', - - # Nim - 'nimble': 'https://nimble.directory', - - # V语言 - 'v': 'https://vpm.itsgriffin.com', - - # Julia - 'julia': 'https://pkg.julialang.org', - - # Lua - 'lua': 'https://luarocks.org', - 'luarocks': 'https://luarocks.org', - - # Elm - 'elm': 'https://package.elm-lang.org', - - # Perl - 'cpan': 'https://www.cpan.org', - 'cpanm': 'https://www.cpan.org', - - # R - 'cran': 'https://cran.r-project.org', - - # LaTeX - 'ctan': 'https://ctan.math.illinois.edu', - - # Haskell - 'haskell': 'https://haskell.org', - - # ======== 容器/云原生 ======== - 'helm': 'https://charts.helm.sh', - 'kubernetes': 'https://dl.k8s.io/release', - 'quay': 'https://quay.io', - 'quay.io': 'https://quay.io', - 'ghcr': 'https://ghcr.io', - 'gcr': 'https://gcr.io', - 'harbor': 'https://harbor.io', - - # ======== 开发工具 ======== - 'jetbrains': 'https://www.jetbrains.com', - 'vscode': 'https://code.visualstudio.com', - 'cuda': 'https://developer.download.nvidia.com/compute/cuda/repos', - - # ======== 语言运行时 ======== - 'node': 'https://nodejs.org/dist', - 'python': 'https://www.python.org/ftp/python', - 'ruby': 'https://www.ruby-lang.org', - 'php': 'https://www.php.net/distributions', - 'java': 'https://download.oracle.com/java', - 'dotnet': 'https://dotnetcli.azureedge.net', - - # ======== Linux 发行版 ======== - 'alpine': 'https://dl-cdn.alpinelinux.org', - 'arch': 'https://mirror.archlinux.org', - 'aur': 'https://aur.archlinux.org', - 'centos': 'https://mirrors.aliyun.com/centos', - 'debian': 'https://deb.debian.org/debian', - 'fedora': 'https://mirrors.fedoraproject.org', - 'gentoo': 'https://distfiles.gentoo.org', - 'opensuse': 'https://download.opensuse.org', - 'void': 'https://repo.voidlinux.org', - 'freebsd': 'https://pkg.freebsd.org', - 'netbsd': 'https://cdn.netbsd.org', - 'openbsd': 'https://cdn.openbsd.org', - 'rocky': 'https://download.rockylinux.org', - 'alma': 'https://repo.almalinux.org', - 'kali': 'http://.kali.org', - 'ubuntu': 'https://releases.ubuntu.com', - 'mint': 'https://packages.linuxmint.com', - 'manjaro': 'https://repo.manjaro.org', - 'slackware': 'https://mirrors.slackware.com', - 'mageia': 'https://www.mageia.org', - 'openmandriva': 'https://download.openmandriva.org', - - # ======== 系统包管理器 ======== - 'homebrew': 'https://github.com/Homebrew/brew', - 'brew': 'https://github.com/Homebrew/brew', - 'chocolatey': 'https://chocolatey.org/api/v2', - 'snap': 'https://snapcraft.io', - 'flatpak': 'https://flathub.org', - 'appimage': 'https://github.com/AppImage/AppImageHub/releases', - 'winget': 'https://github.com/microsoft/winget-pkgs', - 'scoop': 'https://scoop.sh', - - # ======== 数据库/运维 ======== - 'postgresql': 'https://www.postgresql.org', - 'mysql': 'https://dev.mysql.com', - 'mariadb': 'https://mariadb.org', - 'mongodb': 'https://www.mongodb.org', - 'redis': 'https://redis.io', - 'influxdata': 'https://portal.influxdata.com', - 'grafana': 'https://grafana.com', - 'prometheus': 'https://prometheus.io', - 'elastic': 'https://www.elastic.co', - 'bitnami': 'https://bitnami.com', - - # ======== 代码托管/源码 ======== - 'git': 'https://github.com', - 'github': 'https://github.com', - 'gitlab': 'https://gitlab.com', - 'bitbucket': 'https://bitbucket.org', - 'sourceforge': 'https://sourceforge.net', - - # ======== 其他 ======== - 'pacman': 'https://mirror.archlinux.org', - 'nix': 'https://nix-community.org', - 'guix': 'https://guix.gnu.org', - 'termux': 'https://termux.net', - 'msys2': 'https://repo.msys2.org', - 'google-fonts': 'https://fonts.google.com', - 'aurora': 'https://auroralinux.org', - 'cloudflare': 'https://cloudflare.com', - 'fastly': 'https://fastly.com', -} - - -def get_default_upstream(mirror_type: str) -> str: - """获取镜像类型的默认上游URL""" - return DEFAULT_UPSTREAM_URLS.get(mirror_type, 'https://mirror.example.com') - - -def list_available_mirrors() -> list: - """列出可用的镜像类型""" - return [ - # ======== 专用处理器 ======== - { - 'type': 'docker', - 'name': 'Docker Registry', - 'description': 'Docker镜像代理' - }, - { - 'type': 'apt', - 'name': 'APT (Debian/Ubuntu)', - 'description': 'APT软件源代理' - }, - { - 'type': 'yum', - 'name': 'YUM/DNF (RHEL/CentOS)', - 'description': 'YUM/DNF软件源代理' - }, - { - 'type': 'pypi', - 'name': 'PyPI', - 'description': 'Python包索引代理' - }, - { - 'type': 'npm', - 'name': 'npm Registry', - 'description': 'Node.js包管理器代理' - }, - { - 'type': 'go', - 'name': 'Go Modules', - 'description': 'Go模块代理' - }, - - # ======== 包管理器 ======== - # Python - {'type': 'pip', 'name': 'pip', 'description': 'Python pip包管理器'}, - {'type': 'pipenv', 'name': 'Pipenv', 'description': 'Python Pipenv代理'}, - {'type': 'poetry', 'name': 'Poetry', 'description': 'Python Poetry包管理器'}, - {'type': 'conda', 'name': 'Conda', 'description': 'Python Conda包管理器'}, - {'type': 'anaconda', 'name': 'Anaconda', 'description': 'Anaconda Python发行版'}, - - # Node.js - {'type': 'yarn', 'name': 'Yarn', 'description': 'Node.js Yarn包管理器'}, - {'type': 'pnpm', 'name': 'pnpm', 'description': 'Node.js pnpm包管理器'}, - {'type': 'bower', 'name': 'Bower', 'description': '前端包管理器'}, - - # Java - {'type': 'maven', 'name': 'Maven Central', 'description': 'Java/Maven包管理器'}, - {'type': 'gradle', 'name': 'Gradle', 'description': 'Gradle构建工具分发'}, - - # .NET - {'type': 'nuget', 'name': 'NuGet', 'description': '.NET包管理器'}, - - # Ruby - {'type': 'gem', 'name': 'RubyGems', 'description': 'Ruby包管理器'}, - {'type': 'rubygems', 'name': 'RubyGems', 'description': 'Ruby官方仓库'}, - - # Rust - {'type': 'cargo', 'name': 'Crates.io', 'description': 'Rust语言包管理器'}, - {'type': 'rustup', 'name': 'Rustup', 'description': 'Rust工具链'}, - - # PHP - {'type': 'composer', 'name': 'Composer', 'description': 'PHP包管理器'}, - {'type': 'packagist', 'name': 'Packagist', 'description': 'PHP官方包仓库'}, - - # Swift/Apple - {'type': 'cocoapods', 'name': 'CocoaPods', 'description': 'iOS/macOS包管理器'}, - - # C/C++ - {'type': 'conan', 'name': 'Conan', 'description': 'C/C++包管理器'}, - {'type': 'vcpkg', 'name': 'vcpkg', 'description': 'C/C++包管理器'}, - - # Dart/Flutter - {'type': 'pub', 'name': 'Pub.dev', 'description': 'Dart/Flutter包管理器'}, - {'type': 'dart', 'name': 'Dart SDK', 'description': 'Dart SDK'}, - {'type': 'flutter', 'name': 'Flutter', 'description': 'Flutter SDK'}, - - # Haskell - {'type': 'hackage', 'name': 'Hackage', 'description': 'Haskell包管理器'}, - {'type': 'stackage', 'name': 'Stackage', 'description': 'Haskell Stackage'}, - - # OCaml - {'type': 'opam', 'name': 'OPAM', 'description': 'OCaml包管理器'}, - - # D语言 - {'type': 'dub', 'name': 'Dub', 'description': 'D语言包管理器'}, - - # Nim - {'type': 'nimble', 'name': 'Nimble', 'description': 'Nim语言包管理器'}, - - # V语言 - {'type': 'v', 'name': 'V PM', 'description': 'V语言包管理器'}, - - # Julia - {'type': 'julia', 'name': 'Julia Packages', 'description': 'Julia语言包管理器'}, - - # Lua - {'type': 'lua', 'name': 'Lua', 'description': 'Lua语言'}, - {'type': 'luarocks', 'name': 'LuaRocks', 'description': 'Lua包管理器'}, - - # Elm - {'type': 'elm', 'name': 'Elm Packages', 'description': 'Elm语言包管理器'}, - - # Perl - {'type': 'cpan', 'name': 'CPAN', 'description': 'Perl包管理器'}, - {'type': 'cpanm', 'name': 'cpanm', 'description': 'Perl cpanminus'}, - - # R - {'type': 'cran', 'name': 'CRAN', 'description': 'R语言包仓库'}, - - # LaTeX - {'type': 'ctan', 'name': 'CTAN', 'description': 'LaTeX包管理器'}, - - # ======== 容器/云原生 ======== - {'type': 'helm', 'name': 'Helm Charts', 'description': 'Kubernetes Helm包'}, - {'type': 'kubernetes', 'name': 'Kubernetes', 'description': 'Kubernetes发行版'}, - {'type': 'quay', 'name': 'Quay.io', 'description': 'Quay容器仓库'}, - {'type': 'ghcr', 'name': 'GitHub Container Registry', 'description': 'GitHub容器仓库'}, - {'type': 'gcr', 'name': 'Google Container Registry', 'description': 'Google容器仓库'}, - {'type': 'harbor', 'name': 'Harbor', 'description': 'Harbor容器仓库'}, - - # ======== 开发工具 ======== - {'type': 'jetbrains', 'name': 'JetBrains', 'description': 'JetBrains IDE'}, - {'type': 'vscode', 'name': 'VS Code', 'description': 'Visual Studio Code'}, - {'type': 'cuda', 'name': 'CUDA', 'description': 'NVIDIA CUDA工具包'}, - - # ======== 语言运行时 ======== - {'type': 'node', 'name': 'Node.js', 'description': 'Node.js官方'}, - {'type': 'python', 'name': 'Python', 'description': 'Python官方'}, - {'type': 'ruby', 'name': 'Ruby', 'description': 'Ruby官方'}, - {'type': 'php', 'name': 'PHP', 'description': 'PHP官方'}, - {'type': 'java', 'name': 'Java JDK', 'description': 'Oracle/OpenJDK'}, - {'type': 'dotnet', 'name': '.NET SDK', 'description': '.NET SDK'}, - - # ======== Linux 发行版 ======== - {'type': 'alpine', 'name': 'Alpine Linux', 'description': 'Alpine Linux'}, - {'type': 'arch', 'name': 'Arch Linux', 'description': 'Arch Linux'}, - {'type': 'aur', 'name': 'AUR', 'description': 'Arch Linux AUR'}, - {'type': 'centos', 'name': 'CentOS', 'description': 'CentOS'}, - {'type': 'debian', 'name': 'Debian', 'description': 'Debian'}, - {'type': 'fedora', 'name': 'Fedora', 'description': 'Fedora'}, - {'type': 'gentoo', 'name': 'Gentoo', 'description': 'Gentoo'}, - {'type': 'opensuse', 'name': 'openSUSE', 'description': 'openSUSE'}, - {'type': 'void', 'name': 'Void Linux', 'description': 'Void Linux'}, - {'type': 'freebsd', 'name': 'FreeBSD', 'description': 'FreeBSD'}, - {'type': 'netbsd', 'name': 'NetBSD', 'description': 'NetBSD'}, - {'type': 'openbsd', 'name': 'OpenBSD', 'description': 'OpenBSD'}, - {'type': 'rocky', 'name': 'Rocky Linux', 'description': 'Rocky Linux'}, - {'type': 'alma', 'name': 'AlmaLinux', 'description': 'AlmaLinux'}, - {'type': 'kali', 'name': 'Kali Linux', 'description': 'Kali Linux'}, - {'type': 'ubuntu', 'name': 'Ubuntu', 'description': 'Ubuntu'}, - {'type': 'mint', 'name': 'Linux Mint', 'description': 'Linux Mint'}, - {'type': 'manjaro', 'name': 'Manjaro', 'description': 'Manjaro Linux'}, - {'type': 'slackware', 'name': 'Slackware', 'description': 'Slackware'}, - {'type': 'mageia', 'name': 'Mageia', 'description': 'Mageia Linux'}, - {'type': 'openmandriva', 'name': 'OpenMandriva', 'description': 'OpenMandriva'}, - - # ======== 系统包管理器 ======== - {'type': 'homebrew', 'name': 'Homebrew', 'description': 'macOS Homebrew'}, - {'type': 'brew', 'name': 'Brew', 'description': 'Homebrew'}, - {'type': 'chocolatey', 'name': 'Chocolatey', 'description': 'Windows Chocolatey'}, - {'type': 'snap', 'name': 'Snap Store', 'description': 'Snap应用商店'}, - {'type': 'flatpak', 'name': 'Flatpak', 'description': 'Flatpak应用'}, - {'type': 'appimage', 'name': 'AppImage', 'description': 'AppImage应用'}, - {'type': 'winget', 'name': 'winget', 'description': 'Windows winget'}, - {'type': 'scoop', 'name': 'Scoop', 'description': 'Windows Scoop'}, - - # ======== 数据库/运维 ======== - {'type': 'postgresql', 'name': 'PostgreSQL', 'description': 'PostgreSQL数据库'}, - {'type': 'mysql', 'name': 'MySQL', 'description': 'MySQL数据库'}, - {'type': 'mariadb', 'name': 'MariaDB', 'description': 'MariaDB数据库'}, - {'type': 'mongodb', 'name': 'MongoDB', 'description': 'MongoDB数据库'}, - {'type': 'redis', 'name': 'Redis', 'description': 'Redis数据库'}, - {'type': 'influxdata', 'name': 'InfluxData', 'description': 'InfluxDB时序数据库'}, - {'type': 'grafana', 'name': 'Grafana', 'description': 'Grafana可视化'}, - {'type': 'prometheus', 'name': 'Prometheus', 'description': 'Prometheus监控'}, - {'type': 'elastic', 'name': 'Elastic', 'description': 'Elasticsearch'}, - - # ======== 代码托管 ======== - {'type': 'github', 'name': 'GitHub', 'description': 'GitHub代码托管'}, - {'type': 'gitlab', 'name': 'GitLab', 'description': 'GitLab代码托管'}, - {'type': 'bitbucket', 'name': 'Bitbucket', 'description': 'Bitbucket代码托管'}, - {'type': 'sourceforge', 'name': 'SourceForge', 'description': 'SourceForge开源托管'}, - - # ======== 其他 ======== - {'type': 'nix', 'name': 'Nix/NixOS', 'description': 'Nix包管理器'}, - {'type': 'guix', 'name': 'Guix', 'description': 'Guix包管理器'}, - {'type': 'termux', 'name': 'Termux', 'description': 'Termux包管理器'}, - {'type': 'msys2', 'name': 'MSYS2', 'description': 'MSYS2包管理器'}, - {'type': 'google-fonts', 'name': 'Google Fonts', 'description': 'Google字体库'}, - - # ======== 自定义 ======== - {'type': 'custom', 'name': '自定义HTTP', 'description': '自定义HTTP镜像源'} - ] +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +镜像源处理器包 +支持 Docker、APT、YUM、PyPI、npm、Go、Maven、Gradle 等镜像源代理 +""" + +from .docker import DockerMirror +from .apt import APTMirror +from .yum import YUMMirror +from .pypi import PyPIMirror +from .npm import NpmMirror +from .go import GoProxy +from .http import HttpMirror + +# 镜像处理器映射 +# 特殊类型使用专用处理器，其他使用通用HTTP处理器 +MIRROR_HANDLERS = { + # 专用处理器 + 'docker': DockerMirror, + 'apt': APTMirror, + 'yum': YUMMirror, + 'pypi': PyPIMirror, + 'npm': NpmMirror, + 'go': GoProxy, + + # ======== 包管理器 ======== + # Python + 'pip': HttpMirror, + 'pipenv': HttpMirror, + 'poetry': HttpMirror, + 'conda': HttpMirror, + 'anaconda': HttpMirror, + 'bandersnatch': HttpMirror, + 'twine': HttpMirror, + + # Node.js + 'yarn': HttpMirror, + 'pnpm': HttpMirror, + 'bower': HttpMirror, + + # Java + 'maven': HttpMirror, + 'gradle': HttpMirror, + + # .NET + 'nuget': HttpMirror, + + # Ruby + 'gem': HttpMirror, + 'rubygems': HttpMirror, + + # Rust + 'cargo': HttpMirror, + 'rustup': HttpMirror, + + # PHP + 'composer': HttpMirror, + 'packagist': HttpMirror, + + # Swift/Apple + 'cocoapods': HttpMirror, + + # C/C++ + 'conan': HttpMirror, + 'vcpkg': HttpMirror, + + # Dart/Flutter + 'pub': HttpMirror, + 'dart': HttpMirror, + 'flutter': HttpMirror, + + # Haskell + 'hackage': HttpMirror, + 'stackage': HttpMirror, + + # OCaml + 'opam': HttpMirror, + + # D语言 + 'dub': HttpMirror, + + # Nim + 'nimble': HttpMirror, + + # V语言 + 'v': HttpMirror, + + # Julia + 'julia': HttpMirror, + + # Lua + 'lua': HttpMirror, + 'luarocks': HttpMirror, + + # Elm + 'elm': HttpMirror, + + # Perl + 'cpan': HttpMirror, + 'cpanm': HttpMirror, + + # R + 'cran': HttpMirror, + + # LaTeX + 'ctan': HttpMirror, + + # Haskell + 'haskell': HttpMirror, + + # ======== 容器/云原生 ======== + 'helm': HttpMirror, + 'kubernetes': HttpMirror, + 'quay.io': HttpMirror, + 'quay': HttpMirror, + 'ghcr': HttpMirror, + 'gcr': HttpMirror, + 'harbor': HttpMirror, + + # ======== 开发工具 ======== + 'jetbrains': HttpMirror, + 'vscode': HttpMirror, + 'cuda': HttpMirror, + + # ======== 语言运行时 ======== + 'node': HttpMirror, + 'python': HttpMirror, + 'ruby': HttpMirror, + 'php': HttpMirror, + 'java': HttpMirror, + 'dotnet': HttpMirror, + + # ======== Linux 发行版 ======== + 'alpine': HttpMirror, + 'arch': HttpMirror, + 'aur': HttpMirror, + 'centos': HttpMirror, + 'debian': HttpMirror, + 'fedora': HttpMirror, + 'gentoo': HttpMirror, + 'opensuse': HttpMirror, + 'void': HttpMirror, + 'freebsd': HttpMirror, + 'netbsd': HttpMirror, + 'openbsd': HttpMirror, + 'rocky': HttpMirror, + 'alma': HttpMirror, + 'kali': HttpMirror, + 'ubuntu': HttpMirror, + 'mint': HttpMirror, + 'manjaro': HttpMirror, + 'slackware': HttpMirror, + 'mageia': HttpMirror, + 'openmandriva': HttpMirror, + + # ======== 系统包管理器 ======== + 'homebrew': HttpMirror, + 'brew': HttpMirror, + 'chocolatey': HttpMirror, + 'snap': HttpMirror, + 'flatpak': HttpMirror, + 'appimage': HttpMirror, + 'winget': HttpMirror, + 'scoop': HttpMirror, + + # ======== 数据库/运维 ======== + 'postgresql': HttpMirror, + 'mysql': HttpMirror, + 'mariadb': HttpMirror, + 'mongodb': HttpMirror, + 'redis': HttpMirror, + 'influxdata': HttpMirror, + 'grafana': HttpMirror, + 'prometheus': HttpMirror, + 'elastic': HttpMirror, + 'bitnami': HttpMirror, + + # ======== 代码托管/源码 ======== + 'git': HttpMirror, + 'github': HttpMirror, + 'gitlab': HttpMirror, + 'bitbucket': HttpMirror, + 'sourceforge': HttpMirror, + + # ======== 其他 ======== + 'pacman': HttpMirror, + 'nix': HttpMirror, + 'guix': HttpMirror, + 'termux': HttpMirror, + 'msys2': HttpMirror, + 'google-fonts': HttpMirror, + 'aurora': HttpMirror, + 'cloudflare': HttpMirror, + 'fastly': HttpMirror, + + # ======== 自定义 ======== + 'http': HttpMirror, + 'custom': HttpMirror, +} + + +def get_mirror_handler(mirror_type: str): + """获取镜像处理器类""" + return MIRROR_HANDLERS.get(mirror_type) + + +# 各镜像类型的默认上游URL +DEFAULT_UPSTREAM_URLS = { + # ======== 包管理器 ======== + # Python + 'pip': 'https://pypi.org/simple', + 'pipenv': 'https://pypi.org/simple', + 'poetry': 'https://pypi.org/simple', + 'conda': 'https://repo.anaconda.com', + 'anaconda': 'https://repo.anaconda.com', + 'bandersnatch': 'https://pypi.org/simple', + 'twine': 'https://pypi.org/simple', + + # Node.js + 'yarn': 'https://registry.yarnpkg.com', + 'pnpm': 'https://registry.npmjs.org', + 'bower': 'https://registry.bower.io', + + # Java + 'maven': 'https://repo1.maven.org/maven2', + 'gradle': 'https://services.gradle.org/distributions', + + # .NET + 'nuget': 'https://api.nuget.org/v3', + + # Ruby + 'gem': 'https://rubygems.org', + 'rubygems': 'https://rubygems.org', + + # Rust + 'cargo': 'https://crates.io', + 'rustup': 'https://static.rust-lang.org', + + # PHP + 'composer': 'https://repo.packagist.org', + 'packagist': 'https://repo.packagist.org', + + # Swift/Apple + 'cocoapods': 'https://cdn.cocoapods.org', + + # C/C++ + 'conan': 'https://center.conan.io', + 'vcpkg': 'https://github.com/microsoft/vcpkg/archive/refs/heads/main.tar.gz', + + # Dart/Flutter + 'pub': 'https://pub.dev', + 'dart': 'https://storage.googleapis.com/flutter_infra_release', + 'flutter': 'https://storage.flutter-io.cn', + + # Haskell + 'hackage': 'https://hackage.haskell.org', + 'stackage': 'https://www.stackage.org', + + # OCaml + 'opam': 'https://opam.ocaml.org', + + # D语言 + 'dub': 'https://code.dlang.org', + + # Nim + 'nimble': 'https://nimble.directory', + + # V语言 + 'v': 'https://vpm.itsgriffin.com', + + # Julia + 'julia': 'https://pkg.julialang.org', + + # Lua + 'lua': 'https://luarocks.org', + 'luarocks': 'https://luarocks.org', + + # Elm + 'elm': 'https://package.elm-lang.org', + + # Perl + 'cpan': 'https://www.cpan.org', + 'cpanm': 'https://www.cpan.org', + + # R + 'cran': 'https://cran.r-project.org', + + # LaTeX + 'ctan': 'https://ctan.math.illinois.edu', + + # Haskell + 'haskell': 'https://haskell.org', + + # ======== 容器/云原生 ======== + 'helm': 'https://charts.helm.sh', + 'kubernetes': 'https://dl.k8s.io/release', + 'quay': 'https://quay.io', + 'quay.io': 'https://quay.io', + 'ghcr': 'https://ghcr.io', + 'gcr': 'https://gcr.io', + 'harbor': 'https://harbor.io', + + # ======== 开发工具 ======== + 'jetbrains': 'https://www.jetbrains.com', + 'vscode': 'https://code.visualstudio.com', + 'cuda': 'https://developer.download.nvidia.com/compute/cuda/repos', + + # ======== 语言运行时 ======== + 'node': 'https://nodejs.org/dist', + 'python': 'https://www.python.org/ftp/python', + 'ruby': 'https://www.ruby-lang.org', + 'php': 'https://www.php.net/distributions', + 'java': 'https://download.oracle.com/java', + 'dotnet': 'https://dotnetcli.azureedge.net', + + # ======== Linux 发行版 ======== + 'alpine': 'https://dl-cdn.alpinelinux.org', + 'arch': 'https://mirror.archlinux.org', + 'aur': 'https://aur.archlinux.org', + 'centos': 'https://mirrors.aliyun.com/centos', + 'debian': 'https://deb.debian.org/debian', + 'fedora': 'https://mirrors.fedoraproject.org', + 'gentoo': 'https://distfiles.gentoo.org', + 'opensuse': 'https://download.opensuse.org', + 'void': 'https://repo.voidlinux.org', + 'freebsd': 'https://pkg.freebsd.org', + 'netbsd': 'https://cdn.netbsd.org', + 'openbsd': 'https://cdn.openbsd.org', + 'rocky': 'https://download.rockylinux.org', + 'alma': 'https://repo.almalinux.org', + 'kali': 'http://.kali.org', + 'ubuntu': 'https://releases.ubuntu.com', + 'mint': 'https://packages.linuxmint.com', + 'manjaro': 'https://repo.manjaro.org', + 'slackware': 'https://mirrors.slackware.com', + 'mageia': 'https://www.mageia.org', + 'openmandriva': 'https://download.openmandriva.org', + + # ======== 系统包管理器 ======== + 'homebrew': 'https://github.com/Homebrew/brew', + 'brew': 'https://github.com/Homebrew/brew', + 'chocolatey': 'https://chocolatey.org/api/v2', + 'snap': 'https://snapcraft.io', + 'flatpak': 'https://flathub.org', + 'appimage': 'https://github.com/AppImage/AppImageHub/releases', + 'winget': 'https://github.com/microsoft/winget-pkgs', + 'scoop': 'https://scoop.sh', + + # ======== 数据库/运维 ======== + 'postgresql': 'https://www.postgresql.org', + 'mysql': 'https://dev.mysql.com', + 'mariadb': 'https://mariadb.org', + 'mongodb': 'https://www.mongodb.org', + 'redis': 'https://redis.io', + 'influxdata': 'https://portal.influxdata.com', + 'grafana': 'https://grafana.com', + 'prometheus': 'https://prometheus.io', + 'elastic': 'https://www.elastic.co', + 'bitnami': 'https://bitnami.com', + + # ======== 代码托管/源码 ======== + 'git': 'https://github.com', + 'github': 'https://github.com', + 'gitlab': 'https://gitlab.com', + 'bitbucket': 'https://bitbucket.org', + 'sourceforge': 'https://sourceforge.net', + + # ======== 其他 ======== + 'pacman': 'https://mirror.archlinux.org', + 'nix': 'https://nix-community.org', + 'guix': 'https://guix.gnu.org', + 'termux': 'https://termux.net', + 'msys2': 'https://repo.msys2.org', + 'google-fonts': 'https://fonts.google.com', + 'aurora': 'https://auroralinux.org', + 'cloudflare': 'https://cloudflare.com', + 'fastly': 'https://fastly.com', +} + + +def get_default_upstream(mirror_type: str) -> str: + """获取镜像类型的默认上游URL""" + return DEFAULT_UPSTREAM_URLS.get(mirror_type, 'https://mirror.example.com') + + +def list_available_mirrors() -> list: + """列出可用的镜像类型""" + return [ + # ======== 专用处理器 ======== + { + 'type': 'docker', + 'name': 'Docker Registry', + 'description': 'Docker镜像代理' + }, + { + 'type': 'apt', + 'name': 'APT (Debian/Ubuntu)', + 'description': 'APT软件源代理' + }, + { + 'type': 'yum', + 'name': 'YUM/DNF (RHEL/CentOS)', + 'description': 'YUM/DNF软件源代理' + }, + { + 'type': 'pypi', + 'name': 'PyPI', + 'description': 'Python包索引代理' + }, + { + 'type': 'npm', + 'name': 'npm Registry', + 'description': 'Node.js包管理器代理' + }, + { + 'type': 'go', + 'name': 'Go Modules', + 'description': 'Go模块代理' + }, + + # ======== 包管理器 ======== + # Python + {'type': 'pip', 'name': 'pip', 'description': 'Python pip包管理器'}, + {'type': 'pipenv', 'name': 'Pipenv', 'description': 'Python Pipenv代理'}, + {'type': 'poetry', 'name': 'Poetry', 'description': 'Python Poetry包管理器'}, + {'type': 'conda', 'name': 'Conda', 'description': 'Python Conda包管理器'}, + {'type': 'anaconda', 'name': 'Anaconda', 'description': 'Anaconda Python发行版'}, + + # Node.js + {'type': 'yarn', 'name': 'Yarn', 'description': 'Node.js Yarn包管理器'}, + {'type': 'pnpm', 'name': 'pnpm', 'description': 'Node.js pnpm包管理器'}, + {'type': 'bower', 'name': 'Bower', 'description': '前端包管理器'}, + + # Java + {'type': 'maven', 'name': 'Maven Central', 'description': 'Java/Maven包管理器'}, + {'type': 'gradle', 'name': 'Gradle', 'description': 'Gradle构建工具分发'}, + + # .NET + {'type': 'nuget', 'name': 'NuGet', 'description': '.NET包管理器'}, + + # Ruby + {'type': 'gem', 'name': 'RubyGems', 'description': 'Ruby包管理器'}, + {'type': 'rubygems', 'name': 'RubyGems', 'description': 'Ruby官方仓库'}, + + # Rust + {'type': 'cargo', 'name': 'Crates.io', 'description': 'Rust语言包管理器'}, + {'type': 'rustup', 'name': 'Rustup', 'description': 'Rust工具链'}, + + # PHP + {'type': 'composer', 'name': 'Composer', 'description': 'PHP包管理器'}, + {'type': 'packagist', 'name': 'Packagist', 'description': 'PHP官方包仓库'}, + + # Swift/Apple + {'type': 'cocoapods', 'name': 'CocoaPods', 'description': 'iOS/macOS包管理器'}, + + # C/C++ + {'type': 'conan', 'name': 'Conan', 'description': 'C/C++包管理器'}, + {'type': 'vcpkg', 'name': 'vcpkg', 'description': 'C/C++包管理器'}, + + # Dart/Flutter + {'type': 'pub', 'name': 'Pub.dev', 'description': 'Dart/Flutter包管理器'}, + {'type': 'dart', 'name': 'Dart SDK', 'description': 'Dart SDK'}, + {'type': 'flutter', 'name': 'Flutter', 'description': 'Flutter SDK'}, + + # Haskell + {'type': 'hackage', 'name': 'Hackage', 'description': 'Haskell包管理器'}, + {'type': 'stackage', 'name': 'Stackage', 'description': 'Haskell Stackage'}, + + # OCaml + {'type': 'opam', 'name': 'OPAM', 'description': 'OCaml包管理器'}, + + # D语言 + {'type': 'dub', 'name': 'Dub', 'description': 'D语言包管理器'}, + + # Nim + {'type': 'nimble', 'name': 'Nimble', 'description': 'Nim语言包管理器'}, + + # V语言 + {'type': 'v', 'name': 'V PM', 'description': 'V语言包管理器'}, + + # Julia + {'type': 'julia', 'name': 'Julia Packages', 'description': 'Julia语言包管理器'}, + + # Lua + {'type': 'lua', 'name': 'Lua', 'description': 'Lua语言'}, + {'type': 'luarocks', 'name': 'LuaRocks', 'description': 'Lua包管理器'}, + + # Elm + {'type': 'elm', 'name': 'Elm Packages', 'description': 'Elm语言包管理器'}, + + # Perl + {'type': 'cpan', 'name': 'CPAN', 'description': 'Perl包管理器'}, + {'type': 'cpanm', 'name': 'cpanm', 'description': 'Perl cpanminus'}, + + # R + {'type': 'cran', 'name': 'CRAN', 'description': 'R语言包仓库'}, + + # LaTeX + {'type': 'ctan', 'name': 'CTAN', 'description': 'LaTeX包管理器'}, + + # ======== 容器/云原生 ======== + {'type': 'helm', 'name': 'Helm Charts', 'description': 'Kubernetes Helm包'}, + {'type': 'kubernetes', 'name': 'Kubernetes', 'description': 'Kubernetes发行版'}, + {'type': 'quay', 'name': 'Quay.io', 'description': 'Quay容器仓库'}, + {'type': 'ghcr', 'name': 'GitHub Container Registry', 'description': 'GitHub容器仓库'}, + {'type': 'gcr', 'name': 'Google Container Registry', 'description': 'Google容器仓库'}, + {'type': 'harbor', 'name': 'Harbor', 'description': 'Harbor容器仓库'}, + + # ======== 开发工具 ======== + {'type': 'jetbrains', 'name': 'JetBrains', 'description': 'JetBrains IDE'}, + {'type': 'vscode', 'name': 'VS Code', 'description': 'Visual Studio Code'}, + {'type': 'cuda', 'name': 'CUDA', 'description': 'NVIDIA CUDA工具包'}, + + # ======== 语言运行时 ======== + {'type': 'node', 'name': 'Node.js', 'description': 'Node.js官方'}, + {'type': 'python', 'name': 'Python', 'description': 'Python官方'}, + {'type': 'ruby', 'name': 'Ruby', 'description': 'Ruby官方'}, + {'type': 'php', 'name': 'PHP', 'description': 'PHP官方'}, + {'type': 'java', 'name': 'Java JDK', 'description': 'Oracle/OpenJDK'}, + {'type': 'dotnet', 'name': '.NET SDK', 'description': '.NET SDK'}, + + # ======== Linux 发行版 ======== + {'type': 'alpine', 'name': 'Alpine Linux', 'description': 'Alpine Linux'}, + {'type': 'arch', 'name': 'Arch Linux', 'description': 'Arch Linux'}, + {'type': 'aur', 'name': 'AUR', 'description': 'Arch Linux AUR'}, + {'type': 'centos', 'name': 'CentOS', 'description': 'CentOS'}, + {'type': 'debian', 'name': 'Debian', 'description': 'Debian'}, + {'type': 'fedora', 'name': 'Fedora', 'description': 'Fedora'}, + {'type': 'gentoo', 'name': 'Gentoo', 'description': 'Gentoo'}, + {'type': 'opensuse', 'name': 'openSUSE', 'description': 'openSUSE'}, + {'type': 'void', 'name': 'Void Linux', 'description': 'Void Linux'}, + {'type': 'freebsd', 'name': 'FreeBSD', 'description': 'FreeBSD'}, + {'type': 'netbsd', 'name': 'NetBSD', 'description': 'NetBSD'}, + {'type': 'openbsd', 'name': 'OpenBSD', 'description': 'OpenBSD'}, + {'type': 'rocky', 'name': 'Rocky Linux', 'description': 'Rocky Linux'}, + {'type': 'alma', 'name': 'AlmaLinux', 'description': 'AlmaLinux'}, + {'type': 'kali', 'name': 'Kali Linux', 'description': 'Kali Linux'}, + {'type': 'ubuntu', 'name': 'Ubuntu', 'description': 'Ubuntu'}, + {'type': 'mint', 'name': 'Linux Mint', 'description': 'Linux Mint'}, + {'type': 'manjaro', 'name': 'Manjaro', 'description': 'Manjaro Linux'}, + {'type': 'slackware', 'name': 'Slackware', 'description': 'Slackware'}, + {'type': 'mageia', 'name': 'Mageia', 'description': 'Mageia Linux'}, + {'type': 'openmandriva', 'name': 'OpenMandriva', 'description': 'OpenMandriva'}, + + # ======== 系统包管理器 ======== + {'type': 'homebrew', 'name': 'Homebrew', 'description': 'macOS Homebrew'}, + {'type': 'brew', 'name': 'Brew', 'description': 'Homebrew'}, + {'type': 'chocolatey', 'name': 'Chocolatey', 'description': 'Windows Chocolatey'}, + {'type': 'snap', 'name': 'Snap Store', 'description': 'Snap应用商店'}, + {'type': 'flatpak', 'name': 'Flatpak', 'description': 'Flatpak应用'}, + {'type': 'appimage', 'name': 'AppImage', 'description': 'AppImage应用'}, + {'type': 'winget', 'name': 'winget', 'description': 'Windows winget'}, + {'type': 'scoop', 'name': 'Scoop', 'description': 'Windows Scoop'}, + + # ======== 数据库/运维 ======== + {'type': 'postgresql', 'name': 'PostgreSQL', 'description': 'PostgreSQL数据库'}, + {'type': 'mysql', 'name': 'MySQL', 'description': 'MySQL数据库'}, + {'type': 'mariadb', 'name': 'MariaDB', 'description': 'MariaDB数据库'}, + {'type': 'mongodb', 'name': 'MongoDB', 'description': 'MongoDB数据库'}, + {'type': 'redis', 'name': 'Redis', 'description': 'Redis数据库'}, + {'type': 'influxdata', 'name': 'InfluxData', 'description': 'InfluxDB时序数据库'}, + {'type': 'grafana', 'name': 'Grafana', 'description': 'Grafana可视化'}, + {'type': 'prometheus', 'name': 'Prometheus', 'description': 'Prometheus监控'}, + {'type': 'elastic', 'name': 'Elastic', 'description': 'Elasticsearch'}, + + # ======== 代码托管 ======== + {'type': 'github', 'name': 'GitHub', 'description': 'GitHub代码托管'}, + {'type': 'gitlab', 'name': 'GitLab', 'description': 'GitLab代码托管'}, + {'type': 'bitbucket', 'name': 'Bitbucket', 'description': 'Bitbucket代码托管'}, + {'type': 'sourceforge', 'name': 'SourceForge', 'description': 'SourceForge开源托管'}, + + # ======== 其他 ======== + {'type': 'nix', 'name': 'Nix/NixOS', 'description': 'Nix包管理器'}, + {'type': 'guix', 'name': 'Guix', 'description': 'Guix包管理器'}, + {'type': 'termux', 'name': 'Termux', 'description': 'Termux包管理器'}, + {'type': 'msys2', 'name': 'MSYS2', 'description': 'MSYS2包管理器'}, + {'type': 'google-fonts', 'name': 'Google Fonts', 'description': 'Google字体库'}, + + # ======== 自定义 ======== + {'type': 'custom', 'name': '自定义HTTP', 'description': '自定义HTTP镜像源'} + ] diff --git a/mirrors/pypi.py b/mirrors/pypi.py index c728201..925dee4 100644 --- a/mirrors/pypi.py +++ b/mirrors/pypi.py @@ -35,6 +35,14 @@ class PyPIMirror: os.makedirs(self.simple_dir, exist_ok=True) os.makedirs(self.web_dir, exist_ok=True) + def _debug_log(self, msg: str): + """受 debug 配置控制的调试日志（原硬编码 /tmp/pypi_debug.log）""" + try: + if self.config and self.config.get('debug'): + print(f"[PyPI-Mirror] {msg}") + except Exception: + pass + def handle_request(self, handler, path: str) -> bool: """ 处理PyPI请求 @@ -155,29 +163,17 @@ class PyPIMirror: package = package.lower() # 调试 - 确保函数被调用 - debug_file = '/tmp/pypi_debug.log' - with open(debug_file, 'a') as f: - f.write(f"[HANDLE_SIMPLE] START package={package}\n") - # 检查客户端Accept header accept = handler.headers.get('Accept', '') wants_json = 'application/vnd.pypi.simple.v1+json' in accept # 调试 - debug_file = '/tmp/pypi_debug.log' - with open(debug_file, 'a') as f: - f.write(f"[SIMPLE_INDEX] package={package}, wants_json={wants_json}, accept={accept[:50]}\n") - # 根据请求格式选择正确的缓存key，统一使用 simple/ 前缀 cache_key = f"simple/{package}" cached = self._get_cache(cache_key) # 调试缓存 - debug_file = '/tmp/pypi_debug.log' - with open(debug_file, 'a') as f: - f.write(f"[CACHE_CHECK] cache_key={cache_key}, cached={'YES' if cached else 'NO'}\n") - if cached: # 返回缓存，使用正确的Content-Type if wants_json: @@ -213,9 +209,6 @@ class PyPIMirror: cache_key = f"simple/{package}" else: # 转换为HTML格式 - debug_file = '/tmp/pypi_debug.log' - with open(debug_file, 'a') as f: - f.write(f"[CONVERT_CALL] Before conversion\n") try: proxy_data = self._convert_simple_html(package, data) with open(debug_file, 'a') as f: @@ -564,9 +557,6 @@ class PyPIMirror: import sys import os # 写入调试文件 - debug_file = '/tmp/pypi_debug.log' - with open(debug_file, 'a') as f: - f.write(f"[PyPI] Converting HTML for package: {package}\n") # 打印前几个链接用于调试 import re test_matches = re.findall(r'href="([^"]+)"', html)[:3] @@ -621,9 +611,6 @@ class PyPIMirror: def convert_relative_match(match): """转换相对路径链接""" href = match.group(1) - debug_file = '/tmp/pypi_debug.log' - with open(debug_file, 'a') as f: - f.write(f"[CONVERT] Input href: {href[:80]}...\n") # 提取文件名 filename = href.split('/')[-1].split('#')[0] # 提取完整的hash路径（从 packages/ 后面的所有部分除了文件名） diff --git a/requirements.txt b/requirements.txt index 1802840..df47116 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,17 @@ -psutil>=5.9.0 -pyinstaller>=6.0.0 -python-multipart>=0.0.6 -paramiko>=2.11.0 -bcrypt>=4.0.0 -cachetools>=5.3.0 -requests>=2.31.0 -croniter>=2.0.0 - -# Database (SQLite 内置, 以下为可选) -sqlalchemy>=2.0.0 -apscheduler>=3.10.0 - -# PostgreSQL 驱动 -psycopg2-binary>=2.9.0 - -# MySQL 驱动 -pymysql>=1.1.0 -cryptography>=41.0.0 +psutil>=5.9.0 +pyinstaller>=6.0.0 +python-multipart>=0.0.6 +paramiko>=2.11.0 +bcrypt>=4.0.0 +requests>=2.31.0 +croniter>=2.0.0 + +# Database (SQLite 内置, 以下为可选) +sqlalchemy>=2.0.0 + +# PostgreSQL 驱动 +psycopg2-binary>=2.9.0 + +# MySQL 驱动 +pymysql>=1.1.0 +cryptography>=41.0.0 diff --git a/settings.json b/settings.json index 54c0544..d88a115 100644 --- a/settings.json +++ b/settings.json @@ -569,7 +569,7 @@ "shutdown_timeout": 10, "strategy": "graceful" }, - "version": "2.2", + "version": "2.3", "sort_by": "name", "sort_reverse": false, "calculate_hash": false,