Baseline: pr1 HYC下载站 v2.3 before security/functional fixes
This commit is contained in:
HYC Fixer
+37
@@ -0,0 +1,37 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
.venv/
|
||||
venv/
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# 项目特定
|
||||
.claude/
|
||||
.spec-workflow/
|
||||
.zcf/
|
||||
*.log
|
||||
*.bak
|
||||
debug.txt
|
||||
api_keys.json
|
||||
auth_sessions.json
|
||||
auth_token.txt
|
||||
sync_state.json
|
||||
data/
|
||||
logs/
|
||||
downloads/
|
||||
cache/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"extends": [
|
||||
"development"
|
||||
],
|
||||
"hints": {
|
||||
"meta-viewport": "off",
|
||||
"compat-api/css": [
|
||||
"default",
|
||||
{
|
||||
"ignore": [
|
||||
"backdrop-filter"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
# Docker 部署指南
|
||||
|
||||
本项目提供多种 Docker 部署方式，支持多种架构和配置。
|
||||
|
||||
## 目录
|
||||
|
||||
- [快速开始](#快速开始)
|
||||
- [Docker 镜像](#docker-镜像)
|
||||
- [Dockerfile 说明](#dockerfile-说明)
|
||||
- [Docker Compose](#docker-compose)
|
||||
- [Kubernetes 部署](#kubernetes-部署)
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 使用 Docker Hub 镜像（推荐）
|
||||
|
||||
```bash
|
||||
# 拉取最新镜像
|
||||
docker pull hx100cv/hyc-download:latest
|
||||
|
||||
# 运行容器
|
||||
docker run -d \
|
||||
--name hyc-download \
|
||||
-p 8080:8080 \
|
||||
-v /data:/data \
|
||||
-v /downloads:/downloads \
|
||||
hx100cv/hyc-download:latest
|
||||
```
|
||||
|
||||
### 使用 GitHub Container Registry
|
||||
|
||||
```bash
|
||||
# 拉取镜像
|
||||
docker pull ghcr.io/hexikx/mirror_server:latest
|
||||
|
||||
# 运行容器
|
||||
docker run -d \
|
||||
--name hyc-download \
|
||||
-p 8080:8080 \
|
||||
-v /data:/data \
|
||||
-v /downloads:/downloads \
|
||||
ghcr.io/hexikx/mirror_server:latest
|
||||
```
|
||||
|
||||
## Docker 镜像
|
||||
|
||||
### 多架构支持
|
||||
|
||||
项目自动构建以下架构的镜像：
|
||||
|
||||
| 架构 | 镜像标签 |
|
||||
|------|----------|
|
||||
| Linux AMD64 | `hx100cv/hyc-download:amd64-latest` |
|
||||
| Linux ARM64 | `hx100cv/hyc-download:arm64v8-latest` |
|
||||
| Linux ARMv7 | `hx100cv/hyc-download:arm32v7-latest` |
|
||||
|
||||
### 拉取特定架构
|
||||
|
||||
```bash
|
||||
# AMD64
|
||||
docker pull hx100cv/hyc-download:amd64-latest
|
||||
|
||||
# ARM64
|
||||
docker pull hx100cv/hyc-download:arm64v8-latest
|
||||
|
||||
# ARMv7
|
||||
docker pull hx100cv/hyc-download:arm32v7-latest
|
||||
```
|
||||
|
||||
### 使用 Manifest（自动选择架构）
|
||||
|
||||
```bash
|
||||
# 自动选择对应架构的镜像
|
||||
docker pull hx100cv/hyc-download:latest
|
||||
```
|
||||
|
||||
## Dockerfile 说明
|
||||
|
||||
### Dockerfile（推荐）
|
||||
|
||||
完整功能镜像，包含所有依赖。
|
||||
|
||||
```dockerfile
|
||||
FROM hx100cv/hyc-download:latest
|
||||
```
|
||||
|
||||
### Dockerfile.lite
|
||||
|
||||
轻量级镜像，适合资源受限的设备。
|
||||
|
||||
```dockerfile
|
||||
FROM hx100cv/hyc-download:lite
|
||||
```
|
||||
|
||||
### Dockerfile.multiarch
|
||||
|
||||
用于构建多架构镜像的 Dockerfile。
|
||||
|
||||
```bash
|
||||
# 构建多架构镜像
|
||||
docker buildx build -f docker/Dockerfile.multiarch \
|
||||
--platform linux/amd64,linux/arm64,linux/arm/v7 \
|
||||
-t hx100cv/hyc-download:multi \
|
||||
--push .
|
||||
```
|
||||
|
||||
## Docker Compose
|
||||
|
||||
### 基础用法
|
||||
|
||||
```bash
|
||||
# 启动服务
|
||||
docker-compose up -d
|
||||
|
||||
# 查看日志
|
||||
docker-compose logs -f
|
||||
|
||||
# 停止服务
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
### 配置文件
|
||||
|
||||
项目提供以下 Docker Compose 配置文件：
|
||||
|
||||
| 文件 | 说明 |
|
||||
|------|------|
|
||||
| `docker-compose.yml` | 完整配置 |
|
||||
| `docker-compose.lite.yml` | 轻量配置 |
|
||||
| `docker-compose.raspberry.yml` | Raspberry Pi 配置 |
|
||||
|
||||
```bash
|
||||
# 使用完整配置
|
||||
docker-compose -f docker-compose.yml up -d
|
||||
|
||||
# 使用轻量配置
|
||||
docker-compose -f docker-compose.lite.yml up -d
|
||||
|
||||
# 使用 Raspberry Pi 配置
|
||||
docker-compose -f docker-compose.raspberry.yml up -d
|
||||
```
|
||||
|
||||
### 带 Nginx 反向代理
|
||||
|
||||
```bash
|
||||
docker-compose --profile with-nginx up -d
|
||||
```
|
||||
|
||||
## Kubernetes 部署
|
||||
|
||||
### 部署文件
|
||||
|
||||
项目提供以下 K8s 配置文件：
|
||||
|
||||
| 文件 | 说明 |
|
||||
|------|------|
|
||||
| `k8s/deployment.yaml` | Deployment 配置 |
|
||||
| `k8s/service.yaml` | Service 配置 |
|
||||
| `k8s/helm/` | Helm Chart |
|
||||
|
||||
### 使用 Helm 部署
|
||||
|
||||
```bash
|
||||
# 添加 Helm 仓库
|
||||
helm repo add hyc-download https://hexikx.github.io/mirror_server
|
||||
|
||||
# 更新 Helm 仓库
|
||||
helm repo update
|
||||
|
||||
# 部署
|
||||
helm install my-hyc-download hyc-download/hyc-download
|
||||
```
|
||||
|
||||
### 手动部署
|
||||
|
||||
```bash
|
||||
# 应用配置
|
||||
kubectl apply -f k8s/deployment.yaml
|
||||
kubectl apply -f k8s/service.yaml
|
||||
|
||||
# 查看状态
|
||||
kubectl get pods
|
||||
kubectl get services
|
||||
```
|
||||
|
||||
## 数据持久化
|
||||
|
||||
### 挂载卷
|
||||
|
||||
```bash
|
||||
# 数据目录
|
||||
-v /path/to/data:/data
|
||||
|
||||
# 下载目录
|
||||
-v /path/to/downloads:/downloads
|
||||
|
||||
# 配置目录
|
||||
-v /path/to/config:/app/config
|
||||
```
|
||||
|
||||
### 使用命名卷
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- data:/data
|
||||
- downloads:/downloads
|
||||
```
|
||||
|
||||
## 环境变量
|
||||
|
||||
### 支持的环境变量
|
||||
|
||||
| 变量名 | 说明 | 默认值 |
|
||||
|--------|------|--------|
|
||||
| `HYC_HOST` | 监听地址 | `0.0.0.0` |
|
||||
| `HYC_PORT` | 监听端口 | `8080` |
|
||||
| `HYC_BASE_DIR` | 文件存储目录 | `/downloads` |
|
||||
| `HYC_AUTH_TYPE` | 认证类型 | `none` |
|
||||
| `HYC_AUTH_USER` | 认证用户名 | `admin` |
|
||||
| `HYC_AUTH_PASS` | 认证密码 | `password` |
|
||||
| `HYC_ENABLE_MONITOR` | 启用监控 | `true` |
|
||||
| `HYC_ENABLE_MIRRORS` | 启用镜像加速 | `true` |
|
||||
| `HYC_RATE_LIMIT` | 下载限速 | `0` |
|
||||
|
||||
### 示例
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name hyc-download \
|
||||
-p 8080:8080 \
|
||||
-e HYC_AUTH_TYPE=basic \
|
||||
-e HYC_AUTH_USER=admin \
|
||||
-e HYC_AUTH_PASS=secret \
|
||||
hx100cv/hyc-download:latest
|
||||
```
|
||||
|
||||
## 健康检查
|
||||
|
||||
容器内置健康检查：
|
||||
|
||||
```yaml
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/api/v1/health')"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
```
|
||||
|
||||
## 日志
|
||||
|
||||
### 查看日志
|
||||
|
||||
```bash
|
||||
# 实时日志
|
||||
docker logs -f hyc-download
|
||||
|
||||
# 最近 100 行
|
||||
docker logs --tail 100 hyc-download
|
||||
```
|
||||
|
||||
### 日志级别
|
||||
|
||||
```bash
|
||||
# 详细输出
|
||||
docker run -e HYC_VERBOSE=1 hx100cv/hyc-download:latest
|
||||
|
||||
# 静默模式
|
||||
docker run -e HYC_QUIET=1 hx100cv/hyc-download:latest
|
||||
```
|
||||
|
||||
## 更新升级
|
||||
|
||||
### 拉取新版本
|
||||
|
||||
```bash
|
||||
# 拉取最新镜像
|
||||
docker pull hx100cv/hyc-download:latest
|
||||
|
||||
# 停止旧容器
|
||||
docker-compose down
|
||||
|
||||
# 启动新容器
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
### 使用 Watchtower 自动更新
|
||||
|
||||
```bash
|
||||
# 运行 Watchtower
|
||||
docker run -d \
|
||||
--name watchtower \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
containrrr/watchtower \
|
||||
hyc-download
|
||||
```
|
||||
|
||||
## 故障排查
|
||||
|
||||
### 容器无法启动
|
||||
|
||||
```bash
|
||||
# 查看详细错误
|
||||
docker logs hyc-download
|
||||
|
||||
# 进入容器排查
|
||||
docker exec -it hyc-download /bin/bash
|
||||
```
|
||||
|
||||
### 端口冲突
|
||||
|
||||
```bash
|
||||
# 检查端口占用
|
||||
netstat -tlnp | grep 8080
|
||||
|
||||
# 修改映射端口
|
||||
docker run -p 8888:8080 hx100cv/hyc-download:latest
|
||||
```
|
||||
|
||||
### 权限问题
|
||||
|
||||
```bash
|
||||
# 修改目录权限
|
||||
chmod 777 /data /downloads
|
||||
|
||||
# 或使用用户运行
|
||||
docker run -u 1000:1000 hx100cv/hyc-download:latest
|
||||
```
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 HYC Download Station
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,723 @@
|
||||
# HYC下载站 v2.3
|
||||
|
||||
模块化重构的Python镜像文件服务器，支持API版本化、实时通信、系统监控、数据库集成等高级功能。
|
||||
|
||||
## 项目结构
|
||||
|
||||
```
|
||||
mirror_server/
|
||||
├── main.py # 主入口文件
|
||||
├── requirements.txt # 依赖列表
|
||||
├── settings.json # 主配置文件
|
||||
├── auth_token.txt # 认证令牌文件
|
||||
├── api/ # API模块
|
||||
│ ├── __init__.py
|
||||
│ ├── router.py # API路由器
|
||||
│ ├── v1.py # API v1（基础功能）
|
||||
│ ├── v2.py # API v2（增强功能）
|
||||
│ ├── admin.py # 管理接口
|
||||
│ ├── ws_handler.py # WebSocket处理
|
||||
│ ├── sse_handler.py # SSE事件流
|
||||
│ └── docs/ # API文档
|
||||
├── core/ # 核心模块
|
||||
│ ├── __init__.py
|
||||
│ ├── config.py # 配置管理
|
||||
│ ├── config_hotreload.py # 配置热重载
|
||||
│ ├── utils.py # 工具函数
|
||||
│ ├── server.py # 服务器核心
|
||||
│ ├── database.py # 数据库管理
|
||||
│ ├── mirror_sync.py # 镜像同步管理
|
||||
│ ├── sync_scheduler.py # 同步调度器
|
||||
│ ├── sync_engine.py # 同步引擎
|
||||
│ ├── cache_manager.py # 缓存管理
|
||||
│ ├── cache_prewarm.py # 缓存预热
|
||||
│ ├── monitor.py # 系统监控
|
||||
│ ├── health_check.py # 健康检查
|
||||
│ ├── alerts.py # 告警系统
|
||||
│ ├── security.py # 安全模块
|
||||
│ ├── api_auth.py # API认证
|
||||
│ ├── api_docs.py # API文档
|
||||
│ ├── optimization.py # 性能优化
|
||||
│ ├── scheduler.py # 任务调度
|
||||
│ ├── graceful_restart.py # 优雅重启
|
||||
│ └── prometheus.py # Prometheus监控
|
||||
├── handlers/ # HTTP请求处理器
|
||||
│ ├── __init__.py
|
||||
│ └── http_handler.py # HTTP请求处理
|
||||
├── mirrors/ # 镜像加速源模块
|
||||
│ ├── __init__.py
|
||||
│ ├── http.py # HTTP/HTTPS镜像
|
||||
│ ├── docker.py # Docker Hub镜像
|
||||
│ ├── apt.py # APT镜像
|
||||
│ ├── yum.py # YUM镜像
|
||||
│ ├── pypi.py # PyPI镜像
|
||||
│ ├── npm.py # npm镜像
|
||||
│ └── go.py # Go镜像
|
||||
├── scripts/ # 辅助脚本
|
||||
├── data/ # 数据目录
|
||||
├── docker/ # Docker配置
|
||||
└── k8s/ # Kubernetes配置
|
||||
```
|
||||
|
||||
## 安装
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### 依赖
|
||||
|
||||
- psutil - 系统监控
|
||||
- python-multipart - 多部分表单数据处理
|
||||
- paramiko - SFTP支持（可选）
|
||||
|
||||
## 使用
|
||||
|
||||
### 基本使用
|
||||
|
||||
```bash
|
||||
# 启动服务器（默认端口8080）
|
||||
python main.py
|
||||
|
||||
# 指定端口和目录
|
||||
python main.py -p 8080 -d ./downloads
|
||||
|
||||
# 启用HTTPS
|
||||
python main.py --ssl-cert cert.pem --ssl-key key.pem
|
||||
|
||||
# 使用配置文件
|
||||
python main.py --config config.json
|
||||
|
||||
# 指定API版本 (默认: v2)
|
||||
python main.py --api-version v2
|
||||
|
||||
# 使用settings.json配置文件
|
||||
python main.py --settings settings.json
|
||||
|
||||
# 检查系统兼容性
|
||||
python main.py --check-compat
|
||||
```
|
||||
|
||||
### 认证
|
||||
|
||||
```bash
|
||||
# 无认证
|
||||
python main.py --auth-type none
|
||||
|
||||
# 基本认证
|
||||
python main.py --auth-type basic --auth-user admin --auth-pass password
|
||||
|
||||
# 令牌认证
|
||||
python main.py --auth-type token --auth-token your_token_here
|
||||
```
|
||||
|
||||
### 功能开关
|
||||
|
||||
```bash
|
||||
# 禁用监控/同步/加速源
|
||||
python main.py --enable-monitor --enable-sync --enable-mirrors
|
||||
|
||||
# 禁用WebSocket/SSE (适合低端设备)
|
||||
python main.py --disable-ws --disable-sse
|
||||
|
||||
# 禁用文件哈希计算
|
||||
python main.py --disable-hash
|
||||
|
||||
# 启用下载统计
|
||||
python main.py --enable-stats --show-hash
|
||||
|
||||
# 启用目录浏览
|
||||
python main.py --directory-listing true
|
||||
```
|
||||
|
||||
### 镜像加速源
|
||||
|
||||
```bash
|
||||
# 启用/禁用所有加速源功能
|
||||
python main.py --enable-mirrors # 启用
|
||||
python main.py --enable-mirrors=false # 禁用
|
||||
|
||||
# 各个镜像的启用/禁用通过 settings.json 配置
|
||||
# 在 mirrors.xxx.enabled 中设置，例如:
|
||||
# "docker": { "enabled": true, ... }
|
||||
# "pypi": { "enabled": true, ... }
|
||||
```
|
||||
|
||||
### 镜像同步配置
|
||||
|
||||
镜像同步支持多种同步类型，通过 `settings.json` 中的 `sync_sources` 配置：
|
||||
|
||||
```json
|
||||
{
|
||||
"mirrors": {
|
||||
"my-http-mirror": {
|
||||
"type": "http",
|
||||
"url": "https://example.com/mirror/",
|
||||
"target": "downloads/mirror",
|
||||
"enabled": true,
|
||||
"auto_sync": true,
|
||||
"schedule": { "enabled": true, "type": "cron", "cron": "0 4 * * *" }
|
||||
},
|
||||
"my-ftp-mirror": {
|
||||
"type": "ftp",
|
||||
"host": "ftp.example.com",
|
||||
"port": 21,
|
||||
"username": "anonymous",
|
||||
"password": "[email protected]",
|
||||
"remote_path": "/pub",
|
||||
"target": "downloads/ftp",
|
||||
"enabled": true
|
||||
},
|
||||
"my-sftp-mirror": {
|
||||
"type": "sftp",
|
||||
"host": "sftp.example.com",
|
||||
"port": 22,
|
||||
"username": "syncuser",
|
||||
"password": "password",
|
||||
"private_key": "/path/to/id_rsa",
|
||||
"remote_path": "/mirror",
|
||||
"target": "downloads/sftp"
|
||||
},
|
||||
"my-git-repo": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/example/repo.git",
|
||||
"branch": "main",
|
||||
"depth": 1,
|
||||
"target": "downloads/git"
|
||||
},
|
||||
"my-s3-mirror": {
|
||||
"type": "s3",
|
||||
"endpoint": "https://s3.amazonaws.com",
|
||||
"bucket": "my-bucket",
|
||||
"access_key": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "us-east-1",
|
||||
"prefix": "mirrors/",
|
||||
"target": "downloads/s3"
|
||||
},
|
||||
"my-oss-mirror": {
|
||||
"type": "oss",
|
||||
"bucket": "my-bucket",
|
||||
"access_key": "your-access-key",
|
||||
"secret_key": "your-secret-key",
|
||||
"region": "cn-hangzhou",
|
||||
"target": "downloads/oss"
|
||||
},
|
||||
"my-cos-mirror": {
|
||||
"type": "cos",
|
||||
"bucket": "my-bucket",
|
||||
"access_key": "your-access-key",
|
||||
"secret_key": "your-secret-key",
|
||||
"region": "ap-guangzhou",
|
||||
"target": "downloads/cos"
|
||||
},
|
||||
"my-rsync-mirror": {
|
||||
"type": "rsync",
|
||||
"source": "rsync://rsync.example.com/module/path",
|
||||
"target": "downloads/rsync",
|
||||
"exclude": ["*.tmp", "*.log"]
|
||||
},
|
||||
"my-local-mirror": {
|
||||
"type": "local",
|
||||
"path": "/mnt/external drive/mirrors",
|
||||
"target": "downloads/local"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**支持的同步类型：**
|
||||
|
||||
| 类型 | 说明 | 必需配置 |
|
||||
|------|------|----------|
|
||||
| `http` / `https` | HTTP/HTTPS同步 | `url`, `target` |
|
||||
| `ftp` | FTP同步 | `host`, `target` |
|
||||
| `sftp` | SFTP同步 | `host`, `username`, `target` |
|
||||
| `rsync` | Rsync同步 | `source`, `target` |
|
||||
| `git` | Git仓库克隆/更新 | `url`, `target` |
|
||||
| `s3` | AWS S3 / MinIO | `bucket`, `access_key`, `secret_key`, `endpoint` |
|
||||
| `oss` | 阿里云OSS | `bucket`, `access_key`, `secret_key` |
|
||||
| `cos` | 腾讯云COS | `bucket`, `access_key`, `secret_key` |
|
||||
| `webdav` | WebDAV同步 | `url`, `username`, `password` |
|
||||
| `local` | 本地目录同步 | `path`, `target` |
|
||||
|
||||
### 下载限速
|
||||
|
||||
```bash
|
||||
# 全局限速 (字节/秒)
|
||||
python main.py --rate-limit 10485760 # 10MB/s
|
||||
```
|
||||
|
||||
### 低端设备优化
|
||||
|
||||
```bash
|
||||
# 使用预设配置
|
||||
python main.py --preset ultra_low # 超低功耗
|
||||
python main.py --preset low # 低配置
|
||||
python main.py --preset medium # 中等
|
||||
python main.py --preset high # 高性能
|
||||
python main.py --preset auto # 自动检测
|
||||
|
||||
# 自定义参数
|
||||
python main.py --memory-limit 256M \
|
||||
--workers 2 \
|
||||
--chunk-size 64K \
|
||||
--buffer-size 128K
|
||||
```
|
||||
|
||||
### 调试选项
|
||||
|
||||
```bash
|
||||
# 启用所有调试
|
||||
python main.py --debug
|
||||
|
||||
# 启用特定调试
|
||||
python main.py --debug-http
|
||||
python main.py --debug-auth
|
||||
python main.py --debug-api
|
||||
python main.py --debug-v2
|
||||
python main.py --debug-error
|
||||
python main.py --debug-download
|
||||
|
||||
# 输出到调试文件
|
||||
python main.py --debug-log /var/log/hyc-debug.log
|
||||
|
||||
# 指定调试类型
|
||||
python main.py --debug-types http auth api v2 error
|
||||
|
||||
# 详细输出
|
||||
python main.py -v
|
||||
python main.py -vv
|
||||
python main.py -q # 静默模式
|
||||
```
|
||||
|
||||
### 启动参数完整列表
|
||||
|
||||
```
|
||||
服务器配置:
|
||||
--host HOST 监听地址 (默认: 0.0.0.0)
|
||||
-p, --port PORT 监听端口 (默认: 8080)
|
||||
-d, --base-dir DIR 文件存储目录 (默认: ./downloads)
|
||||
--server-name NAME 服务器名称 (默认: HYC下载站)
|
||||
|
||||
HTTPS配置:
|
||||
--ssl-cert FILE SSL证书文件路径
|
||||
--ssl-key FILE SSL私钥文件路径
|
||||
|
||||
认证配置:
|
||||
--auth-type TYPE 认证类型: none/basic/token (默认: none)
|
||||
--auth-user USER 基本认证用户名 (默认: admin)
|
||||
--auth-pass PASS 基本认证密码
|
||||
--auth-token TOKEN 令牌认证密钥
|
||||
|
||||
功能配置:
|
||||
--directory-listing BOOL 启用目录浏览 (默认: True)
|
||||
--enable-stats BOOL 启用下载统计 (默认: True)
|
||||
--show-hash 显示文件哈希值
|
||||
--ignore-hidden 忽略隐藏文件 (默认: True)
|
||||
--max-upload-size SIZE 最大上传文件大小 (默认: 1G)
|
||||
--api-version VERSION API版本: v1/v2 (默认: v2)
|
||||
|
||||
实时通信:
|
||||
--enable-ws 启用WebSocket (默认: True)
|
||||
--enable-sse 启用SSE (默认: True)
|
||||
|
||||
系统监控:
|
||||
--enable-monitor 启用系统监控 (默认: True)
|
||||
--monitor-interval SEC 监控采集间隔(秒) (默认: 5)
|
||||
|
||||
同步配置:
|
||||
--enable-sync 启用镜像同步 (默认: True)
|
||||
--sync-config FILE 同步配置文件路径
|
||||
|
||||
镜像加速源:
|
||||
--enable-mirrors 启用下载加速源 (默认: True)
|
||||
# 各个镜像通过 settings.json 配置 (mirrors.xxx.enabled)
|
||||
|
||||
下载限速:
|
||||
--rate-limit BYTES 全局下载限速 (默认: 0=不限速)
|
||||
|
||||
低端设备优化:
|
||||
--preset PRESET 设备预设: ultra_low/low/medium/high/auto (默认: auto)
|
||||
--memory-limit SIZE 内存限制 (例如: 256M, 512M, 1G)
|
||||
--workers NUM 工作进程数 (0=自动)
|
||||
--chunk-size SIZE 文件传输块大小 (默认: 128K)
|
||||
--buffer-size SIZE 缓冲区大小 (默认: 256K)
|
||||
--disable-ws 禁用WebSocket
|
||||
--disable-sse 禁用SSE
|
||||
--disable-hash 禁用文件哈希计算
|
||||
--check-compat 检查系统兼容性后退出
|
||||
|
||||
日志配置:
|
||||
--access-log FILE 访问日志文件路径
|
||||
-v, --verbose 详细输出 (可叠加: -vvv)
|
||||
-q, --quiet 静默模式
|
||||
|
||||
调试选项:
|
||||
-D, --debug 启用所有调试输出
|
||||
--debug-log FILE 调试日志文件路径
|
||||
--debug-http 调试HTTP请求
|
||||
--debug-auth 调试认证检查
|
||||
--debug-api 调试API路由
|
||||
--debug-v2 调试V2 API
|
||||
--debug-error 调试错误堆栈
|
||||
--debug-download 调试下载记录
|
||||
--debug-types TYPES 指定调试类型列表
|
||||
|
||||
配置文件:
|
||||
--settings FILE 默认配置文件路径 (默认: settings.json)
|
||||
--config FILE 覆盖配置文件路径 (JSON格式)
|
||||
```
|
||||
|
||||
## API版本
|
||||
|
||||
### API v1 - 基础功能
|
||||
|
||||
- 文件管理：上传、下载、删除、搜索
|
||||
- 目录浏览
|
||||
- 基础统计
|
||||
- 同步管理（HTTP/HTTPS、FTP、SFTP、本地）
|
||||
|
||||
#### 示例API端点
|
||||
|
||||
```
|
||||
GET /api/v1/files # 列出文件
|
||||
GET /api/v1/file/{path} # 获取文件信息
|
||||
DELETE /api/v1/file/{path} # 删除文件
|
||||
POST /api/v1/upload # 上传文件
|
||||
GET /api/v1/search?q={term} # 搜索文件
|
||||
GET /api/v1/stats # 获取统计
|
||||
GET /api/v1/health # 健康检查
|
||||
POST /api/v1/sync/start # 开始同步
|
||||
```
|
||||
|
||||
### API v2 - 增强功能
|
||||
|
||||
继承v1所有功能，额外提供：
|
||||
|
||||
- 增强搜索（模糊匹配、正则表达式、内容搜索）
|
||||
- 按标签和日期范围搜索
|
||||
- 文件元数据管理
|
||||
- 文件版本控制
|
||||
- 缩略图生成
|
||||
- 实时服务器监控
|
||||
- Webhook支持
|
||||
- 历史数据分析
|
||||
|
||||
#### 示例API端点
|
||||
|
||||
```
|
||||
GET /api/v2/search/enhanced?q={term} # 增强搜索
|
||||
GET /api/v2/search/by-tag?tag={tag} # 按标签搜索
|
||||
GET /api/v2/search/by-date?start={date} # 按日期搜索
|
||||
GET /api/v2/stats/detailed # 详细统计
|
||||
GET /api/v2/stats/trending # 热门文件
|
||||
GET /api/v2/file/{path}/metadata # 获取文件元数据
|
||||
PUT /api/v2/file/{path}/metadata # 更新文件元数据
|
||||
GET /api/v2/file/{path}/versions # 获取文件版本
|
||||
POST /api/v2/file/{path}/versions # 创建文件版本
|
||||
GET /api/v2/file/{path}/thumbnail?w=200&h=200 # 获取缩略图
|
||||
GET /api/v2/monitor/realtime # 实时监控
|
||||
GET /api/v2/webhooks # 列出webhooks
|
||||
POST /api/v2/webhooks # 创建webhook
|
||||
```
|
||||
|
||||
## 配置
|
||||
|
||||
### 配置文件示例
|
||||
|
||||
参考 `config.example.json`
|
||||
|
||||
### 命令行参数
|
||||
|
||||
```
|
||||
服务器配置:
|
||||
--host 监听地址 (默认: 0.0.0.0)
|
||||
-p, --port 监听端口 (默认: 8080)
|
||||
-d, --base-dir 文件存储目录 (默认: ./downloads)
|
||||
--server-name 服务器名称
|
||||
|
||||
HTTPS配置:
|
||||
--ssl-cert SSL证书文件路径
|
||||
--ssl-key SSL私钥文件路径
|
||||
|
||||
认证配置:
|
||||
--auth-type 认证类型: none, basic, token
|
||||
--auth-user 基本认证用户名
|
||||
--auth-pass 基本认证密码
|
||||
--auth-token 令牌认证密钥
|
||||
|
||||
功能配置:
|
||||
--no-directory-listing 禁用目录浏览
|
||||
--no-stats 禁用下载统计
|
||||
--show-hash 显示文件哈希值
|
||||
--max-upload-size 最大上传文件大小 (默认: 1G)
|
||||
--sync-config 同步配置文件路径
|
||||
--api-version API版本: v1, v2 (默认: v1)
|
||||
|
||||
日志配置:
|
||||
--access-log 访问日志文件路径
|
||||
-v, --verbose 详细输出
|
||||
-q, --quiet 静默模式
|
||||
|
||||
配置:
|
||||
--config 配置文件路径 (JSON格式)
|
||||
```
|
||||
|
||||
## 模块说明
|
||||
|
||||
### core/ - 核心模块
|
||||
|
||||
- **config.py**: 配置管理和验证，支持多级配置合并
|
||||
- **config_hotreload.py**: 配置文件热重载，无需重启生效
|
||||
- **utils.py**: 通用工具函数（文件大小格式化、哈希计算等）
|
||||
- **server.py**: 服务器核心类，基于aiohttp
|
||||
- **database.py**: 数据库管理，支持SQLite/MySQL/PostgreSQL
|
||||
- **mirror_sync.py**: 镜像同步管理器（支持HTTP/HTTPS、FTP、SFTP、本地、Rsync、Git、S3/OSS/COS、WebDAV）
|
||||
- **sync_scheduler.py**: 同步调度器，定时执行同步任务
|
||||
- **sync_engine.py**: 同步引擎，核心同步逻辑
|
||||
- **cache_manager.py**: 缓存管理，智能缓存策略
|
||||
- **cache_prewarm.py**: 缓存预热，启动时预加载热门资源
|
||||
- **monitor.py**: 系统监控，CPU/内存/磁盘/网络实时监控
|
||||
- **health_check.py**: 健康检查，系统组件状态检测
|
||||
- **alerts.py**: 告警系统，异常情况自动告警
|
||||
- **security.py**: 安全模块，防护和审计
|
||||
- **api_auth.py**: API认证，Token/Basic认证
|
||||
- **api_docs.py**: API文档自动生成
|
||||
- **optimization.py**: 性能优化，自动检测设备配置
|
||||
- **scheduler.py**: 通用任务调度器
|
||||
- **graceful_restart.py**: 优雅重启，服务无缝更新
|
||||
- **prometheus.py**: Prometheus监控指标导出
|
||||
|
||||
### api/ - API模块
|
||||
|
||||
- **router.py**: API路由器，支持版本化
|
||||
- **v1.py**: API v1实现（基础功能）
|
||||
- **v2.py**: API v2实现（增强功能，继承v1）
|
||||
- **admin.py**: 管理接口，Webhooks管理
|
||||
- **ws_handler.py**: WebSocket处理，实时双向通信
|
||||
- **sse_handler.py**: SSE事件流，单向实时推送
|
||||
- **docs/**: API文档 (index.html, api.md)
|
||||
|
||||
### handlers/ - HTTP请求处理器
|
||||
|
||||
- **http_handler.py**: HTTP请求处理，包括文件服务、目录浏览等
|
||||
|
||||
## 特性
|
||||
|
||||
- ✅ 模块化设计，易于扩展
|
||||
- ✅ API版本化支持（v1/v2）
|
||||
- ✅ 镜像同步（HTTP/HTTPS、FTP、SFTP、Rsync、Git、S3/OSS/COS、WebDAV、本地目录）
|
||||
- ✅ 文件上传下载（支持断点续传）
|
||||
- ✅ 目录浏览（镜像站风格）
|
||||
- ✅ 认证支持（无、基本认证、令牌认证）
|
||||
- ✅ HTTPS支持
|
||||
- ✅ 下载统计
|
||||
- ✅ 文件搜索（基础/增强/正则）
|
||||
- ✅ 批量操作
|
||||
- ✅ 压缩/解压缩
|
||||
- ✅ 增强搜索（v2）
|
||||
- ✅ 文件元数据管理（v2）
|
||||
- ✅ 服务器监控（v2）
|
||||
- ✅ Webhook支持（v2）
|
||||
- ✅ 实时通信（WebSocket + SSE）
|
||||
- ✅ 数据库支持（SQLite/MySQL/PostgreSQL）
|
||||
- ✅ 缓存系统（智能/手动/全同步）
|
||||
- ✅ 配置热重载
|
||||
- ✅ 优雅重启
|
||||
- ✅ Prometheus监控
|
||||
- ✅ 告警系统
|
||||
- ✅ 设备自动优化
|
||||
- ✅ 多镜像加速源（Docker/APT/YUM/PyPI/npm/Go）
|
||||
- ✅ Web管理界面
|
||||
|
||||
## 开发
|
||||
|
||||
### 添加新的API版本
|
||||
|
||||
1. 在 `api/` 目录下创建 `v3.py`
|
||||
2. 继承 `APIv2` 或 `APIv1`
|
||||
3. 实现所需方法
|
||||
4. 在 `api/router.py` 中注册新版本
|
||||
|
||||
### 添加新的同步类型
|
||||
|
||||
在 `core/mirror_sync.py` 的 `MirrorSyncManager` 类中：
|
||||
1. 添加 `_sync_{type}` 方法
|
||||
2. 实现同步逻辑
|
||||
3. 在 `_sync_worker` 中添加路由
|
||||
|
||||
## 构建和部署
|
||||
|
||||
### 使用 PyInstaller 打包成二进制文件
|
||||
|
||||
#### 构建当前平台
|
||||
|
||||
项目提供了 `build.py` 脚本，可以将项目打包成当前平台的二进制可执行文件。
|
||||
|
||||
```bash
|
||||
# 安装 PyInstaller
|
||||
pip install pyinstaller
|
||||
|
||||
# 查看支持的目标平台
|
||||
python build.py --list
|
||||
|
||||
# 构建当前平台
|
||||
python build.py
|
||||
|
||||
# 清理构建缓存
|
||||
python build.py --clean
|
||||
```
|
||||
|
||||
**支持的平台（仅当前平台）：**
|
||||
- Linux: amd64, arm64, 386
|
||||
- Windows: amd64, 386
|
||||
- macOS: amd64, arm64
|
||||
|
||||
构建完成后，二进制文件和发布包会输出到 `dist/` 目录。
|
||||
|
||||
#### 跨平台构建（GitHub Actions）
|
||||
|
||||
使用 GitHub Actions 可以方便地构建多平台版本（Linux ARM、AMD64 等）：
|
||||
|
||||
```bash
|
||||
# 1. 初始化 git 仓库
|
||||
git init
|
||||
git add .
|
||||
git commit -m "Add build workflow"
|
||||
|
||||
# 2. 添加远程仓库（替换为你的仓库地址）
|
||||
git remote add origin https://github.com/your-username/your-repo.git
|
||||
git branch -M main
|
||||
git push -u origin main
|
||||
```
|
||||
|
||||
推送完成后：
|
||||
|
||||
1. 访问 GitHub 仓库
|
||||
2. 点击 **Actions** 标签
|
||||
3. 选择 **"Build Linux ARM Binaries"**
|
||||
4. 点击 **"Run workflow"**
|
||||
5. 等待 5-10 分钟后下载构建产物
|
||||
|
||||
**优势：**
|
||||
- ✅ 完全免费
|
||||
- ✅ 无需本地配置
|
||||
- ✅ 支持多平台构建
|
||||
- ✅ 自动化持续集成
|
||||
|
||||
详细说明请参考 [BUILD.md](../BUILD.md)
|
||||
|
||||
#### 快速开始
|
||||
|
||||
**Linux/macOS:**
|
||||
```bash
|
||||
# 给脚本添加执行权限
|
||||
chmod +x docker-deploy.sh
|
||||
|
||||
# 构建并运行
|
||||
./docker-deploy.sh build
|
||||
./docker-deploy.sh run
|
||||
|
||||
# 使用 docker-compose
|
||||
./docker-deploy.sh compose
|
||||
```
|
||||
|
||||
**Windows:**
|
||||
```batch
|
||||
docker-deploy.bat build
|
||||
docker-deploy.bat run
|
||||
```
|
||||
|
||||
### HTTPS 支持
|
||||
|
||||
#### 使用 Nginx 反向代理
|
||||
|
||||
1. 创建 SSL 证书目录并放置证书
|
||||
```bash
|
||||
mkdir -p ssl
|
||||
# 将 cert.pem 和 key.pem 放入 ssl/ 目录
|
||||
```
|
||||
|
||||
2. 使用带 Nginx 的 docker-compose 启动
|
||||
```bash
|
||||
docker-compose --profile with-nginx up -d
|
||||
```
|
||||
|
||||
#### 直接启用 HTTPS
|
||||
|
||||
```bash
|
||||
python main.py --ssl-cert cert.pem --ssl-key key.pem
|
||||
```
|
||||
|
||||
详细说明请参考 [DOCKER.md](DOCKER.md)
|
||||
|
||||
## CI/CD 自动构建
|
||||
|
||||
项目使用 GitHub Actions 自动构建 Docker 镜像和可执行文件。
|
||||
|
||||
### GitHub Actions 工作流
|
||||
|
||||
| 工作流文件 | 触发条件 | 功能 |
|
||||
|------------|----------|------|
|
||||
| `.github/workflows/releases.yml` | 打 Tag (`v*`) | 构建并发布二进制可执行文件到 GitHub Releases |
|
||||
| `.github/workflows/docker-multiarch.yml` | 打 Tag (`v*`) | 构建并推送多架构 Docker 镜像到 Docker Hub |
|
||||
| `.github/workflows/ci-cd.yml` | push main / PR | 运行测试 |
|
||||
|
||||
### 发布新版本
|
||||
|
||||
```bash
|
||||
# 1. 推送所有代码
|
||||
git push origin main
|
||||
|
||||
# 2. 创建版本标签
|
||||
git tag v1.0.0
|
||||
|
||||
# 3. 推送到远程
|
||||
git push origin v1.0.0
|
||||
```
|
||||
|
||||
推送 Tag 后，GitHub Actions 会自动：
|
||||
- ✅ 构建 4 个架构的二进制文件（Linux AMD64/ARM64/ARMv7 + Windows AMD64）
|
||||
- ✅ 构建 3 个架构的 Docker 镜像（AMD64/ARM64/ARMv7）
|
||||
- ✅ 上传到 GitHub Releases
|
||||
- ✅ 推送到 Docker Hub 和 GitHub Container Registry
|
||||
|
||||
### GitHub Secrets 配置
|
||||
|
||||
需要配置以下 Secrets 才能正常工作：
|
||||
|
||||
| Secret 名称 | 值 | 说明 |
|
||||
|-------------|-----|------|
|
||||
| `DOCKERHUB_USERNAME` | Docker Hub 用户名 | |
|
||||
| `DOCKERHUB_TOKEN` | Docker Hub Access Token | |
|
||||
|
||||
### Docker Hub 镜像
|
||||
|
||||
构建完成后，镜像会自动推送到：
|
||||
- **Docker Hub**: `hx100cv/hyc-download:latest`
|
||||
- **GitHub Container Registry**: `ghcr.io/hexikx/mirror_server:latest`
|
||||
|
||||
### GitHub Releases
|
||||
|
||||
每个版本会生成以下文件：
|
||||
|
||||
| 文件名 | 说明 |
|
||||
|--------|------|
|
||||
| `hyc-download-linux-amd64` | Linux x64 可执行文件 |
|
||||
| `hyc-download-linux-arm64` | Linux ARM64 可执行文件 |
|
||||
| `hyc-download-linux-armv7` | Linux ARMv7 可执行文件 |
|
||||
| `hyc-download-windows-amd64.exe` | Windows x64 可执行文件 |
|
||||
|
||||
## Stargazers over time
|
||||
|
||||
[](https://starchart.cc/HEXIkx/mirror_server)
|
||||
|
||||
## Contributors
|
||||
|
||||
[](https://github.com/HEXIkx/mirror_server/graphs/contributors)
|
||||
|
||||
## License
|
||||
|
||||
|
||||
[The MIT License (MIT)](https://github.com/HEXIkx/mirror_server/blob/main/LICENSE)
|
||||
@@ -0,0 +1,6 @@
|
||||
# API模块初始化
|
||||
from .router import APIRouter
|
||||
from .v1 import APIv1
|
||||
from .v2 import APIv2
|
||||
|
||||
__all__ = ['APIRouter', 'APIv1', 'APIv2']
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
管理员API处理器
|
||||
提供认证相关API（无 keys 管理）
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from core.api_auth import APIAuthManager, require_auth
|
||||
|
||||
|
||||
class AdminAPI:
|
||||
"""管理员API处理器"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
self.auth_manager = APIAuthManager(config)
|
||||
|
||||
def handle_request(self, handler, method, path, query_params):
|
||||
"""处理管理员API请求"""
|
||||
|
||||
# 解析路径
|
||||
parts = path.strip('/').split('/')
|
||||
|
||||
# 根路径 - 列出API
|
||||
if len(parts) == 0 or parts[0] == '':
|
||||
self._api_overview(handler)
|
||||
return
|
||||
|
||||
action = parts[0]
|
||||
|
||||
if action == 'auth':
|
||||
self._handle_auth(handler, method, query_params)
|
||||
elif action == 'sessions':
|
||||
self._handle_sessions(handler, method, parts[1:] if len(parts) > 1 else [])
|
||||
elif action == 'stats':
|
||||
self._handle_stats(handler)
|
||||
else:
|
||||
handler.send_json_response({
|
||||
"error": f"Unknown admin action: {action}",
|
||||
"available_actions": ["auth", "sessions", "stats"]
|
||||
}, 404)
|
||||
|
||||
def _api_overview(self, handler):
|
||||
"""API概览"""
|
||||
handler.send_json_response({
|
||||
"name": "HYC Admin API",
|
||||
"version": "1.0",
|
||||
"description": "管理员认证API",
|
||||
"endpoints": {
|
||||
"GET /api/v2/admin/sessions": "列出活跃会话",
|
||||
"DELETE /api/v2/admin/sessions/{session_id}": "销毁会话",
|
||||
"POST /api/v2/admin/auth/verify": "验证认证状态",
|
||||
"GET /api/v2/admin/stats": "获取认证统计"
|
||||
},
|
||||
"authentication": {
|
||||
"methods": [
|
||||
"Authorization: Bearer <token>",
|
||||
"X-API-Key: <token>",
|
||||
"Cookie: hyc_auth=<session>",
|
||||
"?key=<token>"
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
def _handle_auth(self, handler, method, query_params):
|
||||
"""处理认证相关"""
|
||||
# POST /api/v2/admin/auth/verify - 验证当前认证状态
|
||||
if method == 'POST':
|
||||
auth_result = handler.auth_result if hasattr(handler, 'auth_result') else {}
|
||||
|
||||
if auth_result.get('authenticated'):
|
||||
handler.send_json_response({
|
||||
"authenticated": True,
|
||||
"level": auth_result.get('level'),
|
||||
"key_id": auth_result.get('key_id'),
|
||||
"name": auth_result.get('name'),
|
||||
"permissions": auth_result.get('permissions', [])
|
||||
})
|
||||
else:
|
||||
handler.send_json_response({
|
||||
"authenticated": False
|
||||
}, 401)
|
||||
|
||||
else:
|
||||
handler.send_json_response({"error": "Invalid method"}, 405)
|
||||
|
||||
def _handle_sessions(self, handler, method, parts):
|
||||
"""处理会话管理"""
|
||||
if method == 'GET':
|
||||
# 列出活跃会话
|
||||
sessions = []
|
||||
for session_id, session in self.auth_manager.sessions.items():
|
||||
if time.time() < session.expires_at:
|
||||
sessions.append({
|
||||
"session_id": session.session_id,
|
||||
"user_id": session.user_id,
|
||||
"level": session.level,
|
||||
"created_at": session.created_at,
|
||||
"expires_at": session.expires_at,
|
||||
"last_activity": session.last_activity
|
||||
})
|
||||
|
||||
handler.send_json_response({
|
||||
"sessions": sessions,
|
||||
"count": len(sessions)
|
||||
})
|
||||
|
||||
elif method == 'DELETE' and len(parts) >= 1 and parts[0]:
|
||||
session_id = parts[0]
|
||||
success = self.auth_manager.destroy_session(session_id)
|
||||
if success:
|
||||
handler.send_json_response({
|
||||
"success": True,
|
||||
"message": f"Session {session_id} destroyed"
|
||||
})
|
||||
else:
|
||||
handler.send_json_response({"error": "Session not found"}, 404)
|
||||
|
||||
else:
|
||||
handler.send_json_response({"error": "Invalid request"}, 400)
|
||||
|
||||
def _handle_stats(self, handler):
|
||||
"""获取认证统计"""
|
||||
stats = self.auth_manager.get_stats()
|
||||
handler.send_json_response(stats)
|
||||
@@ -0,0 +1,282 @@
|
||||
# 低端设备优化指南
|
||||
|
||||
## 支持的架构
|
||||
|
||||
| 架构 | 状态 | 说明 |
|
||||
|------|------|------|
|
||||
| X86_64/AMD64 | ✓ 完全支持 | 主流 PC/服务器 |
|
||||
| X86_32/i386/i686 | ✓ 支持 | 老旧 PC 设备 |
|
||||
| ARM64/AArch64 | ✓ 完全支持 | 树莓派4/5, Jetson |
|
||||
| ARMv7/armhf | ✓ 支持 | 树莓派3/2, Orange Pi |
|
||||
| ARMv6 | ⚠ 实验性 | 树莓派Zero |
|
||||
| MIPS | ⚠ 实验性 | 路由器等 |
|
||||
|
||||
## 设备预设
|
||||
|
||||
### ultra_low (极低端设备)
|
||||
- **内存**: < 256MB RAM
|
||||
- **示例**: 树莓派 Zero, 老旧路由器
|
||||
- **配置**:
|
||||
```bash
|
||||
python main.py --preset ultra_low
|
||||
```
|
||||
- **自动设置**:
|
||||
- Workers: 1
|
||||
- 最大缓存: 50MB
|
||||
- 块大小: 32KB
|
||||
- 禁用: WebSocket, SSE, 哈希计算
|
||||
|
||||
### low (低端设备)
|
||||
- **内存**: 256MB - 512MB RAM
|
||||
- **示例**: 树莓派 2, 老旧 VPS
|
||||
- **配置**:
|
||||
```bash
|
||||
python main.py --preset low
|
||||
```
|
||||
- **自动设置**:
|
||||
- Workers: 1
|
||||
- 最大缓存: 100MB
|
||||
- 块大小: 64KB
|
||||
- 启用所有功能
|
||||
|
||||
### medium (中等设备)
|
||||
- **内存**: 512MB - 1GB RAM
|
||||
- **示例**: 树莓派 4 (1GB), 低配 VPS
|
||||
- **配置**:
|
||||
```bash
|
||||
python main.py --preset medium
|
||||
```
|
||||
- **自动设置**:
|
||||
- Workers: 2
|
||||
- 最大缓存: 256MB
|
||||
- 块大小: 128KB
|
||||
|
||||
### high (高端设备)
|
||||
- **内存**: 1GB+ RAM
|
||||
- **示例**: 树莓派 4 (4GB/8GB), 家用服务器
|
||||
- **配置**:
|
||||
```bash
|
||||
python main.py --preset high
|
||||
```
|
||||
- **自动设置**:
|
||||
- Workers: 4
|
||||
- 最大缓存: 512MB
|
||||
- 块大小: 256KB
|
||||
|
||||
### auto (自动检测)
|
||||
- 根据系统资源自动选择预设
|
||||
- **配置**:
|
||||
```bash
|
||||
python main.py --preset auto # 默认
|
||||
```
|
||||
|
||||
## 手动配置
|
||||
|
||||
### 内存限制
|
||||
```bash
|
||||
# 设置 256MB 内存限制
|
||||
python main.py --memory-limit 256M
|
||||
|
||||
# 设置 512MB 内存限制
|
||||
python main.py --memory-limit 512M
|
||||
```
|
||||
|
||||
### 工作进程
|
||||
```bash
|
||||
# 单进程 (低端设备)
|
||||
python main.py --workers 1
|
||||
|
||||
# 双进程
|
||||
python main.py --workers 2
|
||||
```
|
||||
|
||||
### 传输优化
|
||||
```bash
|
||||
# 小块传输 (节省内存)
|
||||
python main.py --chunk-size 32K --buffer-size 64K
|
||||
```
|
||||
|
||||
### 禁用可选功能
|
||||
```bash
|
||||
# 禁用 WebSocket 和 SSE (节省内存)
|
||||
python main.py --disable-ws --disable-sse
|
||||
|
||||
# 禁用哈希计算 (节省 CPU)
|
||||
python main.py --disable-hash
|
||||
```
|
||||
|
||||
## 树莓派部署
|
||||
|
||||
### 方式一: 使用预编译镜像
|
||||
```bash
|
||||
# 拉取 ARM64 镜像
|
||||
docker pull hx100cv/hyc-download-station:v2.3-arm64
|
||||
|
||||
# 运行
|
||||
docker run -d \
|
||||
--name hyc-server \
|
||||
-p 8080:8080 \
|
||||
-v ./data:/data \
|
||||
-v ./downloads:/downloads \
|
||||
hyc-download-station:v2.3-arm64
|
||||
```
|
||||
|
||||
### 方式二: 使用 Docker Compose
|
||||
```bash
|
||||
# 树莓派专用配置
|
||||
docker-compose -f docker-compose.raspberry.yml up -d
|
||||
```
|
||||
|
||||
### 方式三: 轻量级配置
|
||||
```bash
|
||||
# 适用于 512MB RAM 的树莓派
|
||||
docker-compose -f docker-compose.lite.yml up -d
|
||||
```
|
||||
|
||||
## 系统兼容性检查
|
||||
|
||||
```bash
|
||||
# 检查系统兼容性
|
||||
python main.py --check-compat
|
||||
|
||||
# 或使用脚本
|
||||
./scripts/check-compat.sh
|
||||
```
|
||||
|
||||
输出示例:
|
||||
```
|
||||
========================================
|
||||
HYC下载站 v2.3 - 兼容性检查
|
||||
========================================
|
||||
|
||||
系统信息:
|
||||
- 架构: aarch64
|
||||
- 系统: Linux
|
||||
|
||||
✓ ARM64 (64位) - 完全支持
|
||||
✓ Python 3.11 - 支持
|
||||
|
||||
内存检查:
|
||||
- 总内存: 4096MB
|
||||
✓ 内存 1GB+ - 使用 high 预设
|
||||
|
||||
...
|
||||
|
||||
推荐启动命令:
|
||||
python main.py --preset high
|
||||
```
|
||||
|
||||
## Docker 多架构构建
|
||||
|
||||
### 环境准备
|
||||
```bash
|
||||
# 设置 QEMU 仿真 (x86_64 上构建 ARM)
|
||||
./scripts/setup-qemu.sh
|
||||
```
|
||||
|
||||
### 构建镜像
|
||||
```bash
|
||||
# 构建所有架构
|
||||
./scripts/build-multiarch.sh v2.3 hyc-download-station
|
||||
|
||||
# 或手动构建
|
||||
docker buildx build \
|
||||
--platform linux/amd64,linux/arm64,linux/arm/v7 \
|
||||
--tag hx100cv/hyc-download-station:v2.3 \
|
||||
--file docker/Dockerfile.multiarch \
|
||||
--push .
|
||||
```
|
||||
|
||||
### 手动构建特定架构
|
||||
```bash
|
||||
# ARMv7
|
||||
docker build \
|
||||
--platform linux/arm/v7 \
|
||||
--tag hx100cv/hyc-download-station:v2.3-armv7 \
|
||||
--file docker/Dockerfile.lite \
|
||||
--push .
|
||||
|
||||
# ARM64
|
||||
docker build \
|
||||
--platform linux/arm64 \
|
||||
--tag hx100cv/hyc-download-station:v2.3-arm64 \
|
||||
--file docker/Dockerfile.lite \
|
||||
--push .
|
||||
|
||||
# i386 (32位)
|
||||
docker build \
|
||||
--platform linux/386 \
|
||||
--tag hx100cv/hyc-download-station:v2.3-i386 \
|
||||
--file docker/Dockerfile.lite \
|
||||
--push .
|
||||
```
|
||||
|
||||
## 性能调优建议
|
||||
|
||||
### 树莓派 4 (4GB)
|
||||
```bash
|
||||
# 推荐配置
|
||||
python main.py \
|
||||
--preset high \
|
||||
--memory-limit 1G \
|
||||
--workers 2
|
||||
```
|
||||
|
||||
### 树莓派 3
|
||||
```bash
|
||||
# 推荐配置
|
||||
python main.py \
|
||||
--preset medium \
|
||||
--memory-limit 512M \
|
||||
--workers 2
|
||||
```
|
||||
|
||||
### 树莓派 2/Zero
|
||||
```bash
|
||||
# 推荐配置
|
||||
python main.py \
|
||||
--preset low \
|
||||
--memory-limit 256M \
|
||||
--workers 1 \
|
||||
--chunk-size 16K \
|
||||
--disable-ws \
|
||||
--disable-sse
|
||||
```
|
||||
|
||||
## 内存使用监控
|
||||
|
||||
启动后可以通过 API 查看内存使用:
|
||||
```bash
|
||||
# 查看内存状态
|
||||
curl http://localhost:8080/api/v1/monitor
|
||||
|
||||
# 或在 Web 界面查看
|
||||
# 访问 http://localhost:8080/api/ui/
|
||||
```
|
||||
|
||||
## 故障排除
|
||||
|
||||
### 内存不足
|
||||
```
|
||||
症状: OOM (Out of Memory) 错误
|
||||
解决:
|
||||
1. 使用 --preset ultra_low
|
||||
2. 减小 --memory-limit
|
||||
3. 增加 swap 空间
|
||||
```
|
||||
|
||||
### 构建失败
|
||||
```
|
||||
症状: 无法导入模块
|
||||
解决:
|
||||
1. 重新安装依赖: pip install -r requirements.txt
|
||||
2. 检查 Python 版本: python3 --version
|
||||
```
|
||||
|
||||
### ARM 镜像运行失败
|
||||
```
|
||||
症状: Illegal instruction
|
||||
解决:
|
||||
1. 确保使用正确的架构镜像
|
||||
2. 检查 QEMU 设置
|
||||
```
|
||||
+2055
File diff suppressed because it is too large
Load Diff
+1066
File diff suppressed because it is too large
Load Diff
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""API路由模块"""
|
||||
|
||||
import re
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
from .v1 import APIv1
|
||||
from .v2 import APIv2
|
||||
from .admin import AdminAPI
|
||||
from core.api_auth import APIAuthManager
|
||||
|
||||
|
||||
class APIRouter:
|
||||
"""API路由器 - 支持版本化API"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.api_versions = {
|
||||
'v1': APIv1(config),
|
||||
'v2': APIv2(config)
|
||||
}
|
||||
self.default_version = config.get('api_version', 'v1')
|
||||
|
||||
# 初始化认证管理器（使用共享实例）
|
||||
if not config.get('_auth_manager'):
|
||||
config['_auth_manager'] = APIAuthManager(config)
|
||||
self.auth_manager = config['_auth_manager']
|
||||
|
||||
# 初始化Admin API
|
||||
self.admin_api = AdminAPI(config)
|
||||
|
||||
def handle_request(self, handler, method, path, query):
|
||||
"""处理API请求"""
|
||||
# 注入认证管理器到handler
|
||||
handler.auth_manager = self.auth_manager
|
||||
|
||||
# 解析路径，提取API版本
|
||||
# 格式: /api/v1/... 或 /api/v2/... 或 /api/...
|
||||
|
||||
# 移除 /api/ 前缀
|
||||
if path.startswith('api/'):
|
||||
api_path = path[4:]
|
||||
else:
|
||||
api_path = path
|
||||
|
||||
# 解析版本号
|
||||
parts = api_path.split('/')
|
||||
if parts[0] in ['v1', 'v2']:
|
||||
api_version = parts[0]
|
||||
api_action = '/'.join(parts[1:]) if len(parts) > 1 else ''
|
||||
else:
|
||||
# 检查是否是直接访问的 admin API (不带版本前缀)
|
||||
if api_path.startswith('admin/'):
|
||||
admin_action = api_path[6:] # 移除 'admin/'
|
||||
try:
|
||||
self.admin_api.handle_request(handler, method, admin_action, {})
|
||||
except Exception as e:
|
||||
handler.send_json_response({
|
||||
"error": f"Admin API处理错误: {str(e)}",
|
||||
"path": admin_action
|
||||
}, 500)
|
||||
return
|
||||
else:
|
||||
handler.send_error(400, "未指定API请求版本/指定版本错误")
|
||||
return
|
||||
|
||||
# 解析查询参数
|
||||
parsed_query = parse_qs(query)
|
||||
|
||||
# 检查是否是 admin API (带版本前缀，如 /api/v2/admin/stats)
|
||||
# 注意：auth/verify 应该交给 APIv2 处理，而不是 admin_api
|
||||
if api_version in ['v1', 'v2'] and api_action.startswith('admin/') and not api_action.startswith('admin/auth'):
|
||||
admin_action = api_action[6:] # 移除 'admin/'
|
||||
try:
|
||||
self.admin_api.handle_request(handler, method, admin_action, parsed_query)
|
||||
except Exception as e:
|
||||
handler.send_json_response({
|
||||
"error": f"Admin API处理错误: {str(e)}",
|
||||
"path": admin_action
|
||||
}, 500)
|
||||
return
|
||||
|
||||
# 获取对应的API处理器
|
||||
api_handler = self.api_versions.get(api_version)
|
||||
if not api_handler:
|
||||
handler.send_json_response({
|
||||
"error": f"不支持的API版本: {api_version}",
|
||||
"supported_versions": list(self.api_versions.keys())
|
||||
}, 400)
|
||||
return
|
||||
|
||||
# 调用对应的API处理器
|
||||
try:
|
||||
# 调试模式输出路由信息 (debug-api)
|
||||
if handler._is_debug_enabled('api'):
|
||||
msg = f"\n=== DEBUG API Router ===\n Version: {api_version}\n Action: {api_action}\n Method: {method}\n Query: {parsed_query}"
|
||||
handler._debug_log('api', msg, '\033[35m')
|
||||
|
||||
api_handler.handle_request(handler, method, api_action, parsed_query)
|
||||
except Exception as e:
|
||||
if handler._is_debug_enabled('error'):
|
||||
import traceback
|
||||
tb_str = traceback.format_exc()
|
||||
msg = f"\n=== DEBUG API ERROR ===\n{tb_str}"
|
||||
handler._debug_log('error', msg, '\033[31m')
|
||||
handler.send_json_response({
|
||||
"error": f"API处理错误: {str(e)}",
|
||||
"version": api_version,
|
||||
"path": api_action
|
||||
}, 500)
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
SSE (Server-Sent Events) 处理器模块
|
||||
提供单向事件推送功能
|
||||
"""
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from typing import Dict, Set, Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class SSEClient:
|
||||
"""SSE客户端"""
|
||||
client_id: str
|
||||
handler: any # HTTP请求处理器
|
||||
topics: Set[str] = field(default_factory=set)
|
||||
connected_at: float = field(default_factory=time.time)
|
||||
last_activity: float = field(default_factory=time.time)
|
||||
running: bool = False
|
||||
|
||||
|
||||
class SSEHandler:
|
||||
"""SSE事件处理器"""
|
||||
|
||||
# 预定义事件类型
|
||||
EVENT_STATS = 'stats'
|
||||
EVENT_MONITOR = 'monitor'
|
||||
EVENT_SYNC = 'sync'
|
||||
EVENT_DOWNLOAD = 'download'
|
||||
EVENT_SERVER = 'server'
|
||||
EVENT_ERROR = 'error'
|
||||
EVENT_PING = 'ping'
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.clients: Dict[str, SSEClient] = {}
|
||||
self.lock = threading.Lock()
|
||||
|
||||
# 心跳配置
|
||||
self.heartbeat_interval = 30 # 秒
|
||||
self.retry_interval = 3000 # 毫秒（客户端重连间隔）
|
||||
|
||||
# 消息缓冲区
|
||||
self.message_buffer_size = 100
|
||||
|
||||
# 统计
|
||||
self.stats = {
|
||||
'total_connections': 0,
|
||||
'total_events_sent': 0,
|
||||
'total_bytes_sent': 0
|
||||
}
|
||||
|
||||
def handle_connection(self, handler, topics: list = None) -> Optional[str]:
|
||||
"""
|
||||
处理新的SSE连接
|
||||
返回客户端ID
|
||||
"""
|
||||
client_id = self._generate_client_id()
|
||||
|
||||
# 设置SSE响应头
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/event-stream')
|
||||
handler.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
|
||||
handler.send_header('Connection', 'keep-alive')
|
||||
handler.send_header('Access-Control-Allow-Origin', '*')
|
||||
handler.send_header('X-Accel-Buffering', 'no') # 禁用Nginx缓冲
|
||||
handler.end_headers()
|
||||
|
||||
# 创建客户端
|
||||
client = SSEClient(
|
||||
client_id=client_id,
|
||||
handler=handler,
|
||||
topics=set(topics) if topics else {self.EVENT_STATS, self.EVENT_SYNC}
|
||||
)
|
||||
|
||||
with self.lock:
|
||||
self.clients[client_id] = client
|
||||
self.stats['total_connections'] += 1
|
||||
|
||||
# 启动心跳
|
||||
client.running = True
|
||||
self._start_heartbeat(client_id)
|
||||
|
||||
# 发送初始连接事件
|
||||
self._send_event(client, self.EVENT_SERVER, {
|
||||
'type': 'connected',
|
||||
'client_id': client_id,
|
||||
'timestamp': time.time(),
|
||||
'topics': list(client.topics)
|
||||
})
|
||||
|
||||
return client_id
|
||||
|
||||
def close_connection(self, client_id: str):
|
||||
"""关闭连接"""
|
||||
with self.lock:
|
||||
client = self.clients.pop(client_id, None)
|
||||
if client:
|
||||
client.running = False
|
||||
|
||||
def subscribe(self, client_id: str, *topics: str):
|
||||
"""订阅主题"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if client:
|
||||
client.topics.update(topics)
|
||||
|
||||
def unsubscribe(self, client_id: str, *topics: str):
|
||||
"""取消订阅"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if client:
|
||||
for topic in topics:
|
||||
client.topics.discard(topic)
|
||||
|
||||
def is_subscribed(self, client_id: str, event_type: str) -> bool:
|
||||
"""检查是否订阅了事件类型"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if not client:
|
||||
return False
|
||||
return event_type in client.topics or '*' in client.topics
|
||||
|
||||
def send_event(self, client_id: str, event_type: str, data: dict):
|
||||
"""发送事件到指定客户端"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if not client or not client.running:
|
||||
return False
|
||||
|
||||
if event_type not in client.topics and '*' not in client.topics:
|
||||
return False
|
||||
|
||||
return self._send_event(client, event_type, data)
|
||||
|
||||
def broadcast(self, event_type: str, data: dict, topic: str = None):
|
||||
"""广播事件到所有客户端"""
|
||||
with self.lock:
|
||||
sent_count = 0
|
||||
dead_clients = []
|
||||
|
||||
for client_id, client in self.clients.items():
|
||||
if not client.running:
|
||||
dead_clients.append(client_id)
|
||||
continue
|
||||
|
||||
# 检查主题匹配
|
||||
if topic and event_type != topic:
|
||||
continue
|
||||
|
||||
if self._send_event(client, event_type, data):
|
||||
sent_count += 1
|
||||
|
||||
# 清理死掉的客户端
|
||||
for client_id in dead_clients:
|
||||
self.clients.pop(client_id, None)
|
||||
|
||||
return sent_count
|
||||
|
||||
def broadcast_to_topic(self, topic: str, event_type: str, data: dict):
|
||||
"""广播到订阅特定主题的客户端"""
|
||||
self.broadcast(event_type, data, topic)
|
||||
|
||||
def send_monitor_update(self, stats: dict):
|
||||
"""发送监控更新"""
|
||||
self.broadcast(self.EVENT_MONITOR, stats)
|
||||
|
||||
def send_sync_update(self, sync_data: dict):
|
||||
"""发送同步更新"""
|
||||
self.broadcast(self.EVENT_SYNC, sync_data)
|
||||
|
||||
def send_download_update(self, download_data: dict):
|
||||
"""发送下载更新"""
|
||||
self.broadcast(self.EVENT_DOWNLOAD, download_data)
|
||||
|
||||
def send_server_event(self, event_data: dict):
|
||||
"""发送服务器事件"""
|
||||
self.broadcast(self.EVENT_SERVER, event_data)
|
||||
|
||||
def get_client_count(self) -> int:
|
||||
"""获取客户端数量"""
|
||||
with self.lock:
|
||||
return len(self.clients)
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""获取统计信息"""
|
||||
with self.lock:
|
||||
return {
|
||||
**self.stats,
|
||||
'connected_clients': len(self.clients),
|
||||
'topics': list(set(
|
||||
t for client in self.clients.values()
|
||||
for t in client.topics
|
||||
))
|
||||
}
|
||||
|
||||
def _send_event(self, client: SSEClient, event_type: str, data: dict) -> bool:
|
||||
"""发送单个事件"""
|
||||
try:
|
||||
event_data = {
|
||||
'event': event_type,
|
||||
'timestamp': time.time(),
|
||||
'data': data
|
||||
}
|
||||
|
||||
# SSE格式
|
||||
message = f"event: {event_type}\n"
|
||||
message += f"id: {uuid.uuid4().hex[:16]}\n"
|
||||
message += f"retry: {self.retry_interval}\n"
|
||||
message += "data: " + json.dumps(event_data, ensure_ascii=False) + "\n\n"
|
||||
|
||||
# 发送
|
||||
client.handler.wfile.write(message.encode('utf-8'))
|
||||
client.handler.wfile.flush()
|
||||
|
||||
# 统计
|
||||
self.stats['total_events_sent'] += len(message)
|
||||
client.last_activity = time.time()
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"SSE发送失败 {client.client_id}: {e}")
|
||||
client.running = False
|
||||
return False
|
||||
|
||||
def _start_heartbeat(self, client_id: str):
|
||||
"""启动心跳"""
|
||||
def heartbeat():
|
||||
while True:
|
||||
time.sleep(self.heartbeat_interval)
|
||||
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if not client or not client.running:
|
||||
return
|
||||
|
||||
# 发送心跳
|
||||
self.send_event(client_id, self.EVENT_PING, {
|
||||
'timestamp': time.time()
|
||||
})
|
||||
|
||||
thread = threading.Thread(target=heartbeat, daemon=True)
|
||||
thread.start()
|
||||
|
||||
def _generate_client_id(self) -> str:
|
||||
"""生成客户端ID"""
|
||||
return f"sse_{uuid.uuid4().hex[:12]}"
|
||||
|
||||
def cleanup(self, max_idle_time: float = 300.0):
|
||||
"""清理空闲连接"""
|
||||
now = time.time()
|
||||
idle_clients = []
|
||||
|
||||
with self.lock:
|
||||
for client_id, client in self.clients.items():
|
||||
if now - client.last_activity > max_idle_time:
|
||||
idle_clients.append(client_id)
|
||||
|
||||
for client_id in idle_clients:
|
||||
self.send_event(client_id, self.EVENT_ERROR, {
|
||||
'type': 'timeout',
|
||||
'message': 'Connection timed out'
|
||||
})
|
||||
self.close_connection(client_id)
|
||||
|
||||
def get_client_info(self, client_id: str) -> Optional[dict]:
|
||||
"""获取客户端信息"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if not client:
|
||||
return None
|
||||
|
||||
return {
|
||||
'client_id': client.client_id,
|
||||
'topics': list(client.topics),
|
||||
'connected_at': client.connected_at,
|
||||
'last_activity': client.last_activity,
|
||||
'running': client.running
|
||||
}
|
||||
+5949
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,403 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
WebSocket处理器模块
|
||||
提供双向实时通信功能
|
||||
"""
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Dict, Set, Optional, Callable
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class WebSocketClient:
|
||||
"""WebSocket客户端"""
|
||||
client_id: str
|
||||
connection: any # WebSocket连接对象
|
||||
topics: Set[str] = field(default_factory=set)
|
||||
connected_at: float = field(default_factory=time.time)
|
||||
last_activity: float = field(default_factory=time.time)
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class WebSocketManager:
|
||||
"""WebSocket连接管理器"""
|
||||
|
||||
# 预定义主题
|
||||
TOPIC_MONITOR_CPU = 'monitor:cpu'
|
||||
TOPIC_MONITOR_MEMORY = 'monitor:memory'
|
||||
TOPIC_MONITOR_DISK = 'monitor:disk'
|
||||
TOPIC_MONITOR_NETWORK = 'monitor:network'
|
||||
TOPIC_MONITOR_ALL = 'monitor:*'
|
||||
TOPIC_SYNC_PROGRESS = 'sync:progress'
|
||||
TOPIC_SYNC_STATUS = 'sync:status'
|
||||
TOPIC_SYNC_ALL = 'sync:*'
|
||||
TOPIC_DOWNLOAD_PROGRESS = 'download:progress'
|
||||
TOPIC_DOWNLOAD_STATUS = 'download:status'
|
||||
TOPIC_DOWNLOAD_ALL = 'download:*'
|
||||
TOPIC_SERVER_STATUS = 'server:status'
|
||||
TOPIC_ALL = '*'
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.clients: Dict[str, WebSocketClient] = {}
|
||||
self.lock = threading.Lock()
|
||||
self._running = False
|
||||
|
||||
# 消息队列（用于批量发送）
|
||||
self.message_queues: Dict[str, list] = {}
|
||||
|
||||
# 心跳间隔（秒）
|
||||
self.heartbeat_interval = 30
|
||||
|
||||
# 统计
|
||||
self.stats = {
|
||||
'total_connections': 0,
|
||||
'total_messages_sent': 0,
|
||||
'total_messages_received': 0
|
||||
}
|
||||
|
||||
def register_client(self, client_id: str, connection, metadata: dict = None) -> WebSocketClient:
|
||||
"""注册新客户端"""
|
||||
with self.lock:
|
||||
client = WebSocketClient(
|
||||
client_id=client_id,
|
||||
connection=connection,
|
||||
topics={self.TOPIC_ALL}, # 默认订阅所有
|
||||
metadata=metadata or {}
|
||||
)
|
||||
self.clients[client_id] = client
|
||||
self.stats['total_connections'] += 1
|
||||
|
||||
# 启动心跳
|
||||
self._start_heartbeat(client_id)
|
||||
|
||||
return client
|
||||
|
||||
def unregister_client(self, client_id: str):
|
||||
"""注销客户端"""
|
||||
with self.lock:
|
||||
client = self.clients.pop(client_id, None)
|
||||
if client:
|
||||
self.message_queues.pop(client_id, None)
|
||||
|
||||
def subscribe(self, client_id: str, *topics: str):
|
||||
"""订阅主题"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if client:
|
||||
client.topics.update(topics)
|
||||
|
||||
def unsubscribe(self, client_id: str, *topics: str):
|
||||
"""取消订阅"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if client:
|
||||
for topic in topics:
|
||||
client.topics.discard(topic)
|
||||
|
||||
def is_subscribed(self, client_id: str, topic: str) -> bool:
|
||||
"""检查客户端是否订阅了主题"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if not client:
|
||||
return False
|
||||
|
||||
if self.TOPIC_ALL in client.topics:
|
||||
return True
|
||||
|
||||
# 检查精确匹配或通配符匹配
|
||||
if topic in client.topics:
|
||||
return True
|
||||
|
||||
# 检查通配符匹配
|
||||
topic_parts = topic.split(':')
|
||||
for subscribed_topic in client.topics:
|
||||
if subscribed_topic.endswith('*'):
|
||||
prefix = subscribed_topic.rstrip('*').rstrip(':')
|
||||
if topic.startswith(prefix):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def send_to_client(self, client_id: str, event_type: str, data: dict, callback: Callable = None):
|
||||
"""发送消息到指定客户端"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
if not client:
|
||||
return False
|
||||
|
||||
message = self._format_message(event_type, data)
|
||||
|
||||
try:
|
||||
if hasattr(client.connection, 'send'):
|
||||
client.connection.send(message)
|
||||
self.stats['total_messages_sent'] += 1
|
||||
client.last_activity = time.time()
|
||||
|
||||
if callback:
|
||||
callback(client_id, True)
|
||||
return True
|
||||
else:
|
||||
# 放入消息队列
|
||||
if client_id not in self.message_queues:
|
||||
self.message_queues[client_id] = []
|
||||
self.message_queues[client_id].append(message)
|
||||
|
||||
if callback:
|
||||
callback(client_id, True)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"WebSocket发送失败 {client_id}: {e}")
|
||||
if callback:
|
||||
callback(client_id, False)
|
||||
return False
|
||||
|
||||
def broadcast(self, event_type: str, data: dict, topic: str = None):
|
||||
"""广播消息到所有客户端"""
|
||||
with self.lock:
|
||||
sent_count = 0
|
||||
failed_clients = []
|
||||
|
||||
for client_id, client in self.clients.items():
|
||||
# 检查是否匹配主题
|
||||
if topic and not self.is_subscribed(client_id, topic):
|
||||
continue
|
||||
|
||||
message = self._format_message(event_type, data)
|
||||
|
||||
try:
|
||||
if hasattr(client.connection, 'send'):
|
||||
client.connection.send(message)
|
||||
sent_count += 1
|
||||
else:
|
||||
if client_id not in self.message_queues:
|
||||
self.message_queues[client_id] = []
|
||||
self.message_queues[client_id].append(message)
|
||||
sent_count += 1
|
||||
|
||||
client.last_activity = time.time()
|
||||
|
||||
except Exception as e:
|
||||
print(f"广播到 {client_id} 失败: {e}")
|
||||
failed_clients.append(client_id)
|
||||
|
||||
self.stats['total_messages_sent'] += sent_count
|
||||
|
||||
# 清理失败的客户端
|
||||
for client_id in failed_clients:
|
||||
self.unregister_client(client_id)
|
||||
|
||||
return sent_count
|
||||
|
||||
def broadcast_to_topic(self, topic: str, event_type: str, data: dict):
|
||||
"""广播到订阅特定主题的客户端"""
|
||||
with self.lock:
|
||||
sent_count = 0
|
||||
|
||||
for client_id, client in self.clients.items():
|
||||
if self.is_subscribed(client_id, topic):
|
||||
message = self._format_message(event_type, data)
|
||||
|
||||
try:
|
||||
if hasattr(client.connection, 'send'):
|
||||
client.connection.send(message)
|
||||
sent_count += 1
|
||||
else:
|
||||
if client_id not in self.message_queues:
|
||||
self.message_queues[client_id] = []
|
||||
self.message_queues[client_id].append(message)
|
||||
sent_count += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"发送到 {client_id} 失败: {e}")
|
||||
|
||||
return sent_count
|
||||
|
||||
def broadcast_monitor_update(self, stats: dict):
|
||||
"""广播监控更新"""
|
||||
# 提取关键指标
|
||||
cpu_percent = stats.get('cpu', {}).get('percent', 0)
|
||||
memory_percent = stats.get('memory', {}).get('percent', 0)
|
||||
disk_percent = stats.get('disk', {}).get('percent', 0)
|
||||
|
||||
# 按阈值过滤
|
||||
if cpu_percent > 0 or memory_percent > 0 or disk_percent > 0:
|
||||
self.broadcast('monitor:stats', stats, 'monitor:*')
|
||||
|
||||
def broadcast_sync_progress(self, task_id: str, progress: dict):
|
||||
"""广播同步进度"""
|
||||
self.broadcast('sync:progress', progress, 'sync:*')
|
||||
|
||||
def broadcast_download_progress(self, download_id: str, progress: dict):
|
||||
"""广播下载进度"""
|
||||
self.broadcast('download:progress', progress, 'download:*')
|
||||
|
||||
def get_client_count(self) -> int:
|
||||
"""获取客户端数量"""
|
||||
with self.lock:
|
||||
return len(self.clients)
|
||||
|
||||
def get_client_topics(self, client_id: str) -> Set[str]:
|
||||
"""获取客户端订阅的主题"""
|
||||
with self.lock:
|
||||
client = self.clients.get(client_id)
|
||||
return client.topics.copy() if client else set()
|
||||
|
||||
def get_all_topics(self) -> Set[str]:
|
||||
"""获取所有被订阅的主题"""
|
||||
with self.lock:
|
||||
topics = set()
|
||||
for client in self.clients.values():
|
||||
topics.update(client.topics)
|
||||
return topics
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""获取统计信息"""
|
||||
with self.lock:
|
||||
return {
|
||||
**self.stats,
|
||||
'connected_clients': len(self.clients),
|
||||
'active_topics': len(self.get_all_topics()),
|
||||
'queues_queued': sum(len(q) for q in self.message_queues.values())
|
||||
}
|
||||
|
||||
def _format_message(self, event_type: str, data: dict) -> str:
|
||||
"""格式化消息"""
|
||||
return json.dumps({
|
||||
'type': event_type,
|
||||
'timestamp': time.time(),
|
||||
'data': data
|
||||
}, ensure_ascii=False)
|
||||
|
||||
def _start_heartbeat(self, client_id: str):
|
||||
"""启动心跳"""
|
||||
def heartbeat():
|
||||
while client_id in self.clients:
|
||||
try:
|
||||
# 发送心跳
|
||||
self.send_to_client(
|
||||
client_id,
|
||||
'ping',
|
||||
{'timestamp': time.time()}
|
||||
)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
time.sleep(self.heartbeat_interval)
|
||||
|
||||
thread = threading.Thread(target=heartbeat, daemon=True)
|
||||
thread.start()
|
||||
|
||||
def generate_client_id(self) -> str:
|
||||
"""生成客户端ID"""
|
||||
return f"ws_{uuid.uuid4().hex[:12]}"
|
||||
|
||||
def process_messages(self, client_id: str, messages: list):
|
||||
"""处理客户端消息"""
|
||||
for message in messages:
|
||||
self._handle_message(client_id, message)
|
||||
|
||||
def _handle_message(self, client_id: str, message: str):
|
||||
"""处理客户端消息"""
|
||||
self.stats['total_messages_received'] += 1
|
||||
|
||||
try:
|
||||
data = json.loads(message)
|
||||
|
||||
event_type = data.get('type')
|
||||
payload = data.get('data', {})
|
||||
|
||||
if event_type == 'subscribe':
|
||||
# 订阅主题
|
||||
topics = payload.get('topics', [])
|
||||
self.subscribe(client_id, *topics)
|
||||
|
||||
elif event_type == 'unsubscribe':
|
||||
# 取消订阅
|
||||
topics = payload.get('topics', [])
|
||||
self.unsubscribe(client_id, *topics)
|
||||
|
||||
elif event_type == 'ping':
|
||||
# 心跳响应
|
||||
self.send_to_client(client_id, 'pong', {'timestamp': time.time()})
|
||||
|
||||
elif event_type == 'status':
|
||||
# 请求状态 - 返回服务器和客户端状态
|
||||
response_data = {'status': 'ok'}
|
||||
|
||||
if payload.get('monitor'):
|
||||
# 获取实时监控数据
|
||||
try:
|
||||
if handler.monitor:
|
||||
response_data['monitor'] = handler.monitor.get_realtime_stats()
|
||||
else:
|
||||
# 回退到直接使用 psutil
|
||||
import psutil
|
||||
response_data['monitor'] = {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'cpu': {
|
||||
'percent': psutil.cpu_percent(interval=0.1),
|
||||
'count': psutil.cpu_count()
|
||||
},
|
||||
'memory': psutil.virtual_memory()._asdict(),
|
||||
'disk': psutil.disk_usage(handler.config.get('base_dir', './downloads'))._asdict()
|
||||
}
|
||||
except Exception as e:
|
||||
response_data['monitor'] = {'error': str(e)}
|
||||
|
||||
if payload.get('ws'):
|
||||
# 获取 WebSocket 统计
|
||||
response_data['ws'] = self.get_stats()
|
||||
|
||||
if payload.get('client'):
|
||||
# 获取当前客户端信息
|
||||
client = self.clients.get(client_id)
|
||||
if client:
|
||||
response_data['client'] = {
|
||||
'client_id': client.client_id,
|
||||
'connected_at': client.connected_at,
|
||||
'last_activity': client.last_activity,
|
||||
'topics': list(client.topics),
|
||||
'metadata': client.metadata
|
||||
}
|
||||
|
||||
self.send_to_client(client_id, 'status:response', response_data)
|
||||
|
||||
elif event_type == 'sync':
|
||||
# 同步相关操作
|
||||
sync_action = payload.get('action', 'status')
|
||||
|
||||
if sync_action == 'status':
|
||||
# 获取同步状态
|
||||
response_data = {'action': 'status'}
|
||||
try:
|
||||
from core.sync_scheduler import SyncScheduler
|
||||
scheduler = SyncScheduler()
|
||||
response_data['sync_status'] = scheduler.get_status() if hasattr(scheduler, 'get_status') else {'message': 'sync scheduler running'}
|
||||
except Exception as e:
|
||||
response_data['sync_status'] = {'error': str(e)}
|
||||
|
||||
self.send_to_client(client_id, 'sync:response', response_data)
|
||||
|
||||
elif sync_action == 'list':
|
||||
# 获取同步任务列表
|
||||
response_data = {'action': 'list'}
|
||||
self.send_to_client(client_id, 'sync:response', response_data)
|
||||
|
||||
elif sync_action == 'trigger':
|
||||
# 触发手动同步
|
||||
response_data = {'action': 'trigger', 'status': 'pending'}
|
||||
self.send_to_client(client_id, 'sync:response', response_data)
|
||||
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"处理WebSocket消息失败: {e}")
|
||||
@@ -0,0 +1,15 @@
|
||||
# 核心模块初始化
|
||||
from .config import ConfigManager
|
||||
from .mirror_sync import MirrorSyncManager
|
||||
from .server import MirrorServer
|
||||
from .utils import format_file_size, get_file_hash, parse_size, sanitize_filename
|
||||
|
||||
__all__ = [
|
||||
'ConfigManager',
|
||||
'MirrorSyncManager',
|
||||
'MirrorServer',
|
||||
'format_file_size',
|
||||
'get_file_hash',
|
||||
'parse_size',
|
||||
'sanitize_filename'
|
||||
]
|
||||
+591
@@ -0,0 +1,591 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
告警模块
|
||||
支持邮件告警、Webhook 告警、告警规则引擎
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
import logging
|
||||
import smtplib
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Callable
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from enum import Enum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AlertSeverity(Enum):
|
||||
"""告警级别"""
|
||||
INFO = "info"
|
||||
WARNING = "warning"
|
||||
ERROR = "error"
|
||||
CRITICAL = "critical"
|
||||
|
||||
|
||||
class AlertType(Enum):
|
||||
"""告警类型"""
|
||||
DISK_HIGH = "disk_high"
|
||||
DISK_CRITICAL = "disk_critical"
|
||||
SYNC_FAILED = "sync_failed"
|
||||
SOURCE_UNHEALTHY = "source_unhealthy"
|
||||
CACHE_FULL = "cache_full"
|
||||
SERVICE_DOWN = "service_down"
|
||||
CUSTOM = "custom"
|
||||
|
||||
|
||||
class Alert:
|
||||
"""告警对象"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
alert_type: str,
|
||||
severity: AlertSeverity,
|
||||
title: str,
|
||||
message: str,
|
||||
details: Dict = None,
|
||||
source: str = None
|
||||
):
|
||||
self.id = f"{int(time.time())}_{threading.get_ident()}"
|
||||
self.type = alert_type
|
||||
self.severity = severity
|
||||
self.title = title
|
||||
self.message = message
|
||||
self.details = details or {}
|
||||
self.source = source
|
||||
self.timestamp = datetime.now()
|
||||
self.sent = False
|
||||
self.acknowledged = False
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
'id': self.id,
|
||||
'type': self.type,
|
||||
'severity': self.severity.value,
|
||||
'title': self.title,
|
||||
'message': self.message,
|
||||
'details': self.details,
|
||||
'source': self.source,
|
||||
'timestamp': self.timestamp.isoformat(),
|
||||
'sent': self.sent,
|
||||
'acknowledged': self.acknowledged
|
||||
}
|
||||
|
||||
|
||||
class EmailAlerter:
|
||||
"""邮件告警器"""
|
||||
|
||||
def __init__(self, config: Dict = None):
|
||||
"""
|
||||
初始化邮件告警器
|
||||
|
||||
Args:
|
||||
config: 邮件配置
|
||||
"""
|
||||
self.config = config or {}
|
||||
self.enabled = self.config.get('enabled', False)
|
||||
self.smtp_host = self.config.get('smtp_host', 'localhost')
|
||||
self.smtp_port = self.config.get('smtp_port', 587)
|
||||
self.smtp_user = self.config.get('smtp_user', '')
|
||||
self.smtp_password = self.config.get('smtp_password', '')
|
||||
self.from_address = self.config.get('from_address', 'hyc-mirror@localhost')
|
||||
self.to_addresses = self.config.get('to_addresses', [])
|
||||
self.use_tls = self.config.get('use_tls', True)
|
||||
|
||||
# 连接池
|
||||
self._connection: Optional[smtplib.SMTP] = None
|
||||
self._last_connect_time: Optional[datetime] = None
|
||||
self._connection_timeout = 30
|
||||
|
||||
def _get_connection(self) -> smtplib.SMTP:
|
||||
"""获取 SMTP 连接"""
|
||||
if self._connection:
|
||||
# 检查连接是否仍然有效
|
||||
try:
|
||||
self._connection.noop()
|
||||
return self._connection
|
||||
except Exception:
|
||||
try:
|
||||
self._connection.quit()
|
||||
except Exception:
|
||||
pass
|
||||
self._connection = None
|
||||
|
||||
# 创建新连接
|
||||
try:
|
||||
self._connection = smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=self._connection_timeout)
|
||||
if self.use_tls:
|
||||
self._connection.starttls()
|
||||
if self.smtp_user and self.smtp_password:
|
||||
self._connection.login(self.smtp_user, self.smtp_password)
|
||||
self._last_connect_time = datetime.now()
|
||||
return self._connection
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to SMTP server: {e}")
|
||||
raise
|
||||
|
||||
def send(self, alert: Alert) -> bool:
|
||||
"""
|
||||
发送告警邮件
|
||||
|
||||
Args:
|
||||
alert: 告警对象
|
||||
|
||||
Returns:
|
||||
是否发送成功
|
||||
"""
|
||||
if not self.enabled:
|
||||
logger.debug("Email alerts disabled")
|
||||
return False
|
||||
|
||||
if not self.to_addresses:
|
||||
logger.warning("No recipients configured for email alerts")
|
||||
return False
|
||||
|
||||
try:
|
||||
msg = MIMEMultipart('alternative')
|
||||
msg['Subject'] = f"[{alert.severity.value.upper()}] {alert.title}"
|
||||
msg['From'] = self.from_address
|
||||
msg['To'] = ', '.join(self.to_addresses)
|
||||
|
||||
# HTML 格式
|
||||
html_content = self._format_html(alert)
|
||||
msg.attach(MIMEText(html_content, 'html', 'utf-8'))
|
||||
|
||||
# 纯文本格式
|
||||
text_content = self._format_text(alert)
|
||||
msg.attach(MIMEText(text_content, 'plain', 'utf-8'))
|
||||
|
||||
# 发送邮件
|
||||
server = self._get_connection()
|
||||
server.send_message(msg)
|
||||
logger.info(f"Alert email sent: {alert.title}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send alert email: {e}")
|
||||
return False
|
||||
|
||||
def _format_html(self, alert: Alert) -> str:
|
||||
"""格式化 HTML 内容"""
|
||||
severity_colors = {
|
||||
'info': '#2196F3',
|
||||
'warning': '#FF9800',
|
||||
'error': '#F44336',
|
||||
'critical': '#9C27B0'
|
||||
}
|
||||
color = severity_colors.get(alert.severity.value, '#666666')
|
||||
|
||||
details_html = ''
|
||||
if alert.details:
|
||||
details_html = '<h3>Details</h3><table>'
|
||||
for key, value in alert.details.items():
|
||||
details_html += f'<tr><td><b>{key}:</b></td><td>{value}</td></tr>'
|
||||
details_html += '</table>'
|
||||
|
||||
return f"""
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
body {{ font-family: Arial, sans-serif; margin: 20px; }}
|
||||
.header {{ background-color: {color}; color: white; padding: 10px; }}
|
||||
.title {{ font-size: 24px; margin: 20px 0; }}
|
||||
.message {{ background-color: #f5f5f5; padding: 15px; border-radius: 5px; margin: 20px 0; }}
|
||||
.details {{ margin-top: 20px; }}
|
||||
.footer {{ margin-top: 30px; font-size: 12px; color: #666; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>HYC Mirror Alert</h1>
|
||||
</div>
|
||||
<div class="title">[{alert.severity.value.upper()}] {alert.title}</div>
|
||||
<div class="message">
|
||||
<p><b>Message:</b> {alert.message}</p>
|
||||
<p><b>Time:</b> {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}</p>
|
||||
<p><b>Type:</b> {alert.type}</p>
|
||||
{details_html}
|
||||
</div>
|
||||
<div class="footer">
|
||||
<p>This is an automated alert from HYC Mirror Server</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
def _format_text(self, alert: Alert) -> str:
|
||||
"""格式化纯文本内容"""
|
||||
details_text = ''
|
||||
if alert.details:
|
||||
details_text = '\nDetails:\n'
|
||||
for key, value in alert.details.items():
|
||||
details_text += f" {key}: {value}\n"
|
||||
|
||||
return f"""
|
||||
HYC Mirror Alert
|
||||
================
|
||||
|
||||
Severity: {alert.severity.value.upper()}
|
||||
Title: {alert.title}
|
||||
Message: {alert.message}
|
||||
Time: {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}
|
||||
Type: {alert.type}
|
||||
{details_text}
|
||||
---
|
||||
This is an automated alert from HYC Mirror Server
|
||||
"""
|
||||
|
||||
def test_connection(self) -> Dict:
|
||||
"""测试 SMTP 连接"""
|
||||
try:
|
||||
server = self._get_connection()
|
||||
return {
|
||||
'success': True,
|
||||
'message': 'SMTP connection successful'
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'SMTP connection failed: {str(e)}'
|
||||
}
|
||||
|
||||
def close(self):
|
||||
"""关闭连接"""
|
||||
if self._connection:
|
||||
try:
|
||||
self._connection.quit()
|
||||
except Exception:
|
||||
pass
|
||||
self._connection = None
|
||||
|
||||
|
||||
class WebhookAlerter:
|
||||
"""Webhook 告警器"""
|
||||
|
||||
def __init__(self, config: Dict = None):
|
||||
"""
|
||||
初始化 Webhook 告警器
|
||||
|
||||
Args:
|
||||
config: Webhook 配置
|
||||
"""
|
||||
self.config = config or {}
|
||||
self.enabled = self.config.get('enabled', False)
|
||||
self.webhook_url = self.config.get('webhook_url', '')
|
||||
|
||||
def send(self, alert: Alert) -> bool:
|
||||
"""
|
||||
发送告警到 Webhook
|
||||
|
||||
Args:
|
||||
alert: 告警对象
|
||||
|
||||
Returns:
|
||||
是否发送成功
|
||||
"""
|
||||
if not self.enabled:
|
||||
logger.debug("Webhook alerts disabled")
|
||||
return False
|
||||
|
||||
if not self.webhook_url:
|
||||
logger.warning("No webhook URL configured")
|
||||
return False
|
||||
|
||||
try:
|
||||
payload = {
|
||||
'event': 'alert',
|
||||
'alert': alert.to_dict(),
|
||||
'timestamp': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'HYC-Mirror-Alerts/1.0'
|
||||
}
|
||||
|
||||
response = requests.post(
|
||||
self.webhook_url,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code < 400:
|
||||
logger.info(f"Alert webhook sent: {alert.title}")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Webhook returned error: {response.status_code}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send webhook alert: {e}")
|
||||
return False
|
||||
|
||||
|
||||
class AlertManager:
|
||||
"""告警管理器"""
|
||||
|
||||
def __init__(self, config: Dict = None):
|
||||
"""
|
||||
初始化告警管理器
|
||||
|
||||
Args:
|
||||
config: 告警配置
|
||||
"""
|
||||
self.config = config or {}
|
||||
self.enabled = self.config.get('enabled', False)
|
||||
|
||||
# 初始化告警器
|
||||
self.email_alerter = EmailAlerter(self.config.get('email', {}))
|
||||
self.webhook_alerter = WebhookAlerter(self.config.get('webhook', {}))
|
||||
|
||||
# 告警规则
|
||||
self.rules = self.config.get('rules', {})
|
||||
|
||||
# 告警历史
|
||||
self._alerts: List[Alert] = []
|
||||
self._alerts_lock = threading.Lock()
|
||||
self._max_history = 100
|
||||
|
||||
# 回调函数
|
||||
self._on_alert: Optional[Callable] = None
|
||||
self._on_ack: Optional[Callable] = None
|
||||
|
||||
# 告警冷却（防止重复告警）
|
||||
self._alert_cooldowns: Dict[str, float] = {}
|
||||
self._default_cooldown = 300 # 5 分钟
|
||||
|
||||
def set_alert_callback(self, callback: Callable):
|
||||
"""设置告警回调"""
|
||||
self._on_alert = callback
|
||||
|
||||
def set_ack_callback(self, callback: Callable):
|
||||
"""设置确认回调"""
|
||||
self._on_ack = callback
|
||||
|
||||
def check_rule(self, rule_name: str, data: Dict) -> Optional[Alert]:
|
||||
"""
|
||||
检查规则并生成告警
|
||||
|
||||
Args:
|
||||
rule_name: 规则名称
|
||||
data: 检查数据
|
||||
|
||||
Returns:
|
||||
告警对象或 None
|
||||
"""
|
||||
if not self.enabled:
|
||||
return None
|
||||
|
||||
rule = self.rules.get(rule_name, {})
|
||||
if not rule.get('enabled', False):
|
||||
return None
|
||||
|
||||
severity = AlertSeverity(rule.get('severity', 'warning'))
|
||||
threshold = rule.get('threshold')
|
||||
|
||||
# 磁盘空间检查
|
||||
if rule_name == 'disk_high' and threshold:
|
||||
disk_percent = data.get('disk_percent', 0)
|
||||
if disk_percent >= threshold:
|
||||
return Alert(
|
||||
alert_type=AlertType.DISK_HIGH.value,
|
||||
severity=severity,
|
||||
title=f"Disk usage is high: {disk_percent}%",
|
||||
message=f"Disk usage has reached {disk_percent}%, which is above the {threshold}% threshold.",
|
||||
details={'disk_percent': disk_percent, 'threshold': threshold},
|
||||
source='monitor'
|
||||
)
|
||||
|
||||
if rule_name == 'disk_critical' and threshold:
|
||||
disk_percent = data.get('disk_percent', 0)
|
||||
if disk_percent >= threshold:
|
||||
return Alert(
|
||||
alert_type=AlertType.DISK_CRITICAL.value,
|
||||
severity=severity,
|
||||
title=f"Disk usage is critical: {disk_percent}%",
|
||||
message=f"Disk usage has reached {disk_percent}%, which is above the {critical_threshold}% threshold. Immediate action required!",
|
||||
details={'disk_percent': disk_percent, 'threshold': threshold},
|
||||
source='monitor'
|
||||
)
|
||||
|
||||
# 同步失败检查
|
||||
if rule_name == 'sync_failed':
|
||||
sync_result = data.get('sync_result')
|
||||
if sync_result and not sync_result.get('success', True):
|
||||
return Alert(
|
||||
alert_type=AlertType.SYNC_FAILED.value,
|
||||
severity=severity,
|
||||
title=f"Sync failed: {sync_result.get('source', 'unknown')}",
|
||||
message=sync_result.get('error', 'Unknown sync error'),
|
||||
details=sync_result,
|
||||
source='sync'
|
||||
)
|
||||
|
||||
# 源不健康检查
|
||||
if rule_name == 'source_unhealthy':
|
||||
unhealthy_sources = data.get('unhealthy_sources', [])
|
||||
if unhealthy_sources:
|
||||
return Alert(
|
||||
alert_type=AlertType.SOURCE_UNHEALTHY.value,
|
||||
severity=severity,
|
||||
title=f"Unhealthy mirror sources detected: {len(unhealthy_sources)}",
|
||||
message=f"The following mirror sources are unhealthy: {', '.join(unhealthy_sources)}",
|
||||
details={'unhealthy_sources': unhealthy_sources},
|
||||
source='health_check'
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def trigger_alert(self, alert: Alert) -> bool:
|
||||
"""
|
||||
触发告警
|
||||
|
||||
Args:
|
||||
alert: 告警对象
|
||||
|
||||
Returns:
|
||||
是否发送成功
|
||||
"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
# 检查冷却时间
|
||||
cooldown_key = f"{alert.type}:{alert.source or 'unknown'}"
|
||||
last_alert = self._alert_cooldowns.get(cooldown_key, 0)
|
||||
if time.time() - last_alert < self._default_cooldown:
|
||||
logger.debug(f"Alert {alert.type} in cooldown, skipping")
|
||||
return False
|
||||
|
||||
# 发送告警
|
||||
email_sent = self.email_alerter.send(alert)
|
||||
webhook_sent = self.webhook_alerter.send(alert)
|
||||
|
||||
alert.sent = email_sent or webhook_sent
|
||||
|
||||
# 记录告警
|
||||
with self._alerts_lock:
|
||||
self._alerts.append(alert)
|
||||
if len(self._alerts) > self._max_history:
|
||||
self._alerts = self._alerts[-self._max_history:]
|
||||
|
||||
# 更新冷却时间
|
||||
self._alert_cooldowns[cooldown_key] = time.time()
|
||||
|
||||
# 触发回调
|
||||
if self._on_alert and alert.sent:
|
||||
try:
|
||||
self._on_alert(alert)
|
||||
except Exception as e:
|
||||
logger.error(f"Alert callback failed: {e}")
|
||||
|
||||
return alert.sent
|
||||
|
||||
def acknowledge_alert(self, alert_id: str) -> bool:
|
||||
"""
|
||||
确认告警
|
||||
|
||||
Args:
|
||||
alert_id: 告警 ID
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
with self._alerts_lock:
|
||||
for alert in self._alerts:
|
||||
if alert.id == alert_id:
|
||||
alert.acknowledged = True
|
||||
if self._on_ack:
|
||||
try:
|
||||
self._on_ack(alert)
|
||||
except Exception as e:
|
||||
logger.error(f"Ack callback failed: {e}")
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_alerts(
|
||||
self,
|
||||
acknowledged: bool = None,
|
||||
severity: str = None,
|
||||
limit: int = 50
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
获取告警列表
|
||||
|
||||
Args:
|
||||
acknowledged: 过滤已确认状态
|
||||
severity: 过滤级别
|
||||
limit: 返回数量限制
|
||||
|
||||
Returns:
|
||||
告警列表
|
||||
"""
|
||||
with self._alerts_lock:
|
||||
alerts = [a.to_dict() for a in self._alerts]
|
||||
|
||||
# 过滤
|
||||
if acknowledged is not None:
|
||||
alerts = [a for a in alerts if a['acknowledged'] == acknowledged]
|
||||
|
||||
if severity:
|
||||
alerts = [a for a in alerts if a['severity'] == severity]
|
||||
|
||||
# 返回最近的告警
|
||||
return alerts[-limit:]
|
||||
|
||||
def get_stats(self) -> Dict:
|
||||
"""获取告警统计"""
|
||||
with self._alerts_lock:
|
||||
total = len(self._alerts)
|
||||
unack = sum(1 for a in self._alerts if not a['acknowledged'])
|
||||
by_severity = {}
|
||||
for a in self._alerts:
|
||||
by_severity[a['severity']] = by_severity.get(a['severity'], 0) + 1
|
||||
|
||||
return {
|
||||
'total_alerts': total,
|
||||
'unacknowledged': unack,
|
||||
'by_severity': by_severity,
|
||||
'email_enabled': self.email_alerter.enabled,
|
||||
'webhook_enabled': self.webhook_alerter.enabled,
|
||||
'rules_enabled': sum(1 for r in self.rules.values() if r.get('enabled', False))
|
||||
}
|
||||
|
||||
def clear_history(self) -> bool:
|
||||
"""清除告警历史"""
|
||||
with self._alerts_lock:
|
||||
self._alerts = []
|
||||
return True
|
||||
|
||||
def test_email(self, to_address: str) -> Dict:
|
||||
"""测试邮件发送"""
|
||||
test_alert = Alert(
|
||||
alert_type=AlertType.CUSTOM.value,
|
||||
severity=AlertSeverity.INFO,
|
||||
title="Test Alert",
|
||||
message="This is a test alert from HYC Mirror Server",
|
||||
details={'test': True}
|
||||
)
|
||||
|
||||
# 临时添加收件人
|
||||
original_recipients = self.email_alerter.to_addresses
|
||||
self.email_alerter.to_addresses = [to_address]
|
||||
|
||||
success = self.email_alerter.send(test_alert)
|
||||
|
||||
# 恢复收件人
|
||||
self.email_alerter.to_addresses = original_recipients
|
||||
|
||||
return {
|
||||
'success': success,
|
||||
'message': 'Test email sent successfully' if success else 'Failed to send test email'
|
||||
}
|
||||
@@ -0,0 +1,587 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
API认证模块
|
||||
使用数据库进行认证，支持：
|
||||
- none: 无认证
|
||||
- basic: Basic Auth（用户名密码）
|
||||
- token: Token 认证（登录生成的token）
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import hashlib
|
||||
import time
|
||||
import secrets
|
||||
from typing import Dict, List, Optional
|
||||
from dataclasses import dataclass
|
||||
from functools import wraps
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuthSession:
|
||||
"""认证会话"""
|
||||
session_id: str
|
||||
user_id: str
|
||||
level: str
|
||||
created_at: float
|
||||
expires_at: float
|
||||
last_activity: float
|
||||
permissions: List[str]
|
||||
|
||||
|
||||
class APIAuthManager:
|
||||
"""API认证管理器"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.sessions: Dict[str, AuthSession] = {}
|
||||
|
||||
# 确定基础目录（用于保存会话文件）
|
||||
base_dir = config.get('base_dir', '.') if config else '.'
|
||||
|
||||
# 检测是否是 PyInstaller 打包环境
|
||||
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
|
||||
base_dir = os.path.dirname(os.path.abspath(sys.executable))
|
||||
elif base_dir == '.':
|
||||
base_dir = os.getcwd()
|
||||
|
||||
# 会话文件路径
|
||||
sessions_filename = config.get('auth_sessions_file', 'auth_sessions.json') if config else 'auth_sessions.json'
|
||||
self.sessions_file = os.path.join(base_dir, sessions_filename)
|
||||
|
||||
# 会话超时时间
|
||||
self.session_timeout = config.get('auth_session_timeout', 3600) if config else 3600
|
||||
|
||||
# Cookie名称
|
||||
self.cookie_name = 'hyc_auth'
|
||||
self.cookie_max_age = config.get('auth_cookie_max_age', 86400) if config else 86400
|
||||
|
||||
# IP 白名单
|
||||
self.ip_whitelist = config.get('ip_whitelist', []) if config else []
|
||||
self.ip_whitelist_enabled = config.get('ip_whitelist_enabled', False) if config else False
|
||||
|
||||
# 加载已保存的会话
|
||||
self._load_sessions()
|
||||
|
||||
@property
|
||||
def db(self):
|
||||
"""动态获取数据库实例"""
|
||||
return self.config.get('_db_instance')
|
||||
|
||||
def _get_client_ip(self, handler) -> str:
|
||||
"""从 handler 获取客户端 IP"""
|
||||
try:
|
||||
forwarded = handler.headers.get('X-Forwarded-For')
|
||||
if forwarded:
|
||||
return forwarded.split(',')[0].strip()
|
||||
return handler.client_address[0]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _check_ip_whitelist(self, ip: str) -> bool:
|
||||
"""检查 IP 是否在白名单中"""
|
||||
if not self.ip_whitelist_enabled or not self.ip_whitelist:
|
||||
return True
|
||||
if not ip:
|
||||
return False
|
||||
|
||||
import ipaddress
|
||||
for pattern in self.ip_whitelist:
|
||||
try:
|
||||
if '/' in pattern:
|
||||
network = ipaddress.ip_network(pattern, strict=False)
|
||||
if ipaddress.ip_address(ip) in network:
|
||||
return True
|
||||
elif pattern == ip:
|
||||
return True
|
||||
except ValueError:
|
||||
continue
|
||||
return False
|
||||
|
||||
# === Token 验证 ===
|
||||
|
||||
def validate_token(self, token: str, client_ip: str = None) -> Optional[dict]:
|
||||
"""验证 token（从数据库）"""
|
||||
if not token:
|
||||
return None
|
||||
|
||||
# 优先从数据库验证
|
||||
if self.db:
|
||||
user = self.db.get_user_by_token(token)
|
||||
if user:
|
||||
# 检查 token 是否过期
|
||||
if user.get('token_expires_at') and time.time() > user['token_expires_at']:
|
||||
return {"valid": False, "reason": "Token已过期"}
|
||||
|
||||
# 检查用户是否启用
|
||||
if not user.get('enabled', True):
|
||||
return {"valid": False, "reason": "用户已被禁用"}
|
||||
|
||||
return {
|
||||
"valid": True,
|
||||
"key_id": f"user_{user['id']}",
|
||||
"name": user['username'],
|
||||
"level": user.get('role', 'admin'),
|
||||
"permissions": ["*"],
|
||||
"user_id": user['id'],
|
||||
"username": user['username']
|
||||
}
|
||||
|
||||
return {"valid": False, "reason": "无效的Token"}
|
||||
|
||||
# === Basic Auth 验证 ===
|
||||
|
||||
def validate_basic_auth(self, username: str, password: str, client_ip: str = None) -> dict:
|
||||
"""验证 Basic Auth 用户名密码（从数据库）"""
|
||||
# IP 白名单检查
|
||||
if not self._check_ip_whitelist(client_ip):
|
||||
if self.db:
|
||||
self.db.add_login_log(username, client_ip, 'failed', 'IP不在白名单')
|
||||
return {"valid": False, "reason": "IP不在白名单内"}
|
||||
|
||||
auth_type = self.config.get('auth_type', 'none')
|
||||
|
||||
# 如果认证类型为 none，任何用户都可以通过
|
||||
if auth_type == 'none':
|
||||
return {
|
||||
"valid": True,
|
||||
"user_id": 0,
|
||||
"username": username or 'anonymous',
|
||||
"level": "admin",
|
||||
"key_id": "anonymous",
|
||||
"name": f"Anonymous - {username or 'anonymous'}",
|
||||
"permissions": ["*"]
|
||||
}
|
||||
|
||||
# 从数据库验证
|
||||
if self.db:
|
||||
result = self.db.verify_user(username, password)
|
||||
if result.get('valid'):
|
||||
if self.db:
|
||||
self.db.add_login_log(username, client_ip, 'success', '数据库验证')
|
||||
return {
|
||||
"valid": True,
|
||||
"user_id": result.get('user_id'),
|
||||
"username": username,
|
||||
"level": result.get('role', 'admin'),
|
||||
"key_id": f"user_{result.get('user_id')}",
|
||||
"name": f"User - {username}",
|
||||
"permissions": ["*"]
|
||||
}
|
||||
else:
|
||||
if self.db:
|
||||
self.db.add_login_log(username, client_ip, 'failed', result.get('reason', '验证失败'))
|
||||
return {"valid": False, "reason": result.get('reason', '用户名或密码错误')}
|
||||
|
||||
return {"valid": False, "reason": "数据库不可用"}
|
||||
|
||||
# === Cookie 验证 ===
|
||||
|
||||
def validate_cookie(self, cookie_value: str, client_ip: str = None) -> Optional[dict]:
|
||||
"""验证认证Cookie"""
|
||||
if not cookie_value:
|
||||
return None
|
||||
|
||||
parts = cookie_value.split('.')
|
||||
if len(parts) != 3:
|
||||
return None
|
||||
|
||||
session_id, timestamp, signature = parts
|
||||
|
||||
session = self.sessions.get(session_id)
|
||||
if not session:
|
||||
return None
|
||||
|
||||
if time.time() > session.expires_at:
|
||||
del self.sessions[session_id]
|
||||
return None
|
||||
|
||||
expected_sig = self._generate_signature(session_id, timestamp, session.user_id)
|
||||
if signature != expected_sig:
|
||||
return None
|
||||
|
||||
session.last_activity = time.time()
|
||||
|
||||
return {
|
||||
"valid": True,
|
||||
"session_id": session_id,
|
||||
"user_id": session.user_id,
|
||||
"level": session.level,
|
||||
"permissions": session.permissions
|
||||
}
|
||||
|
||||
def validate_session_id(self, session_id: str) -> Optional[dict]:
|
||||
"""验证会话ID"""
|
||||
if not session_id:
|
||||
return None
|
||||
|
||||
session = self.sessions.get(session_id)
|
||||
if not session:
|
||||
return None
|
||||
|
||||
if time.time() > session.expires_at:
|
||||
del self.sessions[session_id]
|
||||
return None
|
||||
|
||||
session.last_activity = time.time()
|
||||
|
||||
return {
|
||||
"valid": True,
|
||||
"session_id": session_id,
|
||||
"user_id": session.user_id,
|
||||
"level": session.level,
|
||||
"permissions": session.permissions
|
||||
}
|
||||
|
||||
# === 请求验证 ===
|
||||
|
||||
def validate_request(self, handler, required_level: str = "admin") -> dict:
|
||||
"""
|
||||
验证请求的认证状态
|
||||
|
||||
支持的认证方式：
|
||||
1. Authorization: Bearer <token>
|
||||
2. Authorization: Basic <credentials>
|
||||
3. X-API-Key: <token>
|
||||
4. Cookie: hyc_auth=<session>
|
||||
5. ?key=<token>
|
||||
"""
|
||||
import base64
|
||||
|
||||
auth_header = handler.headers.get('Authorization')
|
||||
api_key = handler.headers.get('X-API-Key')
|
||||
cookie = handler.headers.get('Cookie', '')
|
||||
client_ip = handler.client_address[0] if hasattr(handler, 'client_address') else None
|
||||
|
||||
# 提取cookie值
|
||||
cookie_value = None
|
||||
for c in cookie.split(';'):
|
||||
c = c.strip()
|
||||
if c.startswith(f'{self.cookie_name}='):
|
||||
cookie_value = c[len(self.cookie_name)+1:]
|
||||
break
|
||||
|
||||
# 获取查询参数中的key
|
||||
parsed_path = handler.path.split('?')
|
||||
query_key = None
|
||||
if len(parsed_path) > 1:
|
||||
from urllib.parse import parse_qs
|
||||
query = parse_qs(parsed_path[1])
|
||||
query_key = query.get('key', [None])[0]
|
||||
|
||||
# 1. Bearer Token
|
||||
if auth_header and auth_header.startswith('Bearer '):
|
||||
token = auth_header[7:]
|
||||
result = self.validate_token(token, client_ip)
|
||||
if result and result.get('valid'):
|
||||
return {"authenticated": True, "method": "bearer", **result}
|
||||
|
||||
# 2. Basic Auth
|
||||
if auth_header and auth_header.startswith('Basic '):
|
||||
try:
|
||||
credentials = base64.b64decode(auth_header[6:]).decode('utf-8')
|
||||
if ':' in credentials:
|
||||
username, password = credentials.split(':', 1)
|
||||
result = self.validate_basic_auth(username, password, client_ip)
|
||||
if result and result.get('valid'):
|
||||
return {"authenticated": True, "method": "basic", **result}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 3. API Key Header
|
||||
if api_key:
|
||||
result = self.validate_token(api_key, client_ip)
|
||||
if result and result.get('valid'):
|
||||
return {"authenticated": True, "method": "api_key", **result}
|
||||
|
||||
# 4. Cookie
|
||||
if cookie_value:
|
||||
result = self.validate_cookie(cookie_value, client_ip)
|
||||
if result and result.get('valid'):
|
||||
return {"authenticated": True, "method": "cookie", **result}
|
||||
|
||||
# 5. Query Parameter
|
||||
if query_key:
|
||||
result = self.validate_token(query_key, client_ip)
|
||||
if result and result.get('valid'):
|
||||
return {"authenticated": True, "method": "query", **result}
|
||||
|
||||
# 未认证
|
||||
return {
|
||||
"authenticated": False,
|
||||
"error": "Authentication required",
|
||||
"required_level": required_level
|
||||
}
|
||||
|
||||
def check_permission(self, auth_result: dict, permission: str) -> bool:
|
||||
"""检查是否有权限访问特定API"""
|
||||
if not auth_result.get('authenticated'):
|
||||
return False
|
||||
|
||||
permissions = auth_result.get('permissions', [])
|
||||
|
||||
if '*' in permissions:
|
||||
return True
|
||||
|
||||
if permission in permissions:
|
||||
return True
|
||||
|
||||
for p in permissions:
|
||||
if p.endswith('*'):
|
||||
prefix = p.rstrip('*')
|
||||
if permission.startswith(prefix):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# === 会话管理 ===
|
||||
|
||||
def create_session(self, user_id: str, level: str,
|
||||
permissions: List[str] = None) -> dict:
|
||||
"""创建认证会话"""
|
||||
session_id = secrets.token_hex(32)
|
||||
timestamp = time.time()
|
||||
|
||||
session = AuthSession(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
level=level,
|
||||
created_at=timestamp,
|
||||
expires_at=timestamp + self.session_timeout,
|
||||
last_activity=timestamp,
|
||||
permissions=permissions or ['*']
|
||||
)
|
||||
|
||||
self.sessions[session_id] = session
|
||||
self._save_sessions()
|
||||
|
||||
signature = self._generate_signature(session_id, timestamp, user_id)
|
||||
cookie_value = f"{session_id}.{timestamp}.{signature}"
|
||||
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"cookie_name": self.cookie_name,
|
||||
"cookie_value": cookie_value,
|
||||
"cookie_max_age": self.cookie_max_age,
|
||||
"expires": timestamp + self.session_timeout
|
||||
}
|
||||
|
||||
def destroy_session(self, session_id: str) -> bool:
|
||||
"""销毁会话"""
|
||||
if session_id in self.sessions:
|
||||
del self.sessions[session_id]
|
||||
self._save_sessions()
|
||||
return True
|
||||
return False
|
||||
|
||||
# === 内部方法 ===
|
||||
|
||||
def _generate_signature(self, session_id: str, timestamp: str, user_id: str) -> str:
|
||||
"""生成签名"""
|
||||
secret = self.config.get('auth_secret', 'default_secret_change_me')
|
||||
data = f"{session_id}.{timestamp}.{user_id}.{secret}"
|
||||
return hashlib.sha256(data.encode()).hexdigest()[:32]
|
||||
|
||||
def _load_sessions(self):
|
||||
"""加载会话"""
|
||||
if os.path.exists(self.sessions_file):
|
||||
try:
|
||||
with open(self.sessions_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
now = time.time()
|
||||
for item in data:
|
||||
if item.get('expires_at') and now > item['expires_at']:
|
||||
continue
|
||||
session = AuthSession(
|
||||
session_id=item['session_id'],
|
||||
user_id=item['user_id'],
|
||||
level=item['level'],
|
||||
created_at=item['created_at'],
|
||||
expires_at=item['expires_at'],
|
||||
last_activity=item['last_activity'],
|
||||
permissions=item.get('permissions', ['*'])
|
||||
)
|
||||
self.sessions[session.session_id] = session
|
||||
except Exception as e:
|
||||
print(f"加载会话失败: {e}")
|
||||
|
||||
def _save_sessions(self):
|
||||
"""保存会话"""
|
||||
data = []
|
||||
for session in self.sessions.values():
|
||||
data.append({
|
||||
"session_id": session.session_id,
|
||||
"user_id": session.user_id,
|
||||
"level": session.level,
|
||||
"created_at": session.created_at,
|
||||
"expires_at": session.expires_at,
|
||||
"last_activity": session.last_activity,
|
||||
"permissions": session.permissions
|
||||
})
|
||||
|
||||
try:
|
||||
with open(self.sessions_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""获取认证统计"""
|
||||
active_sessions = sum(
|
||||
1 for s in self.sessions.values()
|
||||
if time.time() < s.expires_at
|
||||
)
|
||||
|
||||
return {
|
||||
"active_sessions": active_sessions,
|
||||
"session_timeout": self.session_timeout
|
||||
}
|
||||
|
||||
|
||||
# === API认证装饰器 ===
|
||||
|
||||
def require_auth(required_level: str = "admin", permission: str = None):
|
||||
"""
|
||||
API认证装饰器
|
||||
|
||||
使用方式:
|
||||
@require_auth()
|
||||
def api_endpoint(self, handler):
|
||||
...
|
||||
|
||||
@require_auth(permission="sync:start")
|
||||
def api_sync_start(self, handler):
|
||||
...
|
||||
"""
|
||||
def decorator(func):
|
||||
@wraps(func)
|
||||
def wrapper(self, handler, *args, **kwargs):
|
||||
# 检查是否需要认证
|
||||
if required_level == "none":
|
||||
return func(self, handler, *args, **kwargs)
|
||||
|
||||
config = getattr(handler, 'config', {})
|
||||
auth_type = config.get('auth_type', 'none')
|
||||
|
||||
# 如果auth_type为none，跳过认证
|
||||
if auth_type == "none":
|
||||
handler.auth_result = {
|
||||
"authenticated": True,
|
||||
"level": "admin",
|
||||
"user_id": "anonymous",
|
||||
"permissions": ["*"]
|
||||
}
|
||||
return func(self, handler, *args, **kwargs)
|
||||
|
||||
auth_manager = getattr(handler, 'auth_manager', None)
|
||||
if not auth_manager:
|
||||
handler.send_json_response({
|
||||
"error": "认证系统未初始化",
|
||||
"code": "AUTH_NOT_INITIALIZED"
|
||||
}, 500)
|
||||
return
|
||||
|
||||
auth_result = auth_manager.validate_request(handler, required_level)
|
||||
|
||||
if not auth_result.get('authenticated'):
|
||||
handler.send_response(401)
|
||||
handler.send_header('WWW-Authenticate', 'Bearer realm="HYC API"')
|
||||
handler.send_header('Access-Control-Allow-Origin', '*')
|
||||
handler.send_json_response({
|
||||
"error": "未认证或认证已过期",
|
||||
"code": "UNAUTHORIZED",
|
||||
"required_level": required_level,
|
||||
"auth_methods": [
|
||||
"Authorization: Bearer <token>",
|
||||
"X-API-Key: <token>",
|
||||
f"Cookie: {auth_manager.cookie_name}=<session>",
|
||||
"?key=<token>"
|
||||
]
|
||||
})
|
||||
return
|
||||
|
||||
if permission:
|
||||
if not auth_manager.check_permission(auth_result, permission):
|
||||
handler.send_json_response({
|
||||
"error": "权限不足",
|
||||
"code": "FORBIDDEN",
|
||||
"required_permission": permission
|
||||
}, 403)
|
||||
return
|
||||
|
||||
handler.auth_result = auth_result
|
||||
return func(self, handler, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
# === 需要认证的API端点定义 ===
|
||||
|
||||
ADMIN_API_ENDPOINTS = {
|
||||
# 同步管理
|
||||
'POST:/api/v2/sync/*': 'sync:manage',
|
||||
'POST:/api/v2/sync/*/start': 'sync:start',
|
||||
'POST:/api/v2/sync/*/stop': 'sync:stop',
|
||||
'DELETE:/api/v2/sync/*': 'sync:manage',
|
||||
|
||||
# 缓存管理
|
||||
'POST:/api/v2/cache/clean': 'cache:manage',
|
||||
'DELETE:/api/v2/cache/*': 'cache:manage',
|
||||
|
||||
# Webhook管理
|
||||
'POST:/api/v2/webhooks': 'webhook:create',
|
||||
'PUT:/api/v2/webhooks/*': 'webhook:update',
|
||||
'DELETE:/api/v2/webhooks/*': 'webhook:delete',
|
||||
'POST:/api/v2/webhooks/*/trigger': 'webhook:trigger',
|
||||
|
||||
# 服务器配置
|
||||
'PUT:/api/v2/config': 'config:manage',
|
||||
'POST:/api/v2/server/reload': 'server:reload',
|
||||
|
||||
# 文件管理（高危操作）
|
||||
'DELETE:/api/v2/files/*': 'files:delete',
|
||||
'PUT:/api/v2/files/*/rename': 'files:rename',
|
||||
|
||||
# 用户管理
|
||||
'POST:/api/v2/users': 'users:create',
|
||||
'DELETE:/api/v2/users/*': 'users:delete',
|
||||
'PUT:/api/v2/users/*': 'users:update',
|
||||
|
||||
# === API v1 文件操作认证 ===
|
||||
'DELETE:/api/v1/file/*': 'files:delete',
|
||||
'PUT:/api/v1/mkdir': 'files:create',
|
||||
'POST:/api/v1/upload': 'files:upload',
|
||||
'POST:/api/v1/batch': 'files:batch',
|
||||
'POST:/api/v1/archive': 'files:archive',
|
||||
}
|
||||
|
||||
|
||||
def check_endpoint_auth(method: str, path: str, auth_manager: APIAuthManager) -> dict:
|
||||
"""检查端点是否需要认证"""
|
||||
key = f"{method}:{path}"
|
||||
if key in ADMIN_API_ENDPOINTS:
|
||||
return {
|
||||
"required": True,
|
||||
"permission": ADMIN_API_ENDPOINTS[key]
|
||||
}
|
||||
|
||||
for pattern, permission in ADMIN_API_ENDPOINTS.items():
|
||||
if '*' in pattern:
|
||||
pat_method, pat_path = pattern.split(':', 1)
|
||||
if method == pat_method or pat_method == '*':
|
||||
if pat_path.endswith('*'):
|
||||
prefix = pat_path.rstrip('*').rstrip('/')
|
||||
if path.startswith(prefix):
|
||||
return {
|
||||
"required": True,
|
||||
"permission": permission
|
||||
}
|
||||
|
||||
return {
|
||||
"required": False,
|
||||
"permission": None
|
||||
}
|
||||
@@ -0,0 +1,816 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
API 文档生成器
|
||||
自动生成 OpenAPI/Swagger 格式的 API 文档
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import inspect
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Any, Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class APIDoc:
|
||||
"""API 文档信息"""
|
||||
title: str = "HYC下载站 API"
|
||||
version: str = "2.2.0"
|
||||
description: str = "HYC镜像下载站 REST API 文档"
|
||||
servers: List[Dict] = field(default_factory=list)
|
||||
tags: List[Dict] = field(default_factory=list)
|
||||
paths: Dict = field(default_factory=dict)
|
||||
components: Dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class APIDocGenerator:
|
||||
"""API 文档生成器"""
|
||||
|
||||
def __init__(self, config: Dict = None):
|
||||
self.config = config or {}
|
||||
self.api_doc = APIDoc()
|
||||
|
||||
# 初始化组件
|
||||
self._init_components()
|
||||
|
||||
def _init_components(self):
|
||||
"""初始化文档组件"""
|
||||
self.api_doc.components = {
|
||||
'securitySchemes': {
|
||||
'BearerAuth': {
|
||||
'type': 'http',
|
||||
'scheme': 'bearer',
|
||||
'bearerFormat': 'JWT',
|
||||
'description': 'JWT token 认证'
|
||||
},
|
||||
'ApiKeyAuth': {
|
||||
'type': 'apiKey',
|
||||
'in': 'header',
|
||||
'name': 'X-API-Key',
|
||||
'description': 'API Key 认证'
|
||||
}
|
||||
},
|
||||
'schemas': {
|
||||
'Error': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'error': {'type': 'string', 'description': '错误信息'},
|
||||
'code': {'type': 'string', 'description': '错误代码'},
|
||||
'message': {'type': 'string', 'description': '详细描述'}
|
||||
}
|
||||
},
|
||||
'Success': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'success': {'type': 'boolean'},
|
||||
'message': {'type': 'string'},
|
||||
'data': {'type': 'object'}
|
||||
}
|
||||
},
|
||||
'HealthStatus': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'status': {'type': 'string', 'enum': ['healthy', 'degraded', 'unhealthy']},
|
||||
'components': {'type': 'object'},
|
||||
'timestamp': {'type': 'string', 'format': 'date-time'}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def generate(self) -> Dict:
|
||||
"""生成完整的 API 文档"""
|
||||
doc = {
|
||||
'openapi': '3.0.3',
|
||||
'info': {
|
||||
'title': self.api_doc.title,
|
||||
'version': self.api_doc.version,
|
||||
'description': self.api_doc.description,
|
||||
'contact': {
|
||||
'name': 'HYC Mirror Support',
|
||||
'email': '[email protected]'
|
||||
},
|
||||
'license': {
|
||||
'name': 'MIT',
|
||||
'url': 'https://opensource.org/licenses/MIT'
|
||||
}
|
||||
},
|
||||
'servers': self.api_doc.servers,
|
||||
'tags': self.api_doc.tags,
|
||||
'paths': self.api_doc.paths,
|
||||
'components': self.api_doc.components
|
||||
}
|
||||
|
||||
return doc
|
||||
|
||||
def add_server(self, url: str, description: str = ''):
|
||||
"""添加服务器"""
|
||||
self.api_doc.servers.append({
|
||||
'url': url,
|
||||
'description': description
|
||||
})
|
||||
|
||||
def add_tag(self, name: str, description: str = ''):
|
||||
"""添加标签"""
|
||||
self.api_doc.tags.append({
|
||||
'name': name,
|
||||
'description': description
|
||||
})
|
||||
|
||||
def add_endpoint(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
summary: str,
|
||||
description: str = '',
|
||||
tags: List[str] = None,
|
||||
parameters: List[Dict] = None,
|
||||
requestBody: Dict = None,
|
||||
responses: Dict = None,
|
||||
security: List[Dict] = None,
|
||||
deprecated: bool = False
|
||||
):
|
||||
"""
|
||||
添加 API 端点
|
||||
|
||||
Args:
|
||||
method: HTTP 方法 (GET, POST, PUT, DELETE, PATCH)
|
||||
path: API 路径
|
||||
summary: 简要描述
|
||||
description: 详细描述
|
||||
tags: 标签列表
|
||||
parameters: 参数列表
|
||||
requestBody: 请求体
|
||||
responses: 响应定义
|
||||
security: 安全要求
|
||||
deprecated: 是否废弃
|
||||
"""
|
||||
if parameters is None:
|
||||
parameters = []
|
||||
if responses is None:
|
||||
responses = self._default_responses()
|
||||
if security is None:
|
||||
security = []
|
||||
|
||||
# 转换路径参数
|
||||
path_params = self._extract_path_params(path)
|
||||
for param in path_params:
|
||||
parameters.append({
|
||||
'name': param,
|
||||
'in': 'path',
|
||||
'required': True,
|
||||
'schema': {'type': 'string'},
|
||||
'description': f'Path parameter: {param}'
|
||||
})
|
||||
|
||||
# 转换查询参数
|
||||
query_params = self._extract_query_params(path)
|
||||
for param in query_params:
|
||||
parameters.append({
|
||||
'name': param,
|
||||
'in': 'query',
|
||||
'required': False,
|
||||
'schema': {'type': 'string'},
|
||||
'description': f'Query parameter: {param}'
|
||||
})
|
||||
|
||||
# 构建路径
|
||||
clean_path = path.format(**{p: f'{{{p}}}' for p in path_params})
|
||||
if clean_path not in self.api_doc.paths:
|
||||
self.api_doc.paths[clean_path] = {}
|
||||
|
||||
endpoint = {
|
||||
'summary': summary,
|
||||
'description': description,
|
||||
'tags': tags or [],
|
||||
'parameters': parameters,
|
||||
'responses': responses,
|
||||
'deprecated': deprecated
|
||||
}
|
||||
|
||||
if security:
|
||||
endpoint['security'] = security
|
||||
|
||||
if requestBody:
|
||||
endpoint['requestBody'] = requestBody
|
||||
|
||||
self.api_doc.paths[clean_path][method.lower()] = endpoint
|
||||
|
||||
def _default_responses(self) -> Dict:
|
||||
"""获取默认响应"""
|
||||
return {
|
||||
'200': {
|
||||
'description': 'Successful response',
|
||||
'content': {
|
||||
'application/json': {
|
||||
'schema': {'type': 'object'}
|
||||
}
|
||||
}
|
||||
},
|
||||
'400': {
|
||||
'description': 'Bad request',
|
||||
'content': {
|
||||
'application/json': {
|
||||
'schema': {'$ref': '#/components/schemas/Error'}
|
||||
}
|
||||
}
|
||||
},
|
||||
'401': {
|
||||
'description': 'Unauthorized',
|
||||
'content': {
|
||||
'application/json': {
|
||||
'schema': {'$ref': '#/components/schemas/Error'}
|
||||
}
|
||||
}
|
||||
},
|
||||
'403': {
|
||||
'description': 'Forbidden',
|
||||
'content': {
|
||||
'application/json': {
|
||||
'schema': {'$ref': '#/components/schemas/Error'}
|
||||
}
|
||||
}
|
||||
},
|
||||
'404': {
|
||||
'description': 'Not found',
|
||||
'content': {
|
||||
'application/json': {
|
||||
'schema': {'$ref': '#/components/schemas/Error'}
|
||||
}
|
||||
}
|
||||
},
|
||||
'500': {
|
||||
'description': 'Internal server error',
|
||||
'content': {
|
||||
'application/json': {
|
||||
'schema': {'$ref': '#/components/schemas/Error'}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def _extract_path_params(self, path: str) -> List[str]:
|
||||
"""提取路径参数"""
|
||||
import re
|
||||
return re.findall(r'\{(\w+)\}', path)
|
||||
|
||||
def _extract_query_params(self, path: str) -> List[str]:
|
||||
"""提取查询参数"""
|
||||
import re
|
||||
return re.findall(r':(\w+)', path)
|
||||
|
||||
def save(self, filepath: str, format: str = 'json'):
|
||||
"""
|
||||
保存 API 文档
|
||||
|
||||
Args:
|
||||
filepath: 保存路径
|
||||
format: 格式 (json, yaml)
|
||||
"""
|
||||
doc = self.generate()
|
||||
|
||||
if format == 'yaml':
|
||||
try:
|
||||
import yaml
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(doc, f, default_flow_style=False, allow_unicode=True)
|
||||
except ImportError:
|
||||
# 回退为 JSON
|
||||
format = 'json'
|
||||
|
||||
if format == 'json':
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(doc, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return filepath
|
||||
|
||||
|
||||
class APIEndpointRegistry:
|
||||
"""API 端点注册表"""
|
||||
|
||||
def __init__(self):
|
||||
self.endpoints: List[Dict] = []
|
||||
|
||||
def register(self, method: str, path: str, handler_name: str, description: str = ''):
|
||||
"""注册端点"""
|
||||
self.endpoints.append({
|
||||
'method': method.upper(),
|
||||
'path': path,
|
||||
'handler': handler_name,
|
||||
'description': description
|
||||
})
|
||||
|
||||
def get_all(self) -> List[Dict]:
|
||||
"""获取所有端点"""
|
||||
return self.endpoints
|
||||
|
||||
def generate_docs(self) -> Dict:
|
||||
"""生成文档"""
|
||||
generator = APIDocGenerator()
|
||||
|
||||
for ep in self.endpoints:
|
||||
generator.add_endpoint(
|
||||
method=ep['method'],
|
||||
path=ep['path'],
|
||||
summary=ep['description'],
|
||||
description=ep['description']
|
||||
)
|
||||
|
||||
return generator.generate()
|
||||
|
||||
|
||||
def generate_api_docs(config: Dict = None) -> Dict:
|
||||
"""
|
||||
生成完整的 API 文档
|
||||
|
||||
Args:
|
||||
config: 服务器配置
|
||||
|
||||
Returns:
|
||||
OpenAPI 格式的文档
|
||||
"""
|
||||
generator = APIDocGenerator(config)
|
||||
|
||||
# 设置服务器信息
|
||||
host = config.get('host', 'localhost')
|
||||
port = config.get('port', 8080)
|
||||
protocol = 'https' if config.get('ssl_cert') else 'http'
|
||||
generator.add_server(f'{protocol}://{host}:{port}', 'Production server')
|
||||
|
||||
# 添加标签
|
||||
generator.add_tag('Server', '服务器信息')
|
||||
generator.add_tag('Monitoring', '监控与指标')
|
||||
generator.add_tag('Mirrors', '镜像源管理')
|
||||
generator.add_tag('Sync', '同步管理')
|
||||
generator.add_tag('Cache', '缓存管理')
|
||||
generator.add_tag('Health', '健康检查')
|
||||
generator.add_tag('Alerts', '告警管理')
|
||||
generator.add_tag('Webhooks', 'Webhook管理')
|
||||
generator.add_tag('Configuration', '配置管理')
|
||||
generator.add_tag('Authentication', '认证管理')
|
||||
|
||||
# ========== 服务器信息端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/server/info',
|
||||
summary='获取服务器信息',
|
||||
description='返回服务器的详细信息，包括版本、运行时间、配置等',
|
||||
tags=['Server']
|
||||
)
|
||||
|
||||
# ========== 监控端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/monitor/realtime',
|
||||
summary='获取实时监控数据',
|
||||
description='返回服务器的实时监控数据，包括 CPU、内存、磁盘使用情况',
|
||||
tags=['Monitoring']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/monitor/history',
|
||||
summary='获取历史监控数据',
|
||||
description='返回指定时间段内的历史监控数据',
|
||||
tags=['Monitoring'],
|
||||
parameters=[{
|
||||
'name': 'period',
|
||||
'in': 'query',
|
||||
'schema': {'type': 'string'},
|
||||
'description': '时间周期: 1h, 6h, 24h, 7d, 30d'
|
||||
}]
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/metrics',
|
||||
summary='获取 Prometheus 指标',
|
||||
description='返回 Prometheus 格式的监控指标',
|
||||
tags=['Monitoring']
|
||||
)
|
||||
|
||||
# ========== 镜像源端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/mirrors',
|
||||
summary='列出所有镜像加速源',
|
||||
description='返回所有可用的镜像加速源列表',
|
||||
tags=['Mirrors']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/mirrors/:name',
|
||||
summary='获取镜像源详情',
|
||||
description='返回指定镜像源的详细信息',
|
||||
tags=['Mirrors']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/mirrors',
|
||||
summary='添加自定义镜像源',
|
||||
description='添加新的自定义镜像加速源',
|
||||
tags=['Mirrors'],
|
||||
requestBody={
|
||||
'required': True,
|
||||
'content': {
|
||||
'application/json': {
|
||||
'schema': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'name': {'type': 'string'},
|
||||
'type': {'type': 'string'},
|
||||
'url': {'type': 'string'},
|
||||
'enabled': {'type': 'boolean'}
|
||||
},
|
||||
'required': ['name', 'url']
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'DELETE', '/api/v2/mirrors/:name',
|
||||
summary='删除自定义镜像源',
|
||||
description='删除指定的自定义镜像加速源',
|
||||
tags=['Mirrors']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/mirrors/:name/refresh',
|
||||
summary='刷新镜像源缓存',
|
||||
description='刷新指定镜像源的缓存数据',
|
||||
tags=['Mirrors']
|
||||
)
|
||||
|
||||
# ========== 同步管理端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/sync/sources',
|
||||
summary='获取同步源列表',
|
||||
description='返回所有配置的同步源',
|
||||
tags=['Sync']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/sync/sources',
|
||||
summary='添加同步源',
|
||||
description='添加新的同步源配置',
|
||||
tags=['Sync'],
|
||||
requestBody={
|
||||
'required': True,
|
||||
'content': {
|
||||
'application/json': {
|
||||
'schema': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'name': {'type': 'string'},
|
||||
'type': {'type': 'string'},
|
||||
'url': {'type': 'string'},
|
||||
'schedule': {'type': 'string'}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/sync/:source_name/start',
|
||||
summary='启动同步任务',
|
||||
description='启动指定源的同步任务',
|
||||
tags=['Sync']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/sync/:source_name/stop',
|
||||
summary='停止同步任务',
|
||||
description='停止指定源的同步任务',
|
||||
tags=['Sync']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/sync/:source_name/status',
|
||||
summary='获取同步状态',
|
||||
description='返回指定同步源的当前状态',
|
||||
tags=['Sync']
|
||||
)
|
||||
|
||||
# ========== 缓存管理端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/cache/stats',
|
||||
summary='获取缓存统计',
|
||||
description='返回缓存的使用统计信息',
|
||||
tags=['Cache']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/cache/usage',
|
||||
summary='获取缓存使用详情',
|
||||
description='返回缓存的详细使用情况',
|
||||
tags=['Cache']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/cache/clean',
|
||||
summary='清理缓存',
|
||||
description='清理指定或全部缓存',
|
||||
tags=['Cache'],
|
||||
requestBody={
|
||||
'content': {
|
||||
'application/json': {
|
||||
'schema': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'source': {'type': 'string'}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
# ========== 缓存预热端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/cache/prewarm',
|
||||
summary='获取预热状态',
|
||||
description='返回缓存预热的当前状态',
|
||||
tags=['Cache']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/cache/prewarm',
|
||||
summary='执行缓存预热',
|
||||
description='手动执行缓存预热任务',
|
||||
tags=['Cache']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/cache/prewarm/items',
|
||||
summary='获取预热项目列表',
|
||||
description='返回待预热的项目列表',
|
||||
tags=['Cache']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/cache/prewarm/clear',
|
||||
summary='清空预热队列',
|
||||
description='清空待预热的项目队列',
|
||||
tags=['Cache']
|
||||
)
|
||||
|
||||
# ========== 健康检查端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/health',
|
||||
summary='获取健康状态',
|
||||
description='返回服务器的整体健康状态',
|
||||
tags=['Health']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/health/sources',
|
||||
summary='获取镜像源健康状态',
|
||||
description='返回所有镜像源的健康检查结果',
|
||||
tags=['Health']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/health/check/:source_name',
|
||||
summary='检查指定源健康',
|
||||
description='手动触发指定镜像源的健康检查',
|
||||
tags=['Health']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/health/failover',
|
||||
summary='获取故障切换状态',
|
||||
description='返回故障切换系统的当前状态',
|
||||
tags=['Health']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/health/failover/:mirror_type',
|
||||
summary='触发故障切换',
|
||||
description='手动触发指定镜像类型的故障切换',
|
||||
tags=['Health']
|
||||
)
|
||||
|
||||
# ========== 告警管理端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/alerts',
|
||||
summary='获取告警列表',
|
||||
description='返回当前告警列表',
|
||||
tags=['Alerts'],
|
||||
parameters=[{
|
||||
'name': 'limit',
|
||||
'in': 'query',
|
||||
'schema': {'type': 'integer'},
|
||||
'description': '返回数量限制'
|
||||
}]
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/alerts/:alert_id/acknowledge',
|
||||
summary='确认告警',
|
||||
description='确认指定告警',
|
||||
tags=['Alerts']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/alerts/clear',
|
||||
summary='清除告警历史',
|
||||
description='清除所有告警历史记录',
|
||||
tags=['Alerts']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/alerts/test',
|
||||
summary='测试告警发送',
|
||||
description='发送测试告警以验证配置',
|
||||
tags=['Alerts']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/alerts/config',
|
||||
summary='获取告警配置',
|
||||
description='返回当前的告警配置',
|
||||
tags=['Alerts']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'PUT', '/api/v2/alerts/config',
|
||||
summary='更新告警配置',
|
||||
description='更新告警配置（邮件、Webhook 等）',
|
||||
tags=['Alerts']
|
||||
)
|
||||
|
||||
# ========== Webhook 端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/webhooks',
|
||||
summary='列出所有 Webhook',
|
||||
description='返回所有配置的 Webhook',
|
||||
tags=['Webhooks']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/webhooks',
|
||||
summary='创建 Webhook',
|
||||
description='创建新的 Webhook 配置',
|
||||
tags=['Webhooks']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/webhooks/:webhook_id',
|
||||
summary='获取 Webhook 详情',
|
||||
description='返回指定 Webhook 的详细信息',
|
||||
tags=['Webhooks']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'PUT', '/api/v2/webhooks/:webhook_id',
|
||||
summary='更新 Webhook',
|
||||
description='更新指定 Webhook 的配置',
|
||||
tags=['Webhooks']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'DELETE', '/api/v2/webhooks/:webhook_id',
|
||||
summary='删除 Webhook',
|
||||
description='删除指定的 Webhook',
|
||||
tags=['Webhooks']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/webhooks/:webhook_id/test',
|
||||
summary='测试 Webhook',
|
||||
description='发送测试请求到指定的 Webhook',
|
||||
tags=['Webhooks']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/webhooks/:webhook_id/deliveries',
|
||||
summary='获取 Webhook 交付历史',
|
||||
description='返回指定 Webhook 的交付历史记录',
|
||||
tags=['Webhooks']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/webhooks/:webhook_id/stats',
|
||||
summary='获取 Webhook 统计',
|
||||
description='返回指定 Webhook 的交付统计信息',
|
||||
tags=['Webhooks']
|
||||
)
|
||||
|
||||
# ========== 配置管理端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/config',
|
||||
summary='获取配置',
|
||||
description='返回当前的服务器配置',
|
||||
tags=['Configuration']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'PUT', '/api/v2/config',
|
||||
summary='保存配置',
|
||||
description='保存配置到 settings.json',
|
||||
tags=['Configuration']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/config/reload',
|
||||
summary='重新加载配置',
|
||||
description='重新加载配置文件（热更新）',
|
||||
tags=['Configuration']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/config/changes',
|
||||
summary='获取配置变更历史',
|
||||
description='返回配置变更的历史记录',
|
||||
tags=['Configuration']
|
||||
)
|
||||
|
||||
# ========== 重启管理端点 ==========
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/server/restart',
|
||||
summary='获取重启状态',
|
||||
description='返回服务器重启管理的当前状态',
|
||||
tags=['Server']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/server/restart',
|
||||
summary='准备重启',
|
||||
description='准备执行服务器重启',
|
||||
tags=['Server']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/server/restart/confirm',
|
||||
summary='确认执行重启',
|
||||
description='确认并执行服务器重启',
|
||||
tags=['Server']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/server/restart/immediate',
|
||||
summary='立即重启',
|
||||
description='立即重启服务器（不等待请求完成）',
|
||||
tags=['Server']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/server/restart/pending',
|
||||
summary='获取待处理请求',
|
||||
description='返回当前待处理的请求列表',
|
||||
tags=['Server']
|
||||
)
|
||||
|
||||
generator.add_endpoint(
|
||||
'GET', '/api/v2/server/restart/history',
|
||||
summary='获取重启历史',
|
||||
description='返回服务器重启的历史记录',
|
||||
tags=['Server']
|
||||
)
|
||||
|
||||
# ========== 认证端点 ==========
|
||||
generator.add_endpoint(
|
||||
'POST', '/api/v2/admin/auth/verify',
|
||||
summary='验证认证状态',
|
||||
description='验证当前请求的认证状态',
|
||||
tags=['Authentication']
|
||||
)
|
||||
|
||||
return generator.generate()
|
||||
|
||||
|
||||
def save_api_docs(config: Dict, filepath: str = None, format: str = 'json'):
|
||||
"""
|
||||
生成并保存 API 文档
|
||||
|
||||
Args:
|
||||
config: 服务器配置
|
||||
filepath: 保存路径
|
||||
format: 输出格式 (json, yaml)
|
||||
"""
|
||||
doc = generate_api_docs(config)
|
||||
|
||||
if filepath is None:
|
||||
# 默认保存到项目根目录
|
||||
script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
filepath = os.path.join(script_dir, 'docs', 'api-docs.json')
|
||||
|
||||
# 确保目录存在
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
|
||||
if format == 'yaml':
|
||||
try:
|
||||
import yaml
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(doc, f, default_flow_style=False, allow_unicode=True)
|
||||
except ImportError:
|
||||
format = 'json'
|
||||
|
||||
if format == 'json':
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(doc, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return filepath
|
||||
@@ -0,0 +1,677 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
缓存预热模块
|
||||
预热常用镜像缓存提高命中率
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
import logging
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PrewarmPriority(Enum):
|
||||
"""预热优先级"""
|
||||
CRITICAL = "critical" # 关键，必须预热
|
||||
HIGH = "high" # 高优先级
|
||||
MEDIUM = "medium" # 中优先级
|
||||
LOW = "low" # 低优先级
|
||||
|
||||
|
||||
class ItemStatus(Enum):
|
||||
"""项目状态"""
|
||||
PENDING = "pending"
|
||||
IN_PROGRESS = "in_progress"
|
||||
SUCCESS = "success"
|
||||
FAILED = "failed"
|
||||
SKIPPED = "skipped"
|
||||
|
||||
|
||||
@dataclass
|
||||
class PrewarmItem:
|
||||
"""预热项目"""
|
||||
id: str
|
||||
mirror_type: str
|
||||
item_name: str
|
||||
url: str
|
||||
priority: str
|
||||
status: str = field(default=ItemStatus.PENDING.value)
|
||||
attempts: int = 0
|
||||
max_attempts: int = 3
|
||||
response_time_ms: float = 0
|
||||
error_message: str = None
|
||||
prewarmed_at: float = None
|
||||
size_bytes: int = 0
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
return {
|
||||
'id': self.id,
|
||||
'mirror_type': self.mirror_type,
|
||||
'item_name': self.item_name,
|
||||
'url': self.url,
|
||||
'priority': self.priority,
|
||||
'status': self.status,
|
||||
'attempts': self.attempts,
|
||||
'response_time_ms': self.response_time_ms,
|
||||
'error_message': self.error_message,
|
||||
'prewarmed_at': self.prewarmed_at,
|
||||
'size_bytes': self.size_bytes
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PrewarmTarget:
|
||||
"""预热目标配置"""
|
||||
mirror_type: str
|
||||
priority: str
|
||||
limit: int
|
||||
items: List[str] = field(default_factory=list) # 指定的预热项目列表
|
||||
tags: List[str] = field(default_factory=list) # 按标签筛选
|
||||
|
||||
|
||||
class CachePrewarmer:
|
||||
"""缓存预热管理器"""
|
||||
|
||||
def __init__(self, config: Dict = None):
|
||||
"""
|
||||
初始化缓存预热管理器
|
||||
|
||||
Args:
|
||||
config: 预热配置
|
||||
"""
|
||||
self.config = config or {}
|
||||
self.enabled = self.config.get('enabled', False)
|
||||
|
||||
# 预热目标
|
||||
self.targets = self._parse_targets(self.config.get('targets', []))
|
||||
|
||||
# 调度配置
|
||||
self.schedule = self.config.get('schedule', '0 3 * * *') # 默认每天凌晨3点
|
||||
self.batch_size = self.config.get('batch_size', 10)
|
||||
|
||||
# 状态管理
|
||||
self._items: Dict[str, PrewarmItem] = {}
|
||||
self._items_lock = threading.Lock()
|
||||
self._is_running = False
|
||||
self._run_lock = threading.Lock()
|
||||
|
||||
# 历史记录
|
||||
self._history: List[Dict] = []
|
||||
self._history_lock = threading.Lock()
|
||||
|
||||
# 回调函数
|
||||
self._on_start: Optional[Callable] = None
|
||||
self._on_complete: Optional[Callable] = None
|
||||
self._on_item_complete: Optional[Callable] = None
|
||||
self._on_error: Optional[Callable] = None
|
||||
|
||||
# HTTP 会话
|
||||
self._session = requests.Session()
|
||||
self._session.headers.update({
|
||||
'User-Agent': 'HYC-Mirror-Prewarmer/1.0'
|
||||
})
|
||||
|
||||
# 超时设置
|
||||
self._request_timeout = self.config.get('request_timeout', 30)
|
||||
|
||||
# 常用镜像包列表
|
||||
self._popular_items = self._load_popular_items()
|
||||
|
||||
def _parse_targets(self, targets_config: List[Dict]) -> List[PrewarmTarget]:
|
||||
"""解析预热目标配置"""
|
||||
targets = []
|
||||
for t in targets_config:
|
||||
targets.append(PrewarmTarget(
|
||||
mirror_type=t.get('mirror_type', 'docker'),
|
||||
priority=t.get('priority', 'medium'),
|
||||
limit=t.get('limit', 100),
|
||||
items=t.get('items', []),
|
||||
tags=t.get('tags', [])
|
||||
))
|
||||
return targets
|
||||
|
||||
def _load_popular_items(self) -> Dict[str, List[str]]:
|
||||
"""加载常用镜像包列表"""
|
||||
return {
|
||||
'docker': [
|
||||
'library/alpine:latest',
|
||||
'library/ubuntu:latest',
|
||||
'library/debian:latest',
|
||||
'library/centos:latest',
|
||||
'library/nginx:latest',
|
||||
'library/python:3.9',
|
||||
'library/python:3.10',
|
||||
'library/node:18',
|
||||
'library/node:20',
|
||||
'library/go:1.20',
|
||||
'library/redis:alpine',
|
||||
'library/mysql:8',
|
||||
'library/postgres:15',
|
||||
],
|
||||
'pip': [
|
||||
'requests',
|
||||
'numpy',
|
||||
'pandas',
|
||||
'flask',
|
||||
'django',
|
||||
'scipy',
|
||||
'scikit-learn',
|
||||
'torch',
|
||||
'tensorflow',
|
||||
'celery',
|
||||
'pytest',
|
||||
'black',
|
||||
],
|
||||
'npm': [
|
||||
'react',
|
||||
'vue',
|
||||
'angular',
|
||||
'lodash',
|
||||
'express',
|
||||
'axios',
|
||||
'typescript',
|
||||
'webpack',
|
||||
'vite',
|
||||
'eslint',
|
||||
],
|
||||
'apt': [
|
||||
'ubuntu-desktop',
|
||||
'ubuntu-standard',
|
||||
'nginx',
|
||||
'python3-pip',
|
||||
'nodejs',
|
||||
'docker.io',
|
||||
],
|
||||
'yum': [
|
||||
'epel-release',
|
||||
'nginx',
|
||||
'docker-ce',
|
||||
'python3-pip',
|
||||
'nodejs',
|
||||
],
|
||||
'go': [
|
||||
'golang.org/x/tools',
|
||||
'github.com/gin-gonic/gin',
|
||||
'github.com/beego/beego',
|
||||
'github.com/gorilla/mux',
|
||||
],
|
||||
}
|
||||
|
||||
def _get_base_url(self, mirror_type: str) -> str:
|
||||
"""获取镜像源基础 URL"""
|
||||
mirrors = self.config.get('mirrors', {})
|
||||
|
||||
if isinstance(mirrors, dict):
|
||||
mirror_config = mirrors.get(mirror_type, {})
|
||||
if isinstance(mirror_config, dict):
|
||||
sources = mirror_config.get('sources', [])
|
||||
if sources:
|
||||
source_config = mirror_config.get('sources_config', {}).get(sources[0], {})
|
||||
return source_config.get('url', '')
|
||||
|
||||
return ''
|
||||
|
||||
def _generate_url(self, mirror_type: str, item_name: str) -> str:
|
||||
"""生成预热 URL"""
|
||||
base_url = self._get_base_url(mirror_type)
|
||||
|
||||
if not base_url:
|
||||
return ''
|
||||
|
||||
if mirror_type == 'docker':
|
||||
return f"{base_url}/v2/{item_name}/manifests/latest"
|
||||
elif mirror_type == 'pip':
|
||||
return f"{base_url}/simple/{item_name}/"
|
||||
elif mirror_type == 'npm':
|
||||
return f"{base_url}/{item_name}"
|
||||
elif mirror_type == 'apt':
|
||||
return f"{base_url}/dists/{item_name}/InRelease"
|
||||
elif mirror_type == 'yum':
|
||||
return f"{base_url}/repodata/repomd.xml"
|
||||
elif mirror_type == 'go':
|
||||
return f"{base_url}/{item_name}?go-get=1"
|
||||
else:
|
||||
return f"{base_url}/{item_name}"
|
||||
|
||||
def _create_item(
|
||||
self,
|
||||
mirror_type: str,
|
||||
item_name: str,
|
||||
priority: str = 'medium'
|
||||
) -> PrewarmItem:
|
||||
"""创建预热项目"""
|
||||
url = self._generate_url(mirror_type, item_name)
|
||||
|
||||
return PrewarmItem(
|
||||
id=f"{mirror_type}_{item_name}_{int(time.time())}",
|
||||
mirror_type=mirror_type,
|
||||
item_name=item_name,
|
||||
url=url,
|
||||
priority=priority
|
||||
)
|
||||
|
||||
def add_item(self, item: PrewarmItem):
|
||||
"""添加预热项目"""
|
||||
with self._items_lock:
|
||||
self._items[item.id] = item
|
||||
|
||||
def add_items_batch(self, mirror_type: str, items: List[str], priority: str = 'medium'):
|
||||
"""批量添加预热项目"""
|
||||
for item_name in items:
|
||||
item = self._create_item(mirror_type, item_name, priority)
|
||||
self.add_item(item)
|
||||
|
||||
def _prewarm_item(self, item: PrewarmItem) -> PrewarmItem:
|
||||
"""
|
||||
预热单个项目
|
||||
|
||||
Args:
|
||||
item: 预热项目
|
||||
|
||||
Returns:
|
||||
更新后的项目
|
||||
"""
|
||||
item.attempts += 1
|
||||
item.status = ItemStatus.IN_PROGRESS.value
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
|
||||
response = self._session.get(
|
||||
item.url,
|
||||
timeout=self._request_timeout,
|
||||
allow_redirects=True
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
item.response_time_ms = round((time.time() - start_time) * 1000, 2)
|
||||
item.size_bytes = len(response.content)
|
||||
item.status = ItemStatus.SUCCESS.value
|
||||
item.prewarmed_at = time.time()
|
||||
|
||||
logger.debug(f"Prewarmed {item.mirror_type}/{item.item_name}: {item.response_time_ms}ms")
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
item.error_message = f"Timeout after {self._request_timeout}s"
|
||||
if item.attempts < item.max_attempts:
|
||||
item.status = ItemStatus.PENDING.value
|
||||
else:
|
||||
item.status = ItemStatus.FAILED.value
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
item.error_message = f"HTTP Error: {e.response.status_code}"
|
||||
if item.attempts < item.max_attempts:
|
||||
item.status = ItemStatus.PENDING.value
|
||||
else:
|
||||
item.status = ItemStatus.FAILED.value
|
||||
|
||||
except Exception as e:
|
||||
item.error_message = str(e)
|
||||
item.status = ItemStatus.FAILED.value
|
||||
|
||||
return item
|
||||
|
||||
def run(self, targets: List[PrewarmTarget] = None) -> Dict:
|
||||
"""
|
||||
执行预热
|
||||
|
||||
Args:
|
||||
targets: 预热目标列表，None 表示使用配置的默认目标
|
||||
|
||||
Returns:
|
||||
预热结果
|
||||
"""
|
||||
with self._run_lock:
|
||||
if self._is_running:
|
||||
return {
|
||||
'success': False,
|
||||
'error': 'Prewarm already running'
|
||||
}
|
||||
|
||||
self._is_running = True
|
||||
|
||||
start_time = time.time()
|
||||
result = {
|
||||
'success': True,
|
||||
'total_items': 0,
|
||||
'success_count': 0,
|
||||
'failed_count': 0,
|
||||
'skipped_count': 0,
|
||||
'elapsed_seconds': 0,
|
||||
'errors': []
|
||||
}
|
||||
|
||||
try:
|
||||
# 通知开始
|
||||
if self._on_start:
|
||||
try:
|
||||
self._on_start()
|
||||
except Exception as e:
|
||||
logger.error(f"Prewarm start callback failed: {e}")
|
||||
|
||||
# 确定要预热的目标
|
||||
if targets is None:
|
||||
targets = self.targets
|
||||
|
||||
# 如果没有指定目标，使用流行项目
|
||||
if not targets:
|
||||
for mirror_type, items in self._popular_items.items():
|
||||
targets.append(PrewarmTarget(
|
||||
mirror_type=mirror_type,
|
||||
priority='medium',
|
||||
limit=len(items),
|
||||
items=items
|
||||
))
|
||||
|
||||
# 添加项目到队列
|
||||
total_added = 0
|
||||
for target in targets:
|
||||
if target.items:
|
||||
# 使用指定的预热项目
|
||||
for item_name in target.items[:target.limit]:
|
||||
item = self._create_item(target.mirror_type, item_name, target.priority)
|
||||
self.add_item(item)
|
||||
total_added += 1
|
||||
else:
|
||||
# 使用流行项目列表
|
||||
popular = self._popular_items.get(target.mirror_type, [])
|
||||
for item_name in popular[:target.limit]:
|
||||
item = self._create_item(target.mirror_type, item_name, target.priority)
|
||||
self.add_item(item)
|
||||
total_added += 1
|
||||
|
||||
result['total_items'] = total_added
|
||||
|
||||
# 按优先级排序
|
||||
priority_order = {
|
||||
'critical': 0,
|
||||
'high': 1,
|
||||
'medium': 2,
|
||||
'low': 3
|
||||
}
|
||||
|
||||
with self._items_lock:
|
||||
sorted_items = sorted(
|
||||
self._items.values(),
|
||||
key=lambda x: (priority_order.get(x.priority, 99), x.id)
|
||||
)
|
||||
|
||||
# 分批执行
|
||||
completed = 0
|
||||
with ThreadPoolExecutor(max_workers=self.batch_size) as executor:
|
||||
futures = {
|
||||
executor.submit(self._prewarm_item, item): item
|
||||
for item in sorted_items
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
item = futures[future]
|
||||
try:
|
||||
updated_item = future.result()
|
||||
|
||||
# 更新项目状态
|
||||
with self._items_lock:
|
||||
self._items[item.id] = updated_item
|
||||
|
||||
# 统计
|
||||
if updated_item.status == ItemStatus.SUCCESS.value:
|
||||
result['success_count'] += 1
|
||||
elif updated_item.status == ItemStatus.FAILED.value:
|
||||
result['failed_count'] += 1
|
||||
result['errors'].append({
|
||||
'item': updated_item.item_name,
|
||||
'error': updated_item.error_message
|
||||
})
|
||||
else:
|
||||
result['skipped_count'] += 1
|
||||
|
||||
completed += 1
|
||||
|
||||
# 回调
|
||||
if self._on_item_complete:
|
||||
try:
|
||||
self._on_item_complete(updated_item)
|
||||
except Exception as e:
|
||||
logger.error(f"Item complete callback failed: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Prewarm execution error: {e}")
|
||||
result['failed_count'] += 1
|
||||
result['errors'].append({
|
||||
'item': item.item_name,
|
||||
'error': str(e)
|
||||
})
|
||||
|
||||
result['elapsed_seconds'] = round(time.time() - start_time, 2)
|
||||
|
||||
# 记录历史
|
||||
self._add_to_history(result)
|
||||
|
||||
# 通知完成
|
||||
if self._on_complete:
|
||||
try:
|
||||
self._on_complete(result)
|
||||
except Exception as e:
|
||||
logger.error(f"Prewarm complete callback failed: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Prewarm failed: {e}")
|
||||
result['success'] = False
|
||||
result['errors'].append({'error': str(e)})
|
||||
|
||||
finally:
|
||||
self._is_running = False
|
||||
|
||||
return result
|
||||
|
||||
def _add_to_history(self, result: Dict):
|
||||
"""添加历史记录"""
|
||||
record = {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'success': result.get('success', False),
|
||||
'total_items': result.get('total_items', 0),
|
||||
'success_count': result.get('success_count', 0),
|
||||
'failed_count': result.get('failed_count', 0),
|
||||
'elapsed_seconds': result.get('elapsed_seconds', 0)
|
||||
}
|
||||
|
||||
with self._history_lock:
|
||||
self._history.append(record)
|
||||
# 只保留最近 50 条记录
|
||||
self._history = self._history[-50:]
|
||||
|
||||
def get_status(self) -> Dict:
|
||||
"""获取预热状态"""
|
||||
with self._items_lock:
|
||||
items = list(self._items.values())
|
||||
|
||||
total = len(items)
|
||||
success = sum(1 for i in items if i.status == ItemStatus.SUCCESS.value)
|
||||
failed = sum(1 for i in items if i.status == ItemStatus.FAILED.value)
|
||||
in_progress = sum(1 for i in items if i.status == ItemStatus.IN_PROGRESS.value)
|
||||
pending = sum(1 for i in items if i.status == ItemStatus.PENDING.value)
|
||||
|
||||
return {
|
||||
'enabled': self.enabled,
|
||||
'is_running': self._is_running,
|
||||
'total_items': total,
|
||||
'success_count': success,
|
||||
'failed_count': failed,
|
||||
'in_progress_count': in_progress,
|
||||
'pending_count': pending,
|
||||
'success_rate': (success / total * 100) if total > 0 else 0,
|
||||
'targets_count': len(self.targets)
|
||||
}
|
||||
|
||||
def get_items(
|
||||
self,
|
||||
status: str = None,
|
||||
mirror_type: str = None,
|
||||
limit: int = 50
|
||||
) -> List[Dict]:
|
||||
"""获取预热项目列表"""
|
||||
with self._items_lock:
|
||||
items = [i.to_dict() for i in self._items.values()]
|
||||
|
||||
# 过滤
|
||||
if status:
|
||||
items = [i for i in items if i['status'] == status]
|
||||
if mirror_type:
|
||||
items = [i for i in items if i['mirror_type'] == mirror_type]
|
||||
|
||||
return items[-limit:]
|
||||
|
||||
def get_history(self, limit: int = 20) -> List[Dict]:
|
||||
"""获取预热历史"""
|
||||
with self._history_lock:
|
||||
return list(self._history[-limit:])
|
||||
|
||||
def get_stats(self) -> Dict:
|
||||
"""获取统计信息"""
|
||||
status = self.get_status()
|
||||
history = self.get_history(10)
|
||||
|
||||
avg_duration = 0
|
||||
if history:
|
||||
durations = [h['elapsed_seconds'] for h in history if 'elapsed_seconds' in h]
|
||||
if durations:
|
||||
avg_duration = sum(durations) / len(durations)
|
||||
|
||||
return {
|
||||
'enabled': self.enabled,
|
||||
'is_running': self._is_running,
|
||||
'total_prewarmed': status['success_count'],
|
||||
'total_failed': status['failed_count'],
|
||||
'success_rate': status['success_rate'],
|
||||
'avg_duration_seconds': round(avg_duration, 2),
|
||||
'recent_runs': len(history),
|
||||
'targets': [
|
||||
{
|
||||
'mirror_type': t.mirror_type,
|
||||
'priority': t.priority,
|
||||
'limit': t.limit
|
||||
}
|
||||
for t in self.targets
|
||||
]
|
||||
}
|
||||
|
||||
def clear_items(self, status: str = None):
|
||||
"""清除预热项目"""
|
||||
with self._items_lock:
|
||||
if status:
|
||||
self._items = {
|
||||
k: v for k, v in self._items.items()
|
||||
if v.status != status
|
||||
}
|
||||
else:
|
||||
self._items = {}
|
||||
|
||||
def set_start_callback(self, callback: Callable):
|
||||
"""设置开始回调"""
|
||||
self._on_start = callback
|
||||
|
||||
def set_complete_callback(self, callback: Callable):
|
||||
"""设置完成回调"""
|
||||
self._on_complete = callback
|
||||
|
||||
def set_item_complete_callback(self, callback: Callable):
|
||||
"""设置项目完成回调"""
|
||||
self._on_item_complete = callback
|
||||
|
||||
def set_error_callback(self, callback: Callable):
|
||||
"""设置错误回调"""
|
||||
self._on_error = callback
|
||||
|
||||
def get_popular_items(self, mirror_type: str) -> List[str]:
|
||||
"""获取指定镜像类型的流行项目列表"""
|
||||
return self._popular_items.get(mirror_type, [])
|
||||
|
||||
def add_popular_items_to_queue(
|
||||
self,
|
||||
mirror_type: str,
|
||||
limit: int = None,
|
||||
priority: str = 'medium'
|
||||
):
|
||||
"""
|
||||
添加流行项目到预热队列
|
||||
|
||||
Args:
|
||||
mirror_type: 镜像类型
|
||||
limit: 数量限制
|
||||
priority: 优先级
|
||||
"""
|
||||
popular = self._popular_items.get(mirror_type, [])
|
||||
if limit:
|
||||
popular = popular[:limit]
|
||||
|
||||
self.add_items_batch(mirror_type, popular, priority)
|
||||
|
||||
|
||||
class PrewarmScheduler:
|
||||
"""预热调度器"""
|
||||
|
||||
def __init__(self, config: Dict = None):
|
||||
self.config = config or {}
|
||||
self._scheduler = None
|
||||
self._running = False
|
||||
|
||||
def start(self, prewarmer: CachePrewarmer):
|
||||
"""启动调度器"""
|
||||
if self._running:
|
||||
return
|
||||
|
||||
schedule = self.config.get('schedule', '0 3 * * *')
|
||||
logger.info(f"Starting prewarm scheduler with schedule: {schedule}")
|
||||
|
||||
# 使用简单的时间间隔检查
|
||||
self._running = True
|
||||
self._scheduler_thread = threading.Thread(
|
||||
target=self._run_scheduler,
|
||||
args=(prewarmer,),
|
||||
daemon=True
|
||||
)
|
||||
self._scheduler_thread.start()
|
||||
|
||||
def _run_scheduler(self, prewarmer: CachePrewarmer):
|
||||
"""运行调度器"""
|
||||
import croniter
|
||||
|
||||
schedule = self.config.get('schedule', '0 3 * * *')
|
||||
|
||||
try:
|
||||
cron = croniter.croniter(schedule, datetime.now())
|
||||
next_run = cron.get_next(datetime)
|
||||
except Exception as e:
|
||||
logger.error(f"Invalid cron schedule: {e}")
|
||||
return
|
||||
|
||||
while self._running:
|
||||
now = datetime.now()
|
||||
if now >= next_run:
|
||||
logger.info("Running scheduled prewarm")
|
||||
try:
|
||||
prewarmer.run()
|
||||
except Exception as e:
|
||||
logger.error(f"Scheduled prewarm failed: {e}")
|
||||
cron = croniter.croniter(schedule, datetime.now())
|
||||
next_run = cron.get_next(datetime)
|
||||
|
||||
# 检查间隔
|
||||
time.sleep(60)
|
||||
|
||||
def stop(self):
|
||||
"""停止调度器"""
|
||||
self._running = False
|
||||
+350
@@ -0,0 +1,350 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""配置管理模块"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import hashlib
|
||||
import time
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .utils import parse_size
|
||||
|
||||
|
||||
def get_resource_path(relative_path: str) -> str:
|
||||
"""获取打包后的资源路径"""
|
||||
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
|
||||
# 打包后的路径
|
||||
return os.path.join(sys._MEIPASS, relative_path)
|
||||
return os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), relative_path)
|
||||
|
||||
# 打包模式
|
||||
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
|
||||
# 外部目录：与 exe 同级
|
||||
external_path = os.path.join(os.path.dirname(sys.executable), relative_path)
|
||||
if os.path.exists(external_path):
|
||||
return external_path
|
||||
|
||||
# 打包后的资源路径（_MEIPASS）
|
||||
bundled_path = os.path.join(sys._MEIPASS, relative_path)
|
||||
if os.path.exists(bundled_path):
|
||||
return bundled_path
|
||||
|
||||
return external_path
|
||||
|
||||
# 开发模式
|
||||
return os.path.join(project_root, relative_path)
|
||||
|
||||
|
||||
def deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
深度合并配置
|
||||
- 如果 override 中有的键，会完全替换 base 中的值（除非都是 dict）
|
||||
- 如果都是 dict，则递归合并
|
||||
- 不会修改原参数
|
||||
|
||||
Args:
|
||||
base: 默认配置（基础配置）
|
||||
override: 要合并的配置（优先级更高）
|
||||
|
||||
Returns:
|
||||
合并后的配置
|
||||
"""
|
||||
result = base.copy()
|
||||
|
||||
for key, value in override.items():
|
||||
if key in result and isinstance(result[key], dict) and isinstance(value, dict):
|
||||
# 两者都是字典，递归合并
|
||||
result[key] = deep_merge(result[key], value)
|
||||
else:
|
||||
# 直接覆盖
|
||||
result[key] = value
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def load_json_config(file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""加载 JSON 配置文件"""
|
||||
if not os.path.exists(file_path):
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"警告: 配置文件 {file_path} JSON 格式错误: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"警告: 无法读取配置文件 {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
"""配置管理器"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any] = None, settings_path: str = None):
|
||||
"""
|
||||
初始化配置管理器
|
||||
|
||||
Args:
|
||||
config: 传入的配置（会覆盖默认配置）
|
||||
settings_path: 默认配置文件路径（可以是位置参数或关键字参数）
|
||||
"""
|
||||
# 支持 settings_path 作为位置参数
|
||||
if isinstance(config, str):
|
||||
settings_path = config
|
||||
config = None
|
||||
elif settings_path is None:
|
||||
# 使用打包后的资源路径
|
||||
settings_path = get_resource_path('settings.json')
|
||||
elif settings_path:
|
||||
settings_path = settings_path
|
||||
|
||||
self._settings_path = settings_path
|
||||
|
||||
# 加载默认配置
|
||||
self.default_config = self._load_default_config()
|
||||
|
||||
# 合并传入的配置
|
||||
if config:
|
||||
self.config = self._validate_config(deep_merge(self.default_config, config))
|
||||
else:
|
||||
self.config = self._validate_config(self.default_config.copy())
|
||||
|
||||
def _load_default_config(self) -> Dict[str, Any]:
|
||||
"""加载默认配置文件"""
|
||||
default_config = load_json_config(self._settings_path)
|
||||
if default_config is None:
|
||||
# 如果找不到默认配置，使用内联最小配置
|
||||
default_config = {
|
||||
'server_name': 'HYC下载站',
|
||||
'host': '0.0.0.0',
|
||||
'port': 8080,
|
||||
'base_dir': './downloads',
|
||||
'api_version': 'v2',
|
||||
'directory_listing': True,
|
||||
'enable_stats': True,
|
||||
'auth_type': 'none',
|
||||
'max_upload_size': 1024 * 1024 * 1024,
|
||||
'timeout': 30,
|
||||
'verbose': 0,
|
||||
'enable_range': True,
|
||||
'ignore_hidden': True,
|
||||
'show_hash': False,
|
||||
'calculate_hash': False,
|
||||
'max_search_results': 100,
|
||||
'enable_ws': True,
|
||||
'enable_sse': True,
|
||||
'enable_monitor': True,
|
||||
'monitor_interval': 5,
|
||||
'enable_sync': True,
|
||||
'enable_mirrors': True,
|
||||
'database': {
|
||||
'enabled': True,
|
||||
'type': 'sqlite',
|
||||
'sqlite': {'path': './data/hyc.db'}
|
||||
},
|
||||
'mirrors': {
|
||||
'docker': {'enabled': True},
|
||||
'apt': {'enabled': True},
|
||||
'yum': {'enabled': True},
|
||||
'pypi': {'enabled': True},
|
||||
'npm': {'enabled': True},
|
||||
'go': {'enabled': True}
|
||||
},
|
||||
'sync_sources': {},
|
||||
'webhooks': {'enabled': False, 'storage': 'webhooks.json'},
|
||||
'auth_sessions_file': 'auth_sessions.json',
|
||||
'auth_session_timeout': 3600,
|
||||
'auth_cookie_max_age': 86400
|
||||
}
|
||||
print(f"警告: 未找到默认配置文件 ({self._settings_path})，使用内联默认配置")
|
||||
return default_config
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, custom_config: Dict[str, Any] = None, settings_path: str = None) -> 'ConfigManager':
|
||||
"""
|
||||
从默认配置创建配置管理器
|
||||
|
||||
Args:
|
||||
custom_config: 自定义配置，会覆盖默认配置
|
||||
settings_path: 默认配置文件路径
|
||||
|
||||
Returns:
|
||||
ConfigManager 实例
|
||||
"""
|
||||
return cls(config=custom_config, settings_path=settings_path)
|
||||
|
||||
def _validate_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""验证和修复配置"""
|
||||
# 确保必要配置存在
|
||||
required = ['base_dir', 'host', 'port']
|
||||
for key in required:
|
||||
if key not in config:
|
||||
raise ValueError(f"缺少必要配置: {key}")
|
||||
|
||||
# 修复路径配置
|
||||
config['base_dir'] = os.path.abspath(config['base_dir'])
|
||||
|
||||
# 设置默认值（不在 _validate_config 中处理，由默认配置提供）
|
||||
|
||||
# 验证认证配置
|
||||
auth_type = config.get('auth_type', 'none')
|
||||
if auth_type == 'basic':
|
||||
if 'auth_user' not in config:
|
||||
config['auth_user'] = 'admin'
|
||||
if 'auth_pass' not in config:
|
||||
config['auth_pass'] = 'admin123'
|
||||
elif auth_type == 'token':
|
||||
# 每次运行都重新生成标准的 token
|
||||
import secrets
|
||||
config['auth_token'] = secrets.token_hex(32)
|
||||
|
||||
# 验证上传大小配置
|
||||
if 'max_upload_size' in config:
|
||||
try:
|
||||
if isinstance(config['max_upload_size'], str):
|
||||
config['max_upload_size'] = parse_size(config['max_upload_size'])
|
||||
except ValueError as e:
|
||||
print(f"警告: 无效的上传大小配置: {e}")
|
||||
config['max_upload_size'] = 1024 * 1024 * 1024
|
||||
|
||||
# 验证端口范围
|
||||
if 'port' in config:
|
||||
port = config['port']
|
||||
if not (1 <= port <= 65535):
|
||||
raise ValueError(f"无效的端口号: {port}")
|
||||
|
||||
# 验证并创建必要目录
|
||||
base_dir = config['base_dir']
|
||||
try:
|
||||
if not os.path.exists(base_dir):
|
||||
os.makedirs(base_dir, exist_ok=True)
|
||||
|
||||
# 测试写入权限
|
||||
test_file = os.path.join(base_dir, '.write_test')
|
||||
with open(test_file, 'w') as f:
|
||||
f.write('test')
|
||||
os.remove(test_file)
|
||||
|
||||
except Exception as e:
|
||||
raise ValueError(f"基础目录无法访问: {e}")
|
||||
|
||||
# 获取项目根目录（脚本所在目录）
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
# 在项目根目录创建必要的数据目录
|
||||
necessary_dirs = [
|
||||
os.path.join(project_root, 'data'),
|
||||
os.path.join(project_root, 'logs'),
|
||||
]
|
||||
|
||||
for dir_path in necessary_dirs:
|
||||
if not os.path.exists(dir_path):
|
||||
try:
|
||||
os.makedirs(dir_path, exist_ok=True)
|
||||
except Exception as e:
|
||||
print(f"警告: 无法创建目录 {dir_path}: {e}")
|
||||
|
||||
# 更新配置指向项目根目录
|
||||
config['data_dir'] = os.path.join(project_root, 'data')
|
||||
config['logs_dir'] = os.path.join(project_root, 'logs')
|
||||
|
||||
return config
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
"""获取配置项"""
|
||||
return self.config.get(key, default)
|
||||
|
||||
def update(self, updates: Dict[str, Any]):
|
||||
"""更新配置"""
|
||||
self.config.update(updates)
|
||||
self.config = self._validate_config(self.config)
|
||||
|
||||
def get_full_config(self) -> Dict[str, Any]:
|
||||
"""获取完整配置字典"""
|
||||
return self.config.copy()
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""返回配置字典（不包含敏感信息）"""
|
||||
safe_config = {
|
||||
"server_name": self.config.get("server_name", "Mirror Server"),
|
||||
"version": "2.2",
|
||||
"base_dir": self.config['base_dir'],
|
||||
"directory_listing": self.config.get('directory_listing', True),
|
||||
"max_upload_size": self.config.get('max_upload_size'),
|
||||
"enable_stats": self.config.get('enable_stats', True),
|
||||
"auth_type": self.config.get('auth_type', 'none'),
|
||||
"sort_by": self.config.get('sort_by', 'name'),
|
||||
"sort_reverse": self.config.get('sort_reverse', False),
|
||||
"ignore_hidden": self.config.get('ignore_hidden', True),
|
||||
"enable_range": self.config.get('enable_range', True),
|
||||
"show_hash": self.config.get('show_hash', False),
|
||||
"calculate_hash": self.config.get('calculate_hash', False),
|
||||
"max_search_results": self.config.get('max_search_results', 100),
|
||||
"api_version": self.config.get('api_version', 'v1'),
|
||||
"verbose": self.config.get('verbose', 0)
|
||||
}
|
||||
return safe_config
|
||||
|
||||
|
||||
def load_config_file(config_path: str) -> Dict[str, Any]:
|
||||
"""加载配置文件"""
|
||||
if not os.path.exists(config_path):
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f"错误: 无法加载配置文件 {config_path}: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def load_settings_with_override(settings_path: str, override_path: str = None) -> Dict[str, Any]:
|
||||
"""
|
||||
加载默认配置并合并覆盖配置
|
||||
|
||||
Args:
|
||||
settings_path: 默认配置文件路径
|
||||
override_path: 覆盖配置文件路径（可选）
|
||||
|
||||
Returns:
|
||||
合并后的完整配置
|
||||
"""
|
||||
# 加载默认配置
|
||||
default_config = load_json_config(settings_path) or {}
|
||||
|
||||
# 加载覆盖配置
|
||||
override_config = {}
|
||||
if override_path:
|
||||
override_config = load_json_config(override_path) or {}
|
||||
|
||||
# 深度合并
|
||||
return deep_merge(default_config, override_config)
|
||||
|
||||
|
||||
def save_config_file(config_path: str, config: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
保存配置文件
|
||||
|
||||
Args:
|
||||
config_path: 保存路径
|
||||
config: 配置字典
|
||||
|
||||
Returns:
|
||||
是否保存成功
|
||||
"""
|
||||
try:
|
||||
# 创建目录
|
||||
os.makedirs(os.path.dirname(config_path), exist_ok=True)
|
||||
|
||||
with open(config_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(config, f, ensure_ascii=False, indent=4)
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"错误: 无法保存配置文件 {config_path}: {e}")
|
||||
return False
|
||||
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
配置热更新模块
|
||||
支持不重启服务的情况下重新加载配置
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
import logging
|
||||
from typing import Dict, Any, Optional, Callable
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConfigHotReloader:
|
||||
"""配置热重载管理器"""
|
||||
|
||||
def __init__(self, config_path: str, callback: Callable = None):
|
||||
"""
|
||||
初始化热重载管理器
|
||||
|
||||
Args:
|
||||
config_path: 配置文件路径
|
||||
callback: 配置变更时的回调函数，接收 (config, change_type) 参数
|
||||
"""
|
||||
self.config_path = config_path
|
||||
self.callback = callback
|
||||
|
||||
self._config: Dict[str, Any] = {}
|
||||
self._last_modified: float = 0
|
||||
self._last_load_time: Optional[datetime] = None
|
||||
self._lock = threading.RLock()
|
||||
|
||||
# 配置变更历史
|
||||
self._change_history: list = []
|
||||
|
||||
# 监听器
|
||||
self._listeners: Dict[str, list] = {
|
||||
'on_change': [],
|
||||
'on_error': []
|
||||
}
|
||||
|
||||
# 加载初始配置
|
||||
self.reload()
|
||||
|
||||
def reload(self, silent: bool = False) -> bool:
|
||||
"""
|
||||
重新加载配置
|
||||
|
||||
Args:
|
||||
silent: 静默模式，不触发变更通知
|
||||
|
||||
Returns:
|
||||
是否加载成功
|
||||
"""
|
||||
try:
|
||||
if not os.path.exists(self.config_path):
|
||||
if not silent:
|
||||
logger.warning(f"配置文件不存在: {self.config_path}")
|
||||
return False
|
||||
|
||||
# 获取文件修改时间
|
||||
current_mtime = os.path.getmtime(self.config_path)
|
||||
|
||||
# 检查是否有变化
|
||||
if current_mtime == self._last_modified and not silent:
|
||||
return True
|
||||
|
||||
# 加载配置
|
||||
with open(self.config_path, 'r', encoding='utf-8') as f:
|
||||
new_config = json.load(f)
|
||||
|
||||
# 计算变更
|
||||
changes = self._compute_changes(self._config, new_config)
|
||||
|
||||
with self._lock:
|
||||
old_config = self._config.copy()
|
||||
self._config = new_config
|
||||
self._last_modified = current_mtime
|
||||
self._last_load_time = datetime.now()
|
||||
|
||||
# 记录变更
|
||||
if changes:
|
||||
change_record = {
|
||||
'timestamp': self._last_load_time.isoformat(),
|
||||
'changes': changes,
|
||||
'old_config_keys': list(old_config.keys()),
|
||||
'new_config_keys': list(new_config.keys())
|
||||
}
|
||||
self._change_history.append(change_record)
|
||||
|
||||
# 保持历史记录在合理范围内
|
||||
if len(self._change_history) > 100:
|
||||
self._change_history = self._change_history[-50:]
|
||||
|
||||
if not silent and changes:
|
||||
self._notify_change(changes)
|
||||
|
||||
logger.info(f"配置已重新加载: {self.config_path}")
|
||||
return True
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
error_msg = f"配置 JSON 格式错误: {e}"
|
||||
logger.error(error_msg)
|
||||
self._notify_error(error_msg)
|
||||
return False
|
||||
except Exception as e:
|
||||
error_msg = f"加载配置失败: {e}"
|
||||
logger.error(error_msg)
|
||||
self._notify_error(error_msg)
|
||||
return False
|
||||
|
||||
def _compute_changes(self, old: Dict, new: Dict) -> Dict:
|
||||
"""计算配置变更"""
|
||||
changes = {
|
||||
'added': [],
|
||||
'removed': [],
|
||||
'modified': []
|
||||
}
|
||||
|
||||
old_keys = set(old.keys())
|
||||
new_keys = set(new.keys())
|
||||
|
||||
# 新增的键
|
||||
for key in new_keys - old_keys:
|
||||
changes['added'].append(key)
|
||||
|
||||
# 移除的键
|
||||
for key in old_keys - new_keys:
|
||||
changes['removed'].append(key)
|
||||
|
||||
# 修改的键
|
||||
for key in old_keys & new_keys:
|
||||
if old[key] != new[key]:
|
||||
# 检查是否是嵌套字典
|
||||
if isinstance(old[key], dict) and isinstance(new[key], dict):
|
||||
nested = self._compute_nested_changes(old[key], new[key], f"{key}.")
|
||||
if nested['added'] or nested['removed'] or nested['modified']:
|
||||
changes['modified'].append({
|
||||
'key': key,
|
||||
'type': 'nested',
|
||||
'changes': nested
|
||||
})
|
||||
else:
|
||||
changes['modified'].append({
|
||||
'key': key,
|
||||
'type': 'value',
|
||||
'old_value': old[key],
|
||||
'new_value': new[key]
|
||||
})
|
||||
|
||||
return changes
|
||||
|
||||
def _compute_nested_changes(self, old: Dict, new: Dict, prefix: str = "") -> Dict:
|
||||
"""计算嵌套字典的变更"""
|
||||
changes = {
|
||||
'added': [],
|
||||
'removed': [],
|
||||
'modified': []
|
||||
}
|
||||
|
||||
old_keys = set(old.keys())
|
||||
new_keys = set(new.keys())
|
||||
|
||||
for key in new_keys - old_keys:
|
||||
changes['added'].append(f"{prefix}{key}")
|
||||
|
||||
for key in old_keys - new_keys:
|
||||
changes['removed'].append(f"{prefix}{key}")
|
||||
|
||||
for key in old_keys & new_keys:
|
||||
if old[key] != new[key]:
|
||||
changes['modified'].append(f"{prefix}{key}")
|
||||
|
||||
return changes
|
||||
|
||||
def _notify_change(self, changes: Dict):
|
||||
"""通知配置变更"""
|
||||
for listener in self._listeners['on_change']:
|
||||
try:
|
||||
if callable(listener):
|
||||
listener(self._config, changes)
|
||||
except Exception as e:
|
||||
logger.error(f"配置变更监听器执行失败: {e}")
|
||||
|
||||
if self.callback:
|
||||
try:
|
||||
self.callback(self._config, changes)
|
||||
except Exception as e:
|
||||
logger.error(f"配置回调函数执行失败: {e}")
|
||||
|
||||
def _notify_error(self, error: str):
|
||||
"""通知错误"""
|
||||
for listener in self._listeners['on_error']:
|
||||
try:
|
||||
if callable(listener):
|
||||
listener(error)
|
||||
except Exception as e:
|
||||
logger.error(f"错误监听器执行失败: {e}")
|
||||
|
||||
def add_change_listener(self, callback: Callable):
|
||||
"""添加配置变更监听器"""
|
||||
self._listeners['on_change'].append(callback)
|
||||
|
||||
def add_error_listener(self, callback: Callable):
|
||||
"""添加错误监听器"""
|
||||
self._listeners['on_error'].append(callback)
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
"""获取配置值"""
|
||||
with self._lock:
|
||||
return self._config.get(key, default)
|
||||
|
||||
def get_all(self) -> Dict:
|
||||
"""获取完整配置"""
|
||||
with self._lock:
|
||||
return self._config.copy()
|
||||
|
||||
def set(self, key: str, value: Any, save: bool = True) -> bool:
|
||||
"""设置配置值（仅内存中）"""
|
||||
with self._lock:
|
||||
self._config[key] = value
|
||||
|
||||
if save:
|
||||
return self.save()
|
||||
|
||||
return True
|
||||
|
||||
def save(self, path: str = None) -> bool:
|
||||
"""保存配置到文件"""
|
||||
save_path = path or self.config_path
|
||||
|
||||
try:
|
||||
with open(save_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(self._config, f, ensure_ascii=False, indent=4)
|
||||
self._last_modified = os.path.getmtime(save_path)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存配置失败: {e}")
|
||||
return False
|
||||
|
||||
def get_change_history(self, limit: int = 10) -> list:
|
||||
"""获取配置变更历史"""
|
||||
return self._change_history[-limit:]
|
||||
|
||||
def watch(self, interval: float = 5.0):
|
||||
"""
|
||||
启动后台监控线程
|
||||
|
||||
Args:
|
||||
interval: 检查间隔（秒）
|
||||
"""
|
||||
def _watch_loop():
|
||||
while True:
|
||||
try:
|
||||
self.reload()
|
||||
except Exception as e:
|
||||
logger.error(f"配置监控错误: {e}")
|
||||
time.sleep(interval)
|
||||
|
||||
thread = threading.Thread(target=_watch_loop, daemon=True)
|
||||
thread.start()
|
||||
logger.info(f"配置热监控已启动，间隔: {interval}秒")
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
"""配置管理器 - 支持热更新"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any] = None):
|
||||
self.config = config or {}
|
||||
self._hot_reloader: Optional[ConfigHotReloader] = None
|
||||
|
||||
def load_from_file(self, path: str, enable_watch: bool = False) -> bool:
|
||||
"""从文件加载配置"""
|
||||
if not os.path.exists(path):
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
self.config = json.load(f)
|
||||
|
||||
if enable_watch:
|
||||
self._hot_reloader = ConfigHotReloader(path)
|
||||
self._hot_reloader.watch()
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"加载配置失败: {e}")
|
||||
return False
|
||||
|
||||
def hot_reload(self, path: str = None) -> bool:
|
||||
"""触发热重载"""
|
||||
if self._hot_reloader:
|
||||
return self._hot_reloader.reload()
|
||||
return False
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
"""获取配置值"""
|
||||
return self.config.get(key, default)
|
||||
|
||||
def set(self, key: str, value: Any, persist: bool = False, path: str = None) -> bool:
|
||||
"""设置配置值"""
|
||||
keys = key.split('.')
|
||||
current = self.config
|
||||
|
||||
for k in keys[:-1]:
|
||||
if k not in current:
|
||||
current[k] = {}
|
||||
current = current[k]
|
||||
|
||||
current[keys[-1]] = value
|
||||
|
||||
if persist:
|
||||
if self._hot_reloader:
|
||||
return self._hot_reloader.save(path)
|
||||
elif path:
|
||||
try:
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
json.dump(self.config, f, ensure_ascii=False, indent=4)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存配置失败: {e}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def add_change_listener(self, callback: Callable):
|
||||
"""添加变更监听器"""
|
||||
if self._hot_reloader:
|
||||
self._hot_reloader.add_change_listener(callback)
|
||||
|
||||
def get_all(self) -> Dict:
|
||||
"""获取完整配置"""
|
||||
return self.config.copy()
|
||||
+1545
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,603 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
平滑重启模块
|
||||
支持优雅停止、零停机重启、滚动更新
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import time
|
||||
import threading
|
||||
import logging
|
||||
import subprocess
|
||||
from typing import Dict, List, Optional, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RestartStrategy(Enum):
|
||||
"""重启策略"""
|
||||
GRACEFUL = "graceful" # 优雅停止，等待请求完成
|
||||
ROLLING = "rolling" # 滚动更新，零停机
|
||||
IMMEDIATE = "immediate" # 立即重启
|
||||
|
||||
|
||||
class ServerState(Enum):
|
||||
"""服务器状态"""
|
||||
RUNNING = "running"
|
||||
STOPPING = "stopping"
|
||||
STOPPED = "stopped"
|
||||
STARTING = "starting"
|
||||
RESTARTING = "restarting"
|
||||
|
||||
|
||||
@dataclass
|
||||
class PendingRequest:
|
||||
"""待处理的请求"""
|
||||
request_id: str
|
||||
start_time: float
|
||||
endpoint: str
|
||||
method: str
|
||||
client_address: tuple
|
||||
|
||||
|
||||
class GracefulRestartManager:
|
||||
"""平滑重启管理器"""
|
||||
|
||||
def __init__(self, config: Dict = None):
|
||||
"""
|
||||
初始化平滑重启管理器
|
||||
|
||||
Args:
|
||||
config: 重启配置
|
||||
"""
|
||||
self.config = config or {}
|
||||
self.state = ServerState.STOPPED
|
||||
|
||||
# 等待超时时间
|
||||
self.graceful_timeout = self.config.get('graceful_timeout', 30) # 秒
|
||||
self.shutdown_timeout = self.config.get('shutdown_timeout', 10) # 秒
|
||||
|
||||
# 待处理请求追踪
|
||||
self._pending_requests: Dict[str, PendingRequest] = {}
|
||||
self._requests_lock = threading.Lock()
|
||||
|
||||
# 请求计数器
|
||||
self._request_counter = 0
|
||||
self._counter_lock = threading.Lock()
|
||||
|
||||
# 回调函数
|
||||
self._on_prepare_restart: Optional[Callable] = None
|
||||
self._on_start_restart: Optional[Callable] = None
|
||||
self._on_complete_restart: Optional[Callable] = None
|
||||
self._on_restart_failed: Optional[Callable] = None
|
||||
|
||||
# 状态锁
|
||||
self._state_lock = threading.Lock()
|
||||
|
||||
# 重启历史
|
||||
self._restart_history: List[Dict] = []
|
||||
self._history_lock = threading.Lock()
|
||||
|
||||
# 信号处理器
|
||||
self._setup_signal_handlers()
|
||||
|
||||
def _setup_signal_handlers(self):
|
||||
"""设置信号处理器"""
|
||||
signal.signal(signal.SIGTERM, self._handle_signal)
|
||||
signal.signal(signal.SIGINT, self._handle_signal)
|
||||
signal.signal(signal.SIGHUP, self._handle_signal)
|
||||
|
||||
def _handle_signal(self, signum, frame):
|
||||
"""处理退出信号"""
|
||||
signal_names = {
|
||||
signal.SIGTERM: 'SIGTERM',
|
||||
signal.SIGINT: 'SIGINT',
|
||||
signal.SIGHUP: 'SIGHUP'
|
||||
}
|
||||
sig_name = signal_names.get(signum, str(signum))
|
||||
logger.info(f"收到信号 {sig_name}，准备优雅关闭")
|
||||
|
||||
# 设置状态为停止中
|
||||
self.state = ServerState.STOPPING
|
||||
|
||||
def register_request(self, endpoint: str, method: str, client: tuple) -> str:
|
||||
"""
|
||||
注册一个待处理的请求
|
||||
|
||||
Args:
|
||||
endpoint: 请求端点
|
||||
method: 请求方法
|
||||
client: 客户端地址
|
||||
|
||||
Returns:
|
||||
请求 ID
|
||||
"""
|
||||
with self._counter_lock:
|
||||
self._request_counter += 1
|
||||
request_id = f"req_{self._request_counter}_{int(time.time())}"
|
||||
|
||||
request = PendingRequest(
|
||||
request_id=request_id,
|
||||
start_time=time.time(),
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
client_address=client
|
||||
)
|
||||
|
||||
with self._requests_lock:
|
||||
self._pending_requests[request_id] = request
|
||||
|
||||
return request_id
|
||||
|
||||
def complete_request(self, request_id: str):
|
||||
"""标记请求完成"""
|
||||
with self._requests_lock:
|
||||
self._pending_requests.pop(request_id, None)
|
||||
|
||||
def get_pending_count(self) -> int:
|
||||
"""获取待处理请求数量"""
|
||||
with self._requests_lock:
|
||||
return len(self._pending_requests)
|
||||
|
||||
def get_pending_requests(self) -> List[Dict]:
|
||||
"""获取待处理请求详情"""
|
||||
with self._requests_lock:
|
||||
return [
|
||||
{
|
||||
'request_id': r.request_id,
|
||||
'endpoint': r.endpoint,
|
||||
'method': r.method,
|
||||
'duration': round(time.time() - r.start_time, 2),
|
||||
'client': str(r.client_address[0]) if r.client_address else 'unknown'
|
||||
}
|
||||
for r in self._pending_requests.values()
|
||||
]
|
||||
|
||||
def is_safe_to_restart(self) -> bool:
|
||||
"""检查是否安全重启（没有待处理请求）"""
|
||||
return self.get_pending_count() == 0
|
||||
|
||||
def wait_for_requests(
|
||||
self,
|
||||
timeout: int = None,
|
||||
progress_callback: Callable[[int, int], None] = None
|
||||
) -> bool:
|
||||
"""
|
||||
等待所有请求完成
|
||||
|
||||
Args:
|
||||
timeout: 超时时间（秒）
|
||||
progress_callback: 进度回调函数(remaining, total)
|
||||
|
||||
Returns:
|
||||
是否在超时前完成所有请求
|
||||
"""
|
||||
timeout = timeout or self.graceful_timeout
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
pending = self.get_pending_count()
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(pending, 0)
|
||||
|
||||
if pending == 0:
|
||||
return True
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
return False
|
||||
|
||||
def set_prepare_restart_callback(self, callback: Callable):
|
||||
"""设置准备重启回调"""
|
||||
self._on_prepare_restart = callback
|
||||
|
||||
def set_start_restart_callback(self, callback: Callable):
|
||||
"""设置开始重启回调"""
|
||||
self._on_start_restart = callback
|
||||
|
||||
def set_complete_restart_callback(self, callback: Callable):
|
||||
"""设置完成重启回调"""
|
||||
self._on_complete_restart = callback
|
||||
|
||||
def set_restart_failed_callback(self, callback: Callable):
|
||||
"""设置重启失败回调"""
|
||||
self._on_restart_failed = callback
|
||||
|
||||
def prepare_restart(self) -> Dict:
|
||||
"""
|
||||
准备重启（通知各模块准备）
|
||||
|
||||
Returns:
|
||||
准备结果
|
||||
"""
|
||||
with self._state_lock:
|
||||
self.state = ServerState.STOPPING
|
||||
|
||||
result = {
|
||||
'success': True,
|
||||
'pending_requests': self.get_pending_count(),
|
||||
'message': ''
|
||||
}
|
||||
|
||||
# 通知回调
|
||||
if self._on_prepare_restart:
|
||||
try:
|
||||
self._on_prepare_restart()
|
||||
except Exception as e:
|
||||
logger.error(f"Prepare restart callback failed: {e}")
|
||||
result['success'] = False
|
||||
result['message'] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def perform_restart(
|
||||
self,
|
||||
strategy: RestartStrategy = RestartStrategy.GRACEFUL,
|
||||
new_config: Dict = None,
|
||||
script_path: str = None
|
||||
) -> Dict:
|
||||
"""
|
||||
执行重启
|
||||
|
||||
Args:
|
||||
strategy: 重启策略
|
||||
new_config: 新配置（用于配置热更新）
|
||||
script_path: 服务器脚本路径
|
||||
|
||||
Returns:
|
||||
重启结果
|
||||
"""
|
||||
with self._state_lock:
|
||||
if self.state == ServerState.RESTARTING:
|
||||
return {
|
||||
'success': False,
|
||||
'error': 'Restart already in progress'
|
||||
}
|
||||
self.state = ServerState.RESTARTING
|
||||
|
||||
start_time = time.time()
|
||||
result = {
|
||||
'success': False,
|
||||
'strategy': strategy.value,
|
||||
'elapsed_seconds': 0,
|
||||
'message': ''
|
||||
}
|
||||
|
||||
try:
|
||||
# 准备阶段
|
||||
prepare_result = self.prepare_restart()
|
||||
if not prepare_result['success']:
|
||||
result['message'] = f"Prepare failed: {prepare_result['message']}"
|
||||
self.state = ServerState.RUNNING
|
||||
return result
|
||||
|
||||
# 根据策略执行重启
|
||||
if strategy == RestartStrategy.GRACEFUL:
|
||||
result = self._graceful_restart(prepare_result)
|
||||
elif strategy == RestartStrategy.ROLLING:
|
||||
result = self._rolling_restart(script_path, new_config)
|
||||
elif strategy == RestartStrategy.IMMEDIATE:
|
||||
result = self._immediate_restart(script_path)
|
||||
|
||||
result['elapsed_seconds'] = round(time.time() - start_time, 2)
|
||||
|
||||
# 记录重启历史
|
||||
self._add_to_history(result)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Restart failed: {e}")
|
||||
result['success'] = False
|
||||
result['message'] = str(e)
|
||||
self.state = ServerState.RUNNING
|
||||
|
||||
if self._on_restart_failed:
|
||||
try:
|
||||
self._on_restart_failed(str(e))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _graceful_restart(self, prepare_result: Dict) -> Dict:
|
||||
"""优雅重启（等待请求完成）"""
|
||||
result = {
|
||||
'success': True,
|
||||
'strategy': 'graceful',
|
||||
'pending_requests': prepare_result['pending_requests'],
|
||||
'message': ''
|
||||
}
|
||||
|
||||
# 等待请求完成
|
||||
pending = prepare_result['pending_requests']
|
||||
if pending > 0:
|
||||
logger.info(f"等待 {pending} 个请求完成，超时 {self.graceful_timeout} 秒")
|
||||
|
||||
def progress_callback(remaining, total):
|
||||
if remaining % 5 == 0:
|
||||
logger.info(f"还有 {remaining} 个请求待处理")
|
||||
|
||||
success = self.wait_for_requests(
|
||||
timeout=self.graceful_timeout,
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
|
||||
if not success:
|
||||
remaining = self.get_pending_count()
|
||||
result['success'] = False
|
||||
result['message'] = f"{remaining} 个请求未在 {self.graceful_timeout} 秒内完成"
|
||||
logger.warning(result['message'])
|
||||
self.state = ServerState.RUNNING
|
||||
return result
|
||||
|
||||
result['message'] = '所有请求已完成'
|
||||
logger.info('所有请求已完成')
|
||||
|
||||
# 通知开始重启
|
||||
if self._on_start_restart:
|
||||
try:
|
||||
self._on_start_restart()
|
||||
except Exception as e:
|
||||
logger.error(f"Start restart callback failed: {e}")
|
||||
|
||||
result['success'] = True
|
||||
result['message'] = 'Graceful restart prepared'
|
||||
self.state = ServerState.STOPPED
|
||||
|
||||
return result
|
||||
|
||||
def _rolling_restart(self, script_path: str, new_config: Dict = None) -> Dict:
|
||||
"""滚动重启（零停机）"""
|
||||
result = {
|
||||
'success': True,
|
||||
'strategy': 'rolling',
|
||||
'message': ''
|
||||
}
|
||||
|
||||
if not script_path:
|
||||
script_path = os.path.join(os.path.dirname(__file__), '..', 'main.py')
|
||||
|
||||
script_path = os.path.abspath(script_path)
|
||||
|
||||
# 检查是否有新配置需要应用
|
||||
if new_config:
|
||||
result['config_updated'] = True
|
||||
logger.info("配置将在重启后应用")
|
||||
else:
|
||||
result['config_updated'] = False
|
||||
|
||||
# 通知开始重启
|
||||
if self._on_start_restart:
|
||||
try:
|
||||
self._on_start_restart()
|
||||
except Exception as e:
|
||||
logger.error(f"Start restart callback failed: {e}")
|
||||
|
||||
# 启动新进程（在同一端口，但使用不同进程ID）
|
||||
# 注意：实际实现需要在负载均衡器层面处理
|
||||
logger.info("滚动重启：建议在负载均衡器层面处理零停机")
|
||||
|
||||
self.state = ServerState.STOPPED
|
||||
result['message'] = 'Rolling restart prepared - use load balancer for zero-downtime'
|
||||
|
||||
return result
|
||||
|
||||
def _immediate_restart(self, script_path: str = None) -> Dict:
|
||||
"""立即重启"""
|
||||
result = {
|
||||
'success': True,
|
||||
'strategy': 'immediate',
|
||||
'message': ''
|
||||
}
|
||||
|
||||
if not script_path:
|
||||
script_path = os.path.join(os.path.dirname(__file__), '..', 'main.py')
|
||||
|
||||
script_path = os.path.abspath(script_path)
|
||||
|
||||
# 通知开始重启
|
||||
if self._on_start_restart:
|
||||
try:
|
||||
self._on_start_restart()
|
||||
except Exception as e:
|
||||
logger.error(f"Start restart callback failed: {e}")
|
||||
|
||||
# 发送重启信号给主进程
|
||||
logger.info("立即重启服务器")
|
||||
|
||||
# 在子进程中重启
|
||||
try:
|
||||
# 启动新进程
|
||||
cmd = [sys.executable, script_path]
|
||||
env = os.environ.copy()
|
||||
env['HYC_RESTARTED'] = '1'
|
||||
|
||||
subprocess.Popen(cmd, env=env)
|
||||
|
||||
self.state = ServerState.STOPPED
|
||||
result['message'] = 'Immediate restart initiated'
|
||||
|
||||
except Exception as e:
|
||||
result['success'] = False
|
||||
result['message'] = f"Failed to restart: {str(e)}"
|
||||
self.state = ServerState.RUNNING
|
||||
|
||||
return result
|
||||
|
||||
def _add_to_history(self, result: Dict):
|
||||
"""添加重启历史记录"""
|
||||
record = {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'strategy': result.get('strategy', 'unknown'),
|
||||
'success': result.get('success', False),
|
||||
'elapsed_seconds': result.get('elapsed_seconds', 0),
|
||||
'message': result.get('message', '')
|
||||
}
|
||||
|
||||
with self._history_lock:
|
||||
self._restart_history.append(record)
|
||||
# 只保留最近 20 条记录
|
||||
self._restart_history = self._restart_history[-20:]
|
||||
|
||||
def get_restart_history(self) -> List[Dict]:
|
||||
"""获取重启历史"""
|
||||
with self._history_lock:
|
||||
return list(self._restart_history)
|
||||
|
||||
def get_stats(self) -> Dict:
|
||||
"""获取统计信息"""
|
||||
return {
|
||||
'state': self.state.value,
|
||||
'pending_requests': self.get_pending_count(),
|
||||
'graceful_timeout': self.graceful_timeout,
|
||||
'shutdown_timeout': self.shutdown_timeout,
|
||||
'restart_count': len(self._restart_history),
|
||||
'recent_restarts': self.get_restart_history()[-5:]
|
||||
}
|
||||
|
||||
def update_config(self, new_config: Dict):
|
||||
"""更新配置（热更新）"""
|
||||
if 'graceful_timeout' in new_config:
|
||||
self.graceful_timeout = new_config['graceful_timeout']
|
||||
if 'shutdown_timeout' in new_config:
|
||||
self.shutdown_timeout = new_config['shutdown_timeout']
|
||||
|
||||
logger.info(f"重启配置已更新: timeout={self.graceful_timeout}s")
|
||||
|
||||
|
||||
class ServerHealthChecker:
|
||||
"""服务器健康检查器"""
|
||||
|
||||
def __init__(self, host: str = 'localhost', port: int = 8080):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self._last_check = None
|
||||
self._is_healthy = False
|
||||
|
||||
def check(self) -> Dict:
|
||||
"""
|
||||
检查服务器健康状态
|
||||
|
||||
Returns:
|
||||
健康检查结果
|
||||
"""
|
||||
import socket
|
||||
|
||||
result = {
|
||||
'healthy': False,
|
||||
'latency_ms': 0,
|
||||
'error': None,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
|
||||
# 尝试连接
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(5)
|
||||
sock.connect((self.host, self.port))
|
||||
sock.close()
|
||||
|
||||
result['latency_ms'] = round((time.time() - start_time) * 1000, 2)
|
||||
result['healthy'] = True
|
||||
|
||||
except Exception as e:
|
||||
result['error'] = str(e)
|
||||
|
||||
self._last_check = result
|
||||
return result
|
||||
|
||||
def get_latency(self) -> float:
|
||||
"""获取延迟（毫秒）"""
|
||||
if self._last_check:
|
||||
return self._last_check.get('latency_ms', 0)
|
||||
return 0
|
||||
|
||||
|
||||
class RollingRestartManager:
|
||||
"""滚动重启管理器（支持多实例）"""
|
||||
|
||||
def __init__(self, config: Dict = None):
|
||||
self.config = config or {}
|
||||
self.instances: Dict[str, Dict] = {} # instance_id -> info
|
||||
self.current_instance_id = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def register_instance(self, instance_id: str, info: Dict):
|
||||
"""注册实例"""
|
||||
with self._lock:
|
||||
self.instances[instance_id] = {
|
||||
**info,
|
||||
'registered_at': datetime.now().isoformat(),
|
||||
'status': 'active'
|
||||
}
|
||||
|
||||
def unregister_instance(self, instance_id: str):
|
||||
"""注销实例"""
|
||||
with self._lock:
|
||||
if instance_id in self.instances:
|
||||
self.instances[instance_id]['status'] = 'draining'
|
||||
|
||||
def get_active_instances(self) -> List[str]:
|
||||
"""获取活跃实例列表"""
|
||||
with self._lock:
|
||||
return [
|
||||
i for i, info in self.instances.items()
|
||||
if info['status'] == 'active'
|
||||
]
|
||||
|
||||
def perform_rolling_restart(
|
||||
self,
|
||||
instance_id: str,
|
||||
restart_func: Callable
|
||||
) -> Dict:
|
||||
"""
|
||||
对单个实例执行滚动重启
|
||||
|
||||
Args:
|
||||
instance_id: 实例 ID
|
||||
restart_func: 重启函数
|
||||
|
||||
Returns:
|
||||
重启结果
|
||||
"""
|
||||
with self._lock:
|
||||
if instance_id not in self.instances:
|
||||
return {
|
||||
'success': False,
|
||||
'error': f'Instance {instance_id} not found'
|
||||
}
|
||||
|
||||
# 标记为排水中
|
||||
self.instances[instance_id]['status'] = 'draining'
|
||||
|
||||
# 等待连接耗尽
|
||||
time.sleep(5)
|
||||
|
||||
# 执行重启
|
||||
try:
|
||||
restart_func(instance_id)
|
||||
|
||||
self.instances[instance_id]['status'] = 'active'
|
||||
self.instances[instance_id]['restarted_at'] = datetime.now().isoformat()
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'instance_id': instance_id
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.instances[instance_id]['status'] = 'error'
|
||||
return {
|
||||
'success': False,
|
||||
'instance_id': instance_id,
|
||||
'error': str(e)
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
镜像源健康检查模块
|
||||
自动检测上游镜像源的可用性，支持故障切换
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
import logging
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Callable
|
||||
from enum import Enum
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HealthStatus(Enum):
|
||||
"""健康状态"""
|
||||
UNKNOWN = "unknown"
|
||||
HEALTHY = "healthy"
|
||||
DEGRADED = "degraded"
|
||||
UNHEALTHY = "unhealthy"
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthCheckResult:
|
||||
"""健康检查结果"""
|
||||
source_name: str
|
||||
status: HealthStatus
|
||||
response_time: float # 毫秒
|
||||
http_status: Optional[int] = None
|
||||
error_message: Optional[str] = None
|
||||
last_check: Optional[datetime] = None
|
||||
consecutive_failures: int = 0
|
||||
total_checks: int = 0
|
||||
success_rate: float = 100.0
|
||||
details: Dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class HealthChecker:
|
||||
"""健康检查器"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
"""
|
||||
初始化健康检查器
|
||||
|
||||
Args:
|
||||
config: 健康检查配置
|
||||
"""
|
||||
self.config = config or {}
|
||||
self.default_timeout = self.config.get('timeout', 10) # 秒
|
||||
self.default_interval = self.config.get('interval', 60) # 秒
|
||||
self.max_retries = self.config.get('max_retries', 3)
|
||||
self.failure_threshold = self.config.get('failure_threshold', 3) # 连续失败次数阈值
|
||||
|
||||
# 检查结果存储
|
||||
self._results: Dict[str, HealthCheckResult] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# 状态变化回调
|
||||
self._on_status_change: Optional[Callable] = None
|
||||
|
||||
def set_status_change_callback(self, callback: Callable):
|
||||
"""设置状态变化回调"""
|
||||
self._on_status_change = callback
|
||||
|
||||
def check_source(self, source_name: str, source_config: dict) -> HealthCheckResult:
|
||||
"""
|
||||
检查单个镜像源的健康状态
|
||||
|
||||
Args:
|
||||
source_name: 镜像源名称
|
||||
source_config: 镜像源配置
|
||||
|
||||
Returns:
|
||||
HealthCheckResult: 检查结果
|
||||
"""
|
||||
url = source_config.get('url', '')
|
||||
if not url:
|
||||
return HealthCheckResult(
|
||||
source_name=source_name,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
response_time=0,
|
||||
error_message="No URL configured",
|
||||
last_check=datetime.now()
|
||||
)
|
||||
|
||||
start_time = time.time()
|
||||
http_status = None
|
||||
error_message = None
|
||||
details = {}
|
||||
|
||||
try:
|
||||
# 发送 HEAD 请求（更轻量）
|
||||
req = urllib.request.Request(
|
||||
url.rstrip('/') + '/',
|
||||
method='HEAD',
|
||||
headers={
|
||||
'User-Agent': 'HYC-Mirror-HealthCheck/1.0',
|
||||
'Accept': '*/*'
|
||||
}
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(req, timeout=self.default_timeout) as response:
|
||||
http_status = response.status
|
||||
details['headers'] = dict(response.headers)
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
http_status = e.code
|
||||
error_message = f"HTTP {e.code}"
|
||||
except urllib.error.URLError as e:
|
||||
error_message = f"Connection error: {str(e.reason)}"
|
||||
except Exception as e:
|
||||
error_message = str(e)
|
||||
|
||||
response_time = (time.time() - start_time) * 1000 # 转换为毫秒
|
||||
|
||||
# 判断健康状态
|
||||
if error_message:
|
||||
if http_status and 400 <= http_status < 500:
|
||||
status = HealthStatus.DEGRADED # 客户端错误，可能暂时
|
||||
else:
|
||||
status = HealthStatus.UNHEALTHY
|
||||
elif http_status and 200 <= http_status < 400:
|
||||
status = HealthStatus.HEALTHY
|
||||
elif http_status:
|
||||
status = HealthStatus.DEGRADED
|
||||
else:
|
||||
status = HealthStatus.UNHEALTHY
|
||||
|
||||
# 计算统计数据
|
||||
with self._lock:
|
||||
if source_name not in self._results:
|
||||
self._results[source_name] = HealthCheckResult(
|
||||
source_name=source_name,
|
||||
status=status,
|
||||
response_time=response_time,
|
||||
last_check=datetime.now(),
|
||||
consecutive_failures=0,
|
||||
total_checks=1
|
||||
)
|
||||
else:
|
||||
old_result = self._results[source_name]
|
||||
consecutive_failures = old_result.consecutive_failures + (1 if status == HealthStatus.UNHEALTHY else 0)
|
||||
total_checks = old_result.total_checks + 1
|
||||
success_rate = ((total_checks - consecutive_failures) / total_checks) * 100
|
||||
|
||||
self._results[source_name] = HealthCheckResult(
|
||||
source_name=source_name,
|
||||
status=status,
|
||||
response_time=response_time,
|
||||
http_status=http_status,
|
||||
error_message=error_message,
|
||||
last_check=datetime.now(),
|
||||
consecutive_failures=consecutive_failures,
|
||||
total_checks=total_checks,
|
||||
success_rate=success_rate,
|
||||
details=details
|
||||
)
|
||||
|
||||
# 检查状态变化，触发回调
|
||||
if self._on_status_change and source_name in self._results:
|
||||
old_status = self._results[source_name].status
|
||||
if old_status != status:
|
||||
try:
|
||||
self._on_status_change(source_name, old_status, status)
|
||||
except Exception as e:
|
||||
logger.error(f"状态变化回调执行失败: {e}")
|
||||
|
||||
return self._results[source_name]
|
||||
|
||||
def get_all_results(self) -> List[HealthCheckResult]:
|
||||
"""获取所有检查结果"""
|
||||
with self._lock:
|
||||
return list(self._results.values())
|
||||
|
||||
def get_result(self, source_name: str) -> Optional[HealthCheckResult]:
|
||||
"""获取指定源的结果"""
|
||||
with self._lock:
|
||||
return self._results.get(source_name)
|
||||
|
||||
def is_healthy(self, source_name: str) -> bool:
|
||||
"""检查源是否健康"""
|
||||
result = self.get_result(source_name)
|
||||
if result is None:
|
||||
return True # 未检查过的默认健康
|
||||
return result.status == HealthStatus.HEALTHY
|
||||
|
||||
def get_unhealthy_sources(self) -> List[str]:
|
||||
"""获取不健康的源列表"""
|
||||
with self._lock:
|
||||
return [name for name, result in self._results.items()
|
||||
if result.status == HealthStatus.UNHEALTHY]
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""获取健康检查统计"""
|
||||
with self._lock:
|
||||
total = len(self._results)
|
||||
healthy = sum(1 for r in self._results.values() if r.status == HealthStatus.HEALTHY)
|
||||
degraded = sum(1 for r in self._results.values() if r.status == HealthStatus.DEGRADED)
|
||||
unhealthy = sum(1 for r in self._results.values() if r.status == HealthStatus.UNHEALTHY)
|
||||
|
||||
avg_response_time = 0
|
||||
if total > 0:
|
||||
avg_response_time = sum(r.response_time for r in self._results.values()) / total
|
||||
|
||||
return {
|
||||
'total_sources': total,
|
||||
'healthy': healthy,
|
||||
'degraded': degraded,
|
||||
'unhealthy': unhealthy,
|
||||
'avg_response_time_ms': round(avg_response_time, 2),
|
||||
'timestamp': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
|
||||
class MirrorFailoverManager:
|
||||
"""镜像源故障切换管理器"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
"""
|
||||
初始化故障切换管理器
|
||||
|
||||
Args:
|
||||
config: 配置，包含镜像源列表
|
||||
"""
|
||||
self.config = config or {}
|
||||
self.mirrors: Dict[str, Dict] = self.config.get('mirrors', {})
|
||||
|
||||
# 启用故障切换
|
||||
self.failover_enabled = self.config.get('failover_enabled', True)
|
||||
self.failover_threshold = self.config.get('failover_threshold', 3) # 连续失败次数
|
||||
|
||||
# 健康检查器
|
||||
self.health_checker = HealthChecker(self.config.get('health_check', {}))
|
||||
|
||||
# 当前活跃源
|
||||
self._active_source: Dict[str, str] = {} # mirror_type -> source_name
|
||||
self._source_priority: Dict[str, List[str]] = {} # mirror_type -> [优先列表]
|
||||
|
||||
# 故障切换历史
|
||||
self._failover_history: List[dict] = []
|
||||
|
||||
# 回调
|
||||
self._on_failover: Optional[Callable] = None
|
||||
|
||||
def set_failover_callback(self, callback: Callable):
|
||||
"""设置故障切换回调"""
|
||||
self._on_failover = callback
|
||||
|
||||
def initialize(self):
|
||||
"""初始化，确定各镜像类型的首选源"""
|
||||
for mirror_type, mirror_config in self.mirrors.items():
|
||||
if not isinstance(mirror_config, dict):
|
||||
continue
|
||||
|
||||
sources = mirror_config.get('sources', [])
|
||||
if sources:
|
||||
# 使用配置的优先列表
|
||||
self._source_priority[mirror_type] = sources
|
||||
else:
|
||||
# 使用内置的默认优先列表
|
||||
self._source_priority[mirror_type] = self._get_default_priority(mirror_type)
|
||||
|
||||
# 选择首选源
|
||||
if self._source_priority[mirror_type]:
|
||||
self._active_source[mirror_type] = self._source_priority[mirror_type][0]
|
||||
|
||||
def _get_default_priority(self, mirror_type: str) -> List[str]:
|
||||
"""获取默认的镜像源优先列表"""
|
||||
priorities = {
|
||||
'docker': ['docker.io', 'docker.mirrors.aliyun.com', 'dockerhub.azk8s.cn'],
|
||||
'apt': ['archive.ubuntu.com', 'mirrors.aliyun.com', 'security.ubuntu.com'],
|
||||
'yum': ['mirror.centos.org', 'mirrors.aliyun.com'],
|
||||
'pypi': ['pypi.org', 'pypi.mirrors.aliyun.com'],
|
||||
'npm': ['registry.npmjs.org', 'registry.npmmirror.com'],
|
||||
'go': ['proxy.golang.org', 'goproxy.cn']
|
||||
}
|
||||
return priorities.get(mirror_type, [])
|
||||
|
||||
def check_all(self) -> Dict[str, HealthCheckResult]:
|
||||
"""检查所有镜像源"""
|
||||
results = {}
|
||||
|
||||
for mirror_type, mirror_config in self.mirrors.items():
|
||||
if not isinstance(mirror_config, dict):
|
||||
continue
|
||||
|
||||
sources = mirror_config.get('sources', [])
|
||||
for source_name in sources:
|
||||
if source_name not in results:
|
||||
result = self.health_checker.check_source(source_name, {'url': self._get_source_url(mirror_type, source_name)})
|
||||
results[source_name] = result
|
||||
|
||||
return results
|
||||
|
||||
def _get_source_url(self, mirror_type: str, source_name: str) -> str:
|
||||
"""获取源 URL"""
|
||||
# 从配置中获取
|
||||
sources_config = self.mirrors.get(mirror_type, {}).get('sources_config', {})
|
||||
if source_name in sources_config:
|
||||
return sources_config[source_name].get('url', '')
|
||||
|
||||
# 从 URL 模板生成
|
||||
url_template = self.mirrors.get(mirror_type, {}).get('url_template', '')
|
||||
if url_template and '{mirror}' in url_template:
|
||||
return url_template.replace('{mirror}', source_name)
|
||||
|
||||
return ''
|
||||
|
||||
def get_active_source(self, mirror_type: str) -> Optional[str]:
|
||||
"""获取当前活跃的镜像源"""
|
||||
return self._active_source.get(mirror_type)
|
||||
|
||||
def get_source_for_request(self, mirror_type: str, original_url: str) -> str:
|
||||
"""
|
||||
根据故障切换策略获取请求的源 URL
|
||||
|
||||
Args:
|
||||
mirror_type: 镜像类型
|
||||
original_url: 原始 URL
|
||||
|
||||
Returns:
|
||||
str: 实际请求的 URL
|
||||
"""
|
||||
if not self.failover_enabled:
|
||||
return original_url
|
||||
|
||||
active_source = self._active_source.get(mirror_type)
|
||||
if not active_source:
|
||||
return original_url
|
||||
|
||||
source_url = self._get_source_url(mirror_type, active_source)
|
||||
if not source_url:
|
||||
return original_url
|
||||
|
||||
# 替换 URL 中的主机部分
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(original_url)
|
||||
|
||||
# 构建新 URL
|
||||
new_url = f"{parsed.scheme}://{source_url}{parsed.path}"
|
||||
if parsed.query:
|
||||
new_url += f"?{parsed.query}"
|
||||
|
||||
return new_url
|
||||
except Exception:
|
||||
return original_url
|
||||
|
||||
def perform_failover(self, mirror_type: str) -> bool:
|
||||
"""
|
||||
对指定镜像类型执行故障切换
|
||||
|
||||
Args:
|
||||
mirror_type: 镜像类型
|
||||
|
||||
Returns:
|
||||
bool: 是否成功切换
|
||||
"""
|
||||
priority_list = self._source_priority.get(mirror_type, [])
|
||||
if not priority_list:
|
||||
return False
|
||||
|
||||
current_index = 0
|
||||
if mirror_type in self._active_source:
|
||||
try:
|
||||
current_index = priority_list.index(self._active_source[mirror_type])
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# 查找下一个健康的源
|
||||
old_source = self._active_source.get(mirror_type)
|
||||
|
||||
for i in range(current_index + 1, len(priority_list)):
|
||||
source_name = priority_list[i]
|
||||
result = self.health_checker.get_result(source_name)
|
||||
|
||||
if result and result.status == HealthStatus.HEALTHY:
|
||||
self._active_source[mirror_type] = source_name
|
||||
|
||||
# 记录故障切换
|
||||
failover_record = {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'mirror_type': mirror_type,
|
||||
'old_source': old_source,
|
||||
'new_source': source_name,
|
||||
'reason': 'Health check failed'
|
||||
}
|
||||
self._failover_history.append(failover_record)
|
||||
|
||||
# 保持历史记录在合理范围内
|
||||
if len(self._failover_history) > 100:
|
||||
self._failover_history = self._failover_history[-50:]
|
||||
|
||||
# 触发回调
|
||||
if self._on_failover:
|
||||
try:
|
||||
self._on_failover(mirror_type, old_source, source_name)
|
||||
except Exception as e:
|
||||
logger.error(f"故障切换回调执行失败: {e}")
|
||||
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def get_failover_history(self, limit: int = 10) -> List[dict]:
|
||||
"""获取故障切换历史"""
|
||||
return self._failover_history[-limit:]
|
||||
|
||||
def get_health_summary(self) -> dict:
|
||||
"""获取健康状态摘要"""
|
||||
return {
|
||||
'failover_enabled': self.failover_enabled,
|
||||
'health': self.health_checker.get_stats(),
|
||||
'active_sources': self._active_source.copy(),
|
||||
'failover_history_count': len(self._failover_history)
|
||||
}
|
||||
+1258
File diff suppressed because it is too large
Load Diff
+376
@@ -0,0 +1,376 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
系统监控模块 - 提供实时系统监控功能
|
||||
支持CPU、内存、磁盘、网络等指标的实时采集和历史记录
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
|
||||
class SystemMonitor:
|
||||
"""系统监控器 - 采集和提供系统运行指标"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
self.base_dir = config.get('base_dir', './downloads')
|
||||
|
||||
# 历史数据配置
|
||||
self.history_file = config.get('monitor_history_file', 'monitor_history.json')
|
||||
self.history_hours = config.get('monitor_history_hours', 168) # 默认7天
|
||||
self.history = []
|
||||
self.history_lock = threading.Lock()
|
||||
|
||||
# SSE客户端管理
|
||||
self.sse_clients = {}
|
||||
self.sse_lock = threading.Lock()
|
||||
|
||||
# 监控配置
|
||||
self.collection_interval = config.get('monitor_interval', 5) # 采集间隔(秒)
|
||||
self.enabled = True
|
||||
|
||||
# 负载历史（用于计算趋势）
|
||||
self.load_history = []
|
||||
|
||||
# 加载历史数据
|
||||
self._load_history()
|
||||
|
||||
def get_realtime_stats(self) -> dict:
|
||||
"""获取实时系统状态"""
|
||||
import psutil
|
||||
import traceback
|
||||
|
||||
stats = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"errors": []
|
||||
}
|
||||
|
||||
try:
|
||||
# CPU信息
|
||||
cpu_percent = psutil.cpu_percent(interval=0.1)
|
||||
cpu_count = psutil.cpu_count()
|
||||
cpu_freq = psutil.cpu_freq()
|
||||
stats["cpu"] = {
|
||||
"percent": cpu_percent,
|
||||
"count": cpu_count,
|
||||
"freq_current": round(cpu_freq.current, 0) if cpu_freq else None,
|
||||
"freq_max": round(cpu_freq.max, 0) if cpu_freq else None,
|
||||
"freq_min": round(cpu_freq.min, 0) if cpu_freq else None,
|
||||
"per_core": psutil.cpu_percent(interval=None, percpu=True)
|
||||
}
|
||||
except Exception as e:
|
||||
stats["cpu"] = {"error": str(e)}
|
||||
stats["errors"].append(f"CPU: {str(e)}")
|
||||
|
||||
try:
|
||||
# 内存信息
|
||||
memory = psutil.virtual_memory()
|
||||
swap = psutil.swap_memory()
|
||||
stats["memory"] = {
|
||||
"total": memory.total,
|
||||
"available": memory.available,
|
||||
"used": memory.used,
|
||||
"percent": memory.percent,
|
||||
"swap_total": swap.total,
|
||||
"swap_used": swap.used,
|
||||
"swap_percent": swap.percent
|
||||
}
|
||||
except Exception as e:
|
||||
stats["memory"] = {"error": str(e)}
|
||||
stats["errors"].append(f"内存: {str(e)}")
|
||||
|
||||
try:
|
||||
# 负载平均值（可能在 Termux 中不可用）
|
||||
load_avg = os.getloadavg() if hasattr(os, 'getloadavg') else [0, 0, 0]
|
||||
if "cpu" in stats:
|
||||
stats["cpu"]["load_avg_1m"] = round(load_avg[0], 2)
|
||||
stats["cpu"]["load_avg_5m"] = round(load_avg[1], 2)
|
||||
stats["cpu"]["load_avg_15m"] = round(load_avg[2], 2)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
# 磁盘信息
|
||||
disk_usage = psutil.disk_usage(self.base_dir)
|
||||
disk_io = psutil.disk_io_counters()
|
||||
stats["disk"] = {
|
||||
"total": disk_usage.total,
|
||||
"used": disk_usage.used,
|
||||
"free": disk_usage.free,
|
||||
"percent": disk_usage.percent,
|
||||
"read_bytes": disk_io.read_bytes if disk_io else 0,
|
||||
"write_bytes": disk_io.write_bytes if disk_io else 0,
|
||||
"read_count": disk_io.read_count if disk_io else 0,
|
||||
"write_count": disk_io.write_count if disk_io else 0
|
||||
}
|
||||
except Exception as e:
|
||||
stats["disk"] = {"error": str(e)}
|
||||
stats["errors"].append(f"磁盘: {str(e)}")
|
||||
|
||||
try:
|
||||
# 网络信息（可能在受限环境中失败）
|
||||
net_io = psutil.net_io_counters()
|
||||
connections = psutil.net_connections()
|
||||
stats["network"] = {
|
||||
"bytes_sent": net_io.bytes_sent,
|
||||
"bytes_recv": net_io.bytes_recv,
|
||||
"packets_sent": net_io.packets_sent,
|
||||
"packets_recv": net_io.packets_recv,
|
||||
"connections_count": len(connections),
|
||||
"connections_established": len([c for c in connections if c.status == 'ESTABLISHED'])
|
||||
}
|
||||
except PermissionError as e:
|
||||
stats["network"] = {
|
||||
"note": "权限不足，无法访问网络信息",
|
||||
"error": str(e)
|
||||
}
|
||||
except Exception as e:
|
||||
stats["network"] = {"error": str(e)}
|
||||
stats["errors"].append(f"网络: {str(e)}")
|
||||
|
||||
try:
|
||||
# 进程信息
|
||||
process = psutil.Process()
|
||||
proc_mem = process.memory_info()
|
||||
proc_cpu = process.cpu_percent(interval=0.1)
|
||||
stats["process"] = {
|
||||
"memory_rss": proc_mem.rss,
|
||||
"memory_vms": proc_mem.vms,
|
||||
"cpu_percent": proc_cpu,
|
||||
"thread_count": process.num_threads(),
|
||||
"open_files": process.num_fds() if hasattr(process, 'num_fds') else 0
|
||||
}
|
||||
except Exception as e:
|
||||
stats["process"] = {"error": str(e)}
|
||||
stats["errors"].append(f"进程: {str(e)}")
|
||||
|
||||
# 计算运行时间
|
||||
try:
|
||||
uptime = time.time() - self.config.get('start_time', time.time())
|
||||
stats["uptime"] = round(uptime, 2)
|
||||
except Exception:
|
||||
stats["uptime"] = None
|
||||
|
||||
return stats
|
||||
|
||||
def get_monitor_history(self, hours: int = 24) -> dict:
|
||||
"""获取历史监控数据"""
|
||||
cutoff_time = time.time() - (hours * 3600)
|
||||
|
||||
with self.history_lock:
|
||||
filtered_history = [
|
||||
point for point in self.history
|
||||
if point.get('timestamp_unix', 0) >= cutoff_time
|
||||
]
|
||||
|
||||
return {
|
||||
"hours": hours,
|
||||
"total_points": len(filtered_history),
|
||||
"data": filtered_history
|
||||
}
|
||||
|
||||
def get_stats_summary(self) -> dict:
|
||||
"""获取统计摘要"""
|
||||
history = self.get_monitor_history(24).get('data', [])
|
||||
|
||||
if not history:
|
||||
return {
|
||||
"status": "no_data",
|
||||
"message": "暂无监控数据"
|
||||
}
|
||||
|
||||
# 计算各项指标的平均值和最大值
|
||||
cpu_values = [p.get('cpu', {}).get('percent', 0) for p in history]
|
||||
memory_values = [p.get('memory', {}).get('percent', 0) for p in history]
|
||||
disk_values = [p.get('disk', {}).get('percent', 0) for p in history]
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"period_hours": 24,
|
||||
"cpu": {
|
||||
"avg": round(sum(cpu_values) / len(cpu_values), 1) if cpu_values else 0,
|
||||
"max": max(cpu_values) if cpu_values else 0,
|
||||
"min": min(cpu_values) if cpu_values else 0
|
||||
},
|
||||
"memory": {
|
||||
"avg": round(sum(memory_values) / len(memory_values), 1) if memory_values else 0,
|
||||
"max": max(memory_values) if memory_values else 0,
|
||||
"min": min(memory_values) if memory_values else 0
|
||||
},
|
||||
"disk": {
|
||||
"avg": round(sum(disk_values) / len(disk_values), 1) if disk_values else 0,
|
||||
"max": max(disk_values) if disk_values else 0,
|
||||
"min": min(disk_values) if disk_values else 0
|
||||
},
|
||||
"total_downloads": history[-1].get('downloads', {}).get('total', 0) if history else 0,
|
||||
"total_connections": sum(p.get('network', {}).get('connections_count', 0) for p in history)
|
||||
}
|
||||
|
||||
def start_monitoring(self, callback=None):
|
||||
"""启动监控循环（后台线程）"""
|
||||
def monitor_loop():
|
||||
while self.enabled:
|
||||
try:
|
||||
stats = self.get_realtime_stats()
|
||||
|
||||
# 添加时间戳
|
||||
stats['timestamp_unix'] = time.time()
|
||||
|
||||
# 保存历史
|
||||
self._add_history_point(stats)
|
||||
|
||||
# SSE广播
|
||||
if callback:
|
||||
callback(stats)
|
||||
self._broadcast_sse('stats', stats)
|
||||
|
||||
except Exception as e:
|
||||
print(f"监控采集错误: {e}")
|
||||
|
||||
time.sleep(self.collection_interval)
|
||||
|
||||
thread = threading.Thread(target=monitor_loop, daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
def stop_monitoring(self):
|
||||
"""停止监控"""
|
||||
self.enabled = False
|
||||
|
||||
def register_sse_client(self, client_id: str, topics: List[str] = None) -> None:
|
||||
"""注册SSE客户端"""
|
||||
with self.sse_lock:
|
||||
self.sse_clients[client_id] = {
|
||||
'topics': set(topics) if topics else {'*'},
|
||||
'last_ping': time.time()
|
||||
}
|
||||
|
||||
def unregister_sse_client(self, client_id: str) -> None:
|
||||
"""注销SSE客户端"""
|
||||
with self.sse_lock:
|
||||
self.sse_clients.pop(client_id, None)
|
||||
|
||||
def get_sse_clients_count(self) -> int:
|
||||
"""获取SSE客户端数量"""
|
||||
with self.sse_lock:
|
||||
return len(self.sse_clients)
|
||||
|
||||
def broadcast_event(self, event_type: str, data: dict) -> None:
|
||||
"""广播事件到所有SSE客户端"""
|
||||
self._broadcast_sse(event_type, data)
|
||||
|
||||
def _broadcast_sse(self, event_type: str, data: dict) -> None:
|
||||
"""SSE广播（内部方法）"""
|
||||
message = f"event: {event_type}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
|
||||
|
||||
with self.sse_lock:
|
||||
disconnected = []
|
||||
for client_id, client in self.sse_clients.items():
|
||||
try:
|
||||
if client['topics'] == {'*'} or event_type in client['topics']:
|
||||
# 实际发送需要在request handler中处理
|
||||
# 这里只记录消息
|
||||
pass
|
||||
except Exception:
|
||||
disconnected.append(client_id)
|
||||
|
||||
# 清理断开的客户端
|
||||
for client_id in disconnected:
|
||||
self.sse_clients.pop(client_id, None)
|
||||
|
||||
def _add_history_point(self, stats: dict) -> None:
|
||||
"""添加历史数据点"""
|
||||
# 精简数据以减少存储
|
||||
point = {
|
||||
"timestamp": stats.get('timestamp'),
|
||||
"timestamp_unix": stats.get('timestamp_unix'),
|
||||
"cpu_percent": stats.get('cpu', {}).get('percent', 0),
|
||||
"memory_percent": stats.get('memory', {}).get('percent', 0),
|
||||
"disk_percent": stats.get('disk', {}).get('percent', 0),
|
||||
"network_bytes_sent": stats.get('network', {}).get('bytes_sent', 0),
|
||||
"network_bytes_recv": stats.get('network', {}).get('bytes_recv', 0),
|
||||
"connections_count": stats.get('network', {}).get('connections_count', 0),
|
||||
"load_avg_1m": stats.get('cpu', {}).get('load_avg_1m', 0)
|
||||
}
|
||||
|
||||
with self.history_lock:
|
||||
self.history.append(point)
|
||||
|
||||
# 清理过期数据
|
||||
cutoff_time = time.time() - (self.history_hours * 3600)
|
||||
self.history = [
|
||||
p for p in self.history
|
||||
if p.get('timestamp_unix', 0) >= cutoff_time
|
||||
]
|
||||
|
||||
# 保存到文件
|
||||
self._save_history()
|
||||
|
||||
def _save_history(self) -> None:
|
||||
"""保存历史数据到文件"""
|
||||
try:
|
||||
with open(self.history_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(self.history, f, ensure_ascii=False, indent=2)
|
||||
except Exception as e:
|
||||
print(f"保存监控历史失败: {e}")
|
||||
|
||||
def _load_history(self) -> None:
|
||||
"""从文件加载历史数据"""
|
||||
try:
|
||||
if os.path.exists(self.history_file):
|
||||
with open(self.history_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, list):
|
||||
self.history = data[-10000:] # 限制历史数量
|
||||
except Exception as e:
|
||||
print(f"加载监控历史失败: {e}")
|
||||
self.history = []
|
||||
|
||||
def get_health_status(self) -> dict:
|
||||
"""获取健康状态"""
|
||||
stats = self.get_realtime_stats()
|
||||
|
||||
if 'error' in stats:
|
||||
return {
|
||||
"status": "unhealthy",
|
||||
"error": stats['error']
|
||||
}
|
||||
|
||||
# 检查各项指标
|
||||
warnings = []
|
||||
|
||||
cpu_percent = stats.get('cpu', {}).get('percent', 0)
|
||||
if cpu_percent > 90:
|
||||
warnings.append(f"CPU使用率过高: {cpu_percent}%")
|
||||
elif cpu_percent > 70:
|
||||
warnings.append(f"CPU使用率较高: {cpu_percent}%")
|
||||
|
||||
memory_percent = stats.get('memory', {}).get('percent', 0)
|
||||
if memory_percent > 90:
|
||||
warnings.append(f"内存使用率过高: {memory_percent}%")
|
||||
elif memory_percent > 80:
|
||||
warnings.append(f"内存使用率较高: {memory_percent}%")
|
||||
|
||||
disk_percent = stats.get('disk', {}).get('percent', 0)
|
||||
if disk_percent > 90:
|
||||
warnings.append(f"磁盘使用率过高: {disk_percent}%")
|
||||
elif disk_percent > 80:
|
||||
warnings.append(f"磁盘使用率较高: {disk_percent}%")
|
||||
|
||||
if warnings:
|
||||
return {
|
||||
"status": "degraded",
|
||||
"warnings": warnings,
|
||||
"timestamp": stats.get('timestamp')
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "healthy",
|
||||
"timestamp": stats.get('timestamp')
|
||||
}
|
||||
@@ -0,0 +1,722 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
内存优化模块
|
||||
专为低端设备优化 (2CPU/1G 内存等)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import gc
|
||||
import time
|
||||
import threading
|
||||
from typing import Dict, List, Optional, Callable
|
||||
from dataclasses import dataclass
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
|
||||
# 跨平台兼容处理：resource 模块仅在 Unix/Linux 上可用
|
||||
try:
|
||||
import resource
|
||||
_HAS_RESOURCE = True
|
||||
except ImportError:
|
||||
_HAS_RESOURCE = False
|
||||
resource = None
|
||||
|
||||
# 内存限制配置
|
||||
DEFAULT_MEMORY_LIMIT = 512 * 1024 * 1024 # 512MB
|
||||
LOW_MEMORY_LIMIT = 256 * 1024 * 1024 # 256MB
|
||||
VERY_LOW_MEMORY_LIMIT = 128 * 1024 * 1024 # 128MB
|
||||
|
||||
|
||||
class MemoryManager:
|
||||
"""内存管理器 - 支持定时垃圾回收"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.enabled = self.config.get('enabled', True)
|
||||
self.memory_limit = self.config.get('memory_limit', DEFAULT_MEMORY_LIMIT)
|
||||
self.soft_limit = self.memory_limit * 0.8 # 80% 时触发警告
|
||||
self.check_interval = self.config.get('check_interval', 10) # 秒
|
||||
|
||||
# 定时垃圾回收配置
|
||||
self.gc_interval = self.config.get('gc_interval', 300) # 默认 5 分钟
|
||||
self.enable_scheduled_gc = self.config.get('enable_scheduled_gc', True)
|
||||
|
||||
# 缓存清理回调
|
||||
self.cache_cleaners: List[Callable] = []
|
||||
|
||||
# 回调函数
|
||||
self.on_memory_warning: Optional[Callable] = None
|
||||
self.on_memory_critical: Optional[Callable] = None
|
||||
|
||||
self._running = False
|
||||
self._monitor_thread = None
|
||||
self._gc_thread = None
|
||||
|
||||
def start(self):
|
||||
"""启动内存监控"""
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
self._running = True
|
||||
|
||||
# 启动内存监控线程
|
||||
self._monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
|
||||
self._monitor_thread.start()
|
||||
|
||||
# 启动定时垃圾回收线程
|
||||
if self.enable_scheduled_gc:
|
||||
self._gc_thread = threading.Thread(target=self._gc_loop, daemon=True)
|
||||
self._gc_thread.start()
|
||||
print(f"[内存管理] 定时GC: {self.gc_interval}秒")
|
||||
|
||||
# 设置内存限制
|
||||
self.set_memory_limit(self.memory_limit)
|
||||
|
||||
print(f"[内存管理] 已启动, 限制: {self.memory_limit // 1024 // 1024}MB")
|
||||
|
||||
def stop(self):
|
||||
"""停止内存监控"""
|
||||
self._running = False
|
||||
if self._monitor_thread:
|
||||
self._monitor_thread.join(timeout=2)
|
||||
if self._gc_thread:
|
||||
self._gc_thread.join(timeout=2)
|
||||
|
||||
def register_cache_cleaner(self, cleaner: Callable):
|
||||
"""注册缓存清理回调函数"""
|
||||
self.cache_cleaners.append(cleaner)
|
||||
|
||||
def set_memory_limit(self, limit: int):
|
||||
"""设置内存限制 (Linux/Unix)"""
|
||||
if not _HAS_RESOURCE or resource is None:
|
||||
# Windows 平台不支持内存限制，跳过
|
||||
print(f"[内存管理] 跳过内存限制设置 (Windows平台不支持)")
|
||||
return
|
||||
try:
|
||||
# 软限制
|
||||
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
|
||||
print(f"[内存管理] 已设置内存限制: {limit // 1024 // 1024}MB")
|
||||
except Exception as e:
|
||||
print(f"[内存管理] 设置内存限制失败: {e}")
|
||||
|
||||
def get_memory_usage(self) -> dict:
|
||||
"""获取内存使用情况"""
|
||||
try:
|
||||
# 进程内存
|
||||
import psutil
|
||||
process = psutil.Process(os.getpid())
|
||||
mem_info = process.memory_info()
|
||||
|
||||
# 系统内存
|
||||
sys_mem = psutil.virtual_memory()
|
||||
|
||||
return {
|
||||
'process_rss': mem_info.rss,
|
||||
'process_vms': mem_info.vms,
|
||||
'process_percent': process.memory_percent(),
|
||||
'system_total': sys_mem.total,
|
||||
'system_available': sys_mem.available,
|
||||
'system_percent': sys_mem.percent,
|
||||
'process_rss_mb': mem_info.rss / 1024 / 1024,
|
||||
'system_available_mb': sys_mem.available / 1024 / 1024
|
||||
}
|
||||
except Exception as e:
|
||||
# 备用方法：使用 resource (Unix) 或返回估计值 (Windows)
|
||||
if _HAS_RESOURCE and resource is not None:
|
||||
try:
|
||||
return {
|
||||
'process_rss': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024,
|
||||
'process_rss_mb': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
# Windows 无 psutil 的极端情况
|
||||
return {
|
||||
'process_rss': 0,
|
||||
'process_rss_mb': 0,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
def _monitor_loop(self):
|
||||
"""监控循环"""
|
||||
while self._running:
|
||||
try:
|
||||
usage = self.get_memory_usage()
|
||||
|
||||
# 检查是否达到软限制
|
||||
if usage['process_rss'] >= self.soft_limit:
|
||||
if self.on_memory_warning:
|
||||
self.on_memory_warning(usage)
|
||||
self._aggressive_cleanup()
|
||||
|
||||
# 检查是否达到硬限制
|
||||
if usage['process_rss'] >= self.memory_limit:
|
||||
if self.on_memory_critical:
|
||||
self.on_memory_critical(usage)
|
||||
self._emergency_cleanup()
|
||||
|
||||
time.sleep(self.check_interval)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _gc_loop(self):
|
||||
"""定时垃圾回收循环"""
|
||||
while self._running:
|
||||
try:
|
||||
# 执行垃圾回收
|
||||
self._scheduled_gc()
|
||||
|
||||
# 清理注册过的缓存
|
||||
for cleaner in self.cache_cleaners:
|
||||
try:
|
||||
cleaner()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
time.sleep(self.gc_interval)
|
||||
|
||||
def _scheduled_gc(self):
|
||||
"""定时 GC 执行"""
|
||||
# 标准 GC
|
||||
collected = gc.collect()
|
||||
# 清理 Python 内部缓存
|
||||
if hasattr(sys, 'exc_clear'):
|
||||
sys.exc_clear()
|
||||
|
||||
def _aggressive_cleanup(self):
|
||||
"""激进清理"""
|
||||
# 强制垃圾回收
|
||||
gc.collect()
|
||||
|
||||
# 清理 Python 缓存
|
||||
if hasattr(gc, 'set_threshold'):
|
||||
gc.set_threshold(500, 10, 5)
|
||||
|
||||
# 尝试释放内存
|
||||
try:
|
||||
import psutil
|
||||
process = psutil.Process(os.getpid())
|
||||
process.memory_info().rss # 刷新
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _emergency_cleanup(self):
|
||||
"""紧急清理"""
|
||||
print("[内存管理] ⚠️ 达到内存限制，尝试紧急清理...")
|
||||
|
||||
# 完全垃圾回收
|
||||
gc.collect()
|
||||
gc.collect()
|
||||
gc.collect()
|
||||
|
||||
# 清理所有缓存
|
||||
if hasattr(gc, 'garbage'):
|
||||
del gc.garbage[:]
|
||||
|
||||
# 触发警告
|
||||
print("[内存管理] ⚠️ 内存仍过高，考虑重启服务")
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""获取状态"""
|
||||
usage = self.get_memory_usage()
|
||||
return {
|
||||
'enabled': self.enabled,
|
||||
'memory_limit_mb': self.memory_limit // 1024 // 1024,
|
||||
'current_mb': usage['process_rss_mb'],
|
||||
'available_mb': usage.get('system_available_mb', 0),
|
||||
'percent': (usage['process_rss'] / self.memory_limit * 100) if self.memory_limit else 0
|
||||
}
|
||||
|
||||
|
||||
# ==================== 低内存配置 ====================
|
||||
|
||||
class LowMemoryConfig:
|
||||
"""低端设备配置"""
|
||||
|
||||
# 可禁用的功能列表
|
||||
FEATURES = {
|
||||
'ws': 'enable_ws', # WebSocket
|
||||
'sse': 'enable_sse', # Server-Sent Events
|
||||
'hash_calc': 'calculate_hash', # 文件哈希计算
|
||||
'stats': 'enable_stats', # 统计功能
|
||||
'monitor': 'enable_monitor', # 系统监控
|
||||
'sync': 'enable_sync', # 同步功能
|
||||
'mirrors': 'enable_mirrors', # 加速源
|
||||
}
|
||||
|
||||
# 预设配置
|
||||
PRESETS = {
|
||||
'ultra_low': {
|
||||
'description': '极低端设备 (<256MB RAM)',
|
||||
'workers': 1,
|
||||
'max_cache_size': 32 * 1024 * 1024, # 32MB
|
||||
'chunk_size': 16 * 1024, # 16KB
|
||||
'buffer_size': 32 * 1024, # 32KB
|
||||
'db_pool_size': 1,
|
||||
'max_connections': 3,
|
||||
'monitor_interval': 60,
|
||||
'gc_interval': 180, # 3分钟
|
||||
'timeout': 15,
|
||||
'disable_optional_features': ['ws', 'sse', 'hash_calc', 'stats', 'monitor']
|
||||
},
|
||||
'low': {
|
||||
'description': '低端设备 (256-512MB RAM)',
|
||||
'workers': 1,
|
||||
'max_cache_size': 64 * 1024 * 1024, # 64MB
|
||||
'chunk_size': 32 * 1024, # 32KB
|
||||
'buffer_size': 64 * 1024, # 64KB
|
||||
'db_pool_size': 1,
|
||||
'max_connections': 5,
|
||||
'monitor_interval': 30,
|
||||
'gc_interval': 300, # 5分钟
|
||||
'timeout': 20,
|
||||
'disable_optional_features': ['ws', 'sse', 'hash_calc', 'stats']
|
||||
},
|
||||
'medium': {
|
||||
'description': '中等设备 (512MB-1GB RAM)',
|
||||
'workers': 2,
|
||||
'max_cache_size': 128 * 1024 * 1024, # 128MB
|
||||
'chunk_size': 64 * 1024, # 64KB
|
||||
'buffer_size': 128 * 1024, # 128KB
|
||||
'db_pool_size': 2,
|
||||
'max_connections': 15,
|
||||
'monitor_interval': 15,
|
||||
'gc_interval': 600, # 10分钟
|
||||
'timeout': 30,
|
||||
'disable_optional_features': []
|
||||
},
|
||||
'high': {
|
||||
'description': '高端设备 (1GB+ RAM)',
|
||||
'workers': 4,
|
||||
'max_cache_size': 256 * 1024 * 1024, # 256MB
|
||||
'chunk_size': 128 * 1024, # 128KB
|
||||
'buffer_size': 256 * 1024, # 256KB
|
||||
'db_pool_size': 4,
|
||||
'max_connections': 50,
|
||||
'monitor_interval': 5,
|
||||
'gc_interval': 900, # 15分钟
|
||||
'timeout': 30,
|
||||
'disable_optional_features': []
|
||||
},
|
||||
'performance': {
|
||||
'description': '高性能设备 (4GB+ RAM)',
|
||||
'workers': 8,
|
||||
'max_cache_size': 1024 * 1024 * 1024, # 1GB
|
||||
'chunk_size': 256 * 1024, # 256KB
|
||||
'buffer_size': 512 * 1024, # 512KB
|
||||
'db_pool_size': 8,
|
||||
'max_connections': 200,
|
||||
'monitor_interval': 3,
|
||||
'gc_interval': 1800, # 30分钟
|
||||
'timeout': 60,
|
||||
'disable_optional_features': []
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self, preset: str = 'auto', custom_config: dict = None):
|
||||
"""
|
||||
初始化配置
|
||||
|
||||
Args:
|
||||
preset: 预设 ('ultra_low', 'low', 'medium', 'high', 'auto')
|
||||
custom_config: 自定义配置
|
||||
"""
|
||||
if preset == 'auto':
|
||||
preset = self._detect_preset()
|
||||
|
||||
self.preset = preset
|
||||
self.config = self.PRESETS.get(preset, self.PRESETS['low']).copy()
|
||||
|
||||
if custom_config:
|
||||
self.config.update(custom_config)
|
||||
|
||||
def _detect_preset(self) -> str:
|
||||
"""自动检测设备配置
|
||||
|
||||
检测逻辑：
|
||||
1. 获取总内存和可用内存
|
||||
2. 计算可用内存占比
|
||||
3. 结合总内存和可用内存占比综合判断
|
||||
"""
|
||||
try:
|
||||
import psutil
|
||||
mem = psutil.virtual_memory()
|
||||
total_ram = mem.total
|
||||
available_ram = mem.available
|
||||
percent_used = mem.percent # 已使用百分比
|
||||
|
||||
# 转换为MB
|
||||
total_mb = total_ram / (1024 * 1024)
|
||||
|
||||
# 根据总内存和可用内存占比综合判断
|
||||
if total_mb < 200:
|
||||
# 低于 200MB 总内存
|
||||
return 'ultra_low'
|
||||
elif total_mb < 400:
|
||||
# 200MB - 400MB
|
||||
return 'ultra_low'
|
||||
elif total_mb < 700:
|
||||
# 400MB - 700MB
|
||||
if percent_used > 80:
|
||||
return 'ultra_low'
|
||||
return 'low'
|
||||
elif total_mb < 1200:
|
||||
# 700MB - 1.2GB
|
||||
if percent_used > 70:
|
||||
return 'ultra_low'
|
||||
elif percent_used > 50:
|
||||
return 'low'
|
||||
return 'medium'
|
||||
elif total_mb < 2500:
|
||||
# 1.2GB - 2.5GB
|
||||
if percent_used > 70:
|
||||
return 'low'
|
||||
elif percent_used > 40:
|
||||
return 'medium'
|
||||
return 'high'
|
||||
elif total_mb < 5000:
|
||||
# 2.5GB - 5GB
|
||||
if percent_used > 60:
|
||||
return 'medium'
|
||||
return 'high'
|
||||
else:
|
||||
# 5GB+
|
||||
return 'performance'
|
||||
except ImportError:
|
||||
# 如果没有 psutil，使用保守的 low 配置
|
||||
return 'low'
|
||||
except Exception:
|
||||
return 'low'
|
||||
|
||||
def get_device_info(self) -> dict:
|
||||
"""获取设备详细信息用于显示"""
|
||||
try:
|
||||
import psutil
|
||||
mem = psutil.virtual_memory()
|
||||
cpu_count = psutil.cpu_count(logical=True) or 1
|
||||
|
||||
return {
|
||||
'total_ram_mb': mem.total / (1024 * 1024),
|
||||
'available_ram_mb': mem.available / (1024 * 1024),
|
||||
'percent_used': mem.percent,
|
||||
'cpu_count': cpu_count,
|
||||
'preset': self.preset
|
||||
}
|
||||
except Exception:
|
||||
return {
|
||||
'total_ram_mb': 0,
|
||||
'available_ram_mb': 0,
|
||||
'percent_used': 0,
|
||||
'cpu_count': 1,
|
||||
'preset': self.preset
|
||||
}
|
||||
|
||||
def apply_to_config(self, base_config: dict) -> dict:
|
||||
"""应用配置到基础配置"""
|
||||
config = base_config.copy()
|
||||
|
||||
# 应用通用设置
|
||||
config['workers'] = self.config.get('workers', 1)
|
||||
config['max_cache_size'] = self.config.get('max_cache_size', 64 * 1024 * 1024)
|
||||
config['chunk_size'] = self.config.get('chunk_size', 64 * 1024)
|
||||
config['buffer_size'] = self.config.get('buffer_size', 128 * 1024)
|
||||
config['timeout'] = self.config.get('timeout', 30)
|
||||
|
||||
# 数据库池
|
||||
if 'database' not in config:
|
||||
config['database'] = {}
|
||||
config['database']['db_pool_size'] = self.config.get('db_pool_size', 2)
|
||||
|
||||
# 定时 GC 配置
|
||||
config['gc_interval'] = self.config.get('gc_interval', 300)
|
||||
|
||||
# 禁用可选功能
|
||||
for feature in self.config.get('disable_optional_features', []):
|
||||
feature_key = self.FEATURES.get(feature, feature)
|
||||
if feature_key.startswith('enable_') or feature_key == 'calculate_hash':
|
||||
config[feature_key] = False
|
||||
|
||||
return config
|
||||
|
||||
def get_config(self) -> dict:
|
||||
"""获取配置"""
|
||||
return self.config.copy()
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""获取状态"""
|
||||
return {
|
||||
'preset': self.preset,
|
||||
'description': self.config['description'],
|
||||
'settings': self.config
|
||||
}
|
||||
|
||||
|
||||
# ==================== 流式处理优化 ====================
|
||||
|
||||
class StreamingOptimizer:
|
||||
"""流式处理优化器"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.chunk_size = self.config.get('chunk_size', 128 * 1024)
|
||||
self.buffer_size = self.config.get('buffer_size', 256 * 1024)
|
||||
|
||||
# 内存池
|
||||
self._chunk_pool = None
|
||||
self._use_memory_pool = self.config.get('use_memory_pool', True)
|
||||
|
||||
def get_optimized_chunk_size(self, file_size: int) -> int:
|
||||
"""根据文件大小获取优化的块大小"""
|
||||
if file_size < 1024 * 1024: # < 1MB
|
||||
return 16 * 1024 # 16KB
|
||||
elif file_size < 10 * 1024 * 1024: # < 10MB
|
||||
return 32 * 1024 # 32KB
|
||||
elif file_size < 100 * 1024 * 1024: # < 100MB
|
||||
return 64 * 1024 # 64KB
|
||||
else:
|
||||
return self.chunk_size
|
||||
|
||||
@contextmanager
|
||||
def memory_efficient_file_read(self, file_path: str, chunk_size: int = None):
|
||||
"""
|
||||
内存高效的文件读取
|
||||
|
||||
Usage:
|
||||
with optimizer.memory_efficient_file_read('/path/to/file') as f:
|
||||
for chunk in f:
|
||||
process(chunk)
|
||||
"""
|
||||
chunk_size = chunk_size or self.chunk_size
|
||||
file_size = os.path.getsize(file_path)
|
||||
chunk_size = self.get_optimized_chunk_size(file_size)
|
||||
|
||||
file = open(file_path, 'rb')
|
||||
try:
|
||||
yield file
|
||||
finally:
|
||||
file.close()
|
||||
|
||||
@contextmanager
|
||||
def memory_efficient_file_write(self, file_path: str, chunk_size: int = None):
|
||||
"""内存高效的文件写入"""
|
||||
chunk_size = chunk_size or self.chunk_size
|
||||
file = open(file_path, 'wb')
|
||||
try:
|
||||
yield file
|
||||
finally:
|
||||
file.close()
|
||||
|
||||
|
||||
# ==================== 架构检测 ====================
|
||||
|
||||
class ArchitectureDetector:
|
||||
"""架构检测器"""
|
||||
|
||||
@staticmethod
|
||||
def get_architecture() -> dict:
|
||||
"""
|
||||
获取架构信息
|
||||
|
||||
Returns:
|
||||
dict: 包含架构信息的字典
|
||||
"""
|
||||
info = {
|
||||
'platform': sys.platform,
|
||||
'architecture': 'unknown',
|
||||
'machine': 'unknown',
|
||||
'processor': 'unknown',
|
||||
'python_version': sys.version,
|
||||
'byte_order': sys.byteorder
|
||||
}
|
||||
|
||||
# 机器类型
|
||||
info['machine'] = os.uname().machine if hasattr(os, 'uname') else 'unknown'
|
||||
|
||||
# 检测 32位/64位
|
||||
if info['machine'] in ['x86_64', 'amd64', 'aarch64', 'arm64']:
|
||||
info['architecture'] = '64bit'
|
||||
elif info['machine'] in ['i386', 'i686', 'armv7l', 'armv6l']:
|
||||
info['architecture'] = '32bit'
|
||||
elif info['machine'] in ['armv8l', 'aarch32']:
|
||||
info['architecture'] = '32bit' # 32位ARM
|
||||
|
||||
# ARM 变体
|
||||
if info['machine'].startswith('arm'):
|
||||
if info['machine'] in ['armv7l', 'armv7hl']:
|
||||
info['arm_variant'] = 'armv7'
|
||||
elif info['machine'].startswith('armv8'):
|
||||
info['arm_variant'] = 'armv8'
|
||||
elif info['machine'].startswith('armv6'):
|
||||
info['arm_variant'] = 'armv6'
|
||||
else:
|
||||
info['arm_variant'] = 'unknown'
|
||||
|
||||
# x86 变体
|
||||
if info['machine'] in ['i386', 'i686']:
|
||||
info['x86_variant'] = 'i386'
|
||||
elif info['machine'] == 'x86_64':
|
||||
info['x86_variant'] = 'x86_64'
|
||||
|
||||
return info
|
||||
|
||||
@staticmethod
|
||||
def is_low_end_device() -> bool:
|
||||
"""检测是否为低端设备"""
|
||||
try:
|
||||
import psutil
|
||||
mem = psutil.virtual_memory()
|
||||
return mem.total < 1024 * 1024 * 1024 # < 1GB
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_recommended_config() -> dict:
|
||||
"""获取推荐的配置"""
|
||||
arch = ArchitectureDetector.get_architecture()
|
||||
|
||||
if arch['architecture'] == '32bit':
|
||||
return {
|
||||
'max_workers': 2,
|
||||
'max_cache_size': 100 * 1024 * 1024, # 100MB
|
||||
'enable_threading': True,
|
||||
'use_processes': False, # 32位进程数有限制
|
||||
'max_file_handles': 256
|
||||
}
|
||||
elif ArchitectureDetector.is_low_end_device():
|
||||
return {
|
||||
'max_workers': 1,
|
||||
'max_cache_size': 50 * 1024 * 1024, # 50MB
|
||||
'enable_threading': True,
|
||||
'use_processes': False,
|
||||
'max_file_handles': 128
|
||||
}
|
||||
else:
|
||||
return {
|
||||
'max_workers': 4,
|
||||
'max_cache_size': 500 * 1024 * 1024, # 500MB
|
||||
'enable_threading': True,
|
||||
'use_processes': True,
|
||||
'max_file_handles': 1024
|
||||
}
|
||||
|
||||
|
||||
# ==================== 兼容性检查 ====================
|
||||
|
||||
def check_compatibility() -> dict:
|
||||
"""
|
||||
检查系统兼容性
|
||||
|
||||
Returns:
|
||||
dict: 兼容性检查结果
|
||||
"""
|
||||
results = {
|
||||
'compatible': True,
|
||||
'warnings': [],
|
||||
'errors': [],
|
||||
'info': {}
|
||||
}
|
||||
|
||||
# Python 版本检查
|
||||
if sys.version_info < (3, 8):
|
||||
results['compatible'] = False
|
||||
results['errors'].append(f"Python 3.8+ 所需, 当前版本: {sys.version}")
|
||||
|
||||
# 架构信息
|
||||
arch_info = ArchitectureDetector.get_architecture()
|
||||
results['info']['architecture'] = arch_info
|
||||
|
||||
# 检查必需模块
|
||||
required_modules = [
|
||||
('os', '标准库'),
|
||||
('json', '标准库'),
|
||||
('http', '标准库'),
|
||||
('sqlite3', '标准库')
|
||||
]
|
||||
|
||||
optional_modules = [
|
||||
('psutil', '系统监控 (推荐)'),
|
||||
('sqlalchemy', '数据库 (推荐)'),
|
||||
(' cryptography', '加密 (推荐)'),
|
||||
('aiohttp', '异步HTTP (可选)'),
|
||||
('paramiko', 'SSH/SFTP (可选)')
|
||||
]
|
||||
|
||||
for module, desc in required_modules:
|
||||
try:
|
||||
__import__(module)
|
||||
except ImportError:
|
||||
results['compatible'] = False
|
||||
results['errors'].append(f"必需模块缺失: {module} ({desc})")
|
||||
|
||||
for module, desc in optional_modules:
|
||||
try:
|
||||
__import__(module)
|
||||
except ImportError:
|
||||
results['warnings'].append(f"可选模块缺失: {module} ({desc})")
|
||||
|
||||
# 内存检查
|
||||
try:
|
||||
import psutil
|
||||
mem = psutil.virtual_memory()
|
||||
if mem.total < 256 * 1024 * 1024:
|
||||
results['warnings'].append("内存低于 256MB，可能无法正常运行")
|
||||
except Exception:
|
||||
results['warnings'].append("无法检测内存，可能内存不足")
|
||||
|
||||
# 磁盘空间检查
|
||||
try:
|
||||
disk = psutil.disk_usage('.')
|
||||
if disk.free < 100 * 1024 * 1024: # 100MB
|
||||
results['warnings'].append("可用磁盘空间不足 100MB")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ==================== 便捷函数 ====================
|
||||
|
||||
def get_memory_manager(config: dict = None) -> MemoryManager:
|
||||
"""获取内存管理器"""
|
||||
return MemoryManager(config)
|
||||
|
||||
|
||||
def get_low_memory_config(preset: str = 'auto') -> LowMemoryConfig:
|
||||
"""获取低端设备配置"""
|
||||
return LowMemoryConfig(preset)
|
||||
|
||||
|
||||
def detect_and_configure() -> dict:
|
||||
"""
|
||||
自动检测并配置
|
||||
|
||||
Returns:
|
||||
dict: 配置信息
|
||||
"""
|
||||
# 检查兼容性
|
||||
compat = check_compatibility()
|
||||
if not compat['compatible']:
|
||||
print("⚠️ 系统兼容性警告:")
|
||||
for error in compat['errors']:
|
||||
print(f" - {error}")
|
||||
|
||||
# 获取推荐配置
|
||||
arch_config = ArchitectureDetector.get_recommended_config()
|
||||
|
||||
# 获取低端设备配置
|
||||
low_mem_config = get_low_memory_config('auto')
|
||||
arch_info = ArchitectureDetector.get_architecture()
|
||||
|
||||
return {
|
||||
'compatible': compat['compatible'],
|
||||
'architecture': arch_info,
|
||||
'recommended': arch_config,
|
||||
'low_memory': low_mem_config.get_status(),
|
||||
'warnings': compat['warnings']
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
Prometheus 指标导出模块
|
||||
提供 /metrics 端点暴露监控数据
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
|
||||
# 添加项目根目录到路径
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
class PrometheusMetrics:
|
||||
"""Prometheus 指标收集器"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self._metrics: Dict[str, dict] = {}
|
||||
|
||||
# 初始化指标
|
||||
self._init_metrics()
|
||||
|
||||
def _init_metrics(self):
|
||||
"""初始化指标定义"""
|
||||
self._metrics = {
|
||||
# 服务器指标
|
||||
'hyc_server_uptime_seconds': {
|
||||
'type': 'gauge',
|
||||
'description': 'Server uptime in seconds',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_server_start_time': {
|
||||
'type': 'gauge',
|
||||
'description': 'Server start timestamp',
|
||||
'value': time.time()
|
||||
},
|
||||
|
||||
# 文件指标
|
||||
'hyc_files_total': {
|
||||
'type': 'gauge',
|
||||
'description': 'Total number of files in the system',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_files_size_bytes': {
|
||||
'type': 'gauge',
|
||||
'description': 'Total size of all files in bytes',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_downloads_total': {
|
||||
'type': 'counter',
|
||||
'description': 'Total number of downloads',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_downloads_today': {
|
||||
'type': 'counter',
|
||||
'description': 'Number of downloads today',
|
||||
'value': 0
|
||||
},
|
||||
|
||||
# 缓存指标
|
||||
'hyc_cache_size_bytes': {
|
||||
'type': 'gauge',
|
||||
'description': 'Current cache size in bytes',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_cache_entries': {
|
||||
'type': 'gauge',
|
||||
'description': 'Number of cache entries',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_cache_hits_total': {
|
||||
'type': 'counter',
|
||||
'description': 'Total number of cache hits',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_cache_misses_total': {
|
||||
'type': 'counter',
|
||||
'description': 'Total number of cache misses',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_cache_hit_ratio': {
|
||||
'type': 'gauge',
|
||||
'description': 'Cache hit ratio (0-1)',
|
||||
'value': 0
|
||||
},
|
||||
|
||||
# 同步指标
|
||||
'hyc_sync_sources_total': {
|
||||
'type': 'gauge',
|
||||
'description': 'Total number of sync sources',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_sync_running': {
|
||||
'type': 'gauge',
|
||||
'description': 'Number of currently running sync operations',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_sync_files_total': {
|
||||
'type': 'counter',
|
||||
'description': 'Total number of synced files',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_sync_last_timestamp': {
|
||||
'type': 'gauge',
|
||||
'description': 'Timestamp of last successful sync',
|
||||
'value': 0
|
||||
},
|
||||
|
||||
# 数据库指标
|
||||
'hyc_db_files_total': {
|
||||
'type': 'gauge',
|
||||
'description': 'Number of files in database',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_db_sync_records': {
|
||||
'type': 'gauge',
|
||||
'description': 'Number of sync records in database',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_db_cache_records': {
|
||||
'type': 'gauge',
|
||||
'description': 'Number of cache records in database',
|
||||
'value': 0
|
||||
},
|
||||
|
||||
# 系统资源指标 (从监控模块获取)
|
||||
'hyc_cpu_percent': {
|
||||
'type': 'gauge',
|
||||
'description': 'CPU usage percentage',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_memory_percent': {
|
||||
'type': 'gauge',
|
||||
'description': 'Memory usage percentage',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_disk_percent': {
|
||||
'type': 'gauge',
|
||||
'description': 'Disk usage percentage',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_disk_free_bytes': {
|
||||
'type': 'gauge',
|
||||
'description': 'Free disk space in bytes',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_disk_total_bytes': {
|
||||
'type': 'gauge',
|
||||
'description': 'Total disk space in bytes',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_network_rx_bytes': {
|
||||
'type': 'counter',
|
||||
'description': 'Network receive bytes',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_network_tx_bytes': {
|
||||
'type': 'counter',
|
||||
'description': 'Network transmit bytes',
|
||||
'value': 0
|
||||
},
|
||||
'hyc_active_connections': {
|
||||
'type': 'gauge',
|
||||
'description': 'Number of active connections',
|
||||
'value': 0
|
||||
},
|
||||
|
||||
# 镜像源指标
|
||||
'hyc_mirror_enabled': {
|
||||
'type': 'gauge',
|
||||
'description': 'Whether a mirror is enabled (1=enabled, 0=disabled)',
|
||||
'labels': ['mirror_type'],
|
||||
'value': {}
|
||||
},
|
||||
'hyc_mirror_last_sync': {
|
||||
'type': 'gauge',
|
||||
'description': 'Timestamp of last mirror sync',
|
||||
'labels': ['mirror_type'],
|
||||
'value': {}
|
||||
},
|
||||
}
|
||||
|
||||
def set_uptime(self, seconds: float):
|
||||
"""设置运行时间"""
|
||||
self._metrics['hyc_server_uptime_seconds']['value'] = seconds
|
||||
|
||||
def set_files(self, count: int, size_bytes: int):
|
||||
"""设置文件统计"""
|
||||
self._metrics['hyc_files_total']['value'] = count
|
||||
self._metrics['hyc_files_size_bytes']['value'] = size_bytes
|
||||
|
||||
def set_downloads(self, total: int, today: int):
|
||||
"""设置下载统计"""
|
||||
self._metrics['hyc_downloads_total']['value'] = total
|
||||
self._metrics['hyc_downloads_today']['value'] = today
|
||||
|
||||
def set_cache(self, size_bytes: int, entries: int, hits: int, misses: int):
|
||||
"""设置缓存统计"""
|
||||
self._metrics['hyc_cache_size_bytes']['value'] = size_bytes
|
||||
self._metrics['hyc_cache_entries']['value'] = entries
|
||||
self._metrics['hyc_cache_hits_total']['value'] = hits
|
||||
self._metrics['hyc_cache_misses_total']['value'] = misses
|
||||
|
||||
# 计算命中率
|
||||
total = hits + misses
|
||||
if total > 0:
|
||||
self._metrics['hyc_cache_hit_ratio']['value'] = hits / total
|
||||
else:
|
||||
self._metrics['hyc_cache_hit_ratio']['value'] = 0
|
||||
|
||||
def set_sync(self, running: int, files_total: int, last_timestamp: float):
|
||||
"""设置同步统计"""
|
||||
self._metrics['hyc_sync_running']['value'] = running
|
||||
self._metrics['hyc_sync_files_total']['value'] = files_total
|
||||
self._metrics['hyc_sync_last_timestamp']['value'] = last_timestamp
|
||||
|
||||
def set_db_stats(self, files: int, sync_records: int, cache_records: int):
|
||||
"""设置数据库统计"""
|
||||
self._metrics['hyc_db_files_total']['value'] = files
|
||||
self._metrics['hyc_db_sync_records']['value'] = sync_records
|
||||
self._metrics['hyc_db_cache_records']['value'] = cache_records
|
||||
|
||||
def set_system(self, cpu: float, memory: float, disk: float,
|
||||
disk_free: int, disk_total: int,
|
||||
network_rx: int, network_tx: int):
|
||||
"""设置系统资源统计"""
|
||||
self._metrics['hyc_cpu_percent']['value'] = cpu
|
||||
self._metrics['hyc_memory_percent']['value'] = memory
|
||||
self._metrics['hyc_disk_percent']['value'] = disk
|
||||
self._metrics['hyc_disk_free_bytes']['value'] = disk_free
|
||||
self._metrics['hyc_disk_total_bytes']['value'] = disk_total
|
||||
self._metrics['hyc_network_rx_bytes']['value'] = network_rx
|
||||
self._metrics['hyc_network_tx_bytes']['value'] = network_tx
|
||||
|
||||
def set_connections(self, count: int):
|
||||
"""设置连接数"""
|
||||
self._metrics['hyc_active_connections']['value'] = count
|
||||
|
||||
def set_mirror_status(self, mirror_type: str, enabled: bool, last_sync: float):
|
||||
"""设置镜像源状态"""
|
||||
key = 'hyc_mirror_enabled'
|
||||
if 'labels' not in self._metrics[key]:
|
||||
self._metrics[key]['labels'] = ['mirror_type']
|
||||
if 'value' not in self._metrics[key]:
|
||||
self._metrics[key]['value'] = {}
|
||||
self._metrics[key]['value'][mirror_type] = 1 if enabled else 0
|
||||
|
||||
key = 'hyc_mirror_last_sync'
|
||||
if 'labels' not in self._metrics[key]:
|
||||
self._metrics[key]['labels'] = ['mirror_type']
|
||||
if 'value' not in self._metrics[key]:
|
||||
self._metrics[key]['value'] = {}
|
||||
self._metrics[key]['value'][mirror_type] = last_sync
|
||||
|
||||
def increment_downloads(self, count: int = 1):
|
||||
"""增加下载计数"""
|
||||
self._metrics['hyc_downloads_total']['value'] += count
|
||||
|
||||
def increment_cache_hits(self, count: int = 1):
|
||||
"""增加缓存命中计数"""
|
||||
self._metrics['hyc_cache_hits_total']['value'] += count
|
||||
|
||||
def increment_cache_misses(self, count: int = 1):
|
||||
"""增加缓存未命中计数"""
|
||||
self._metrics['hyc_cache_misses_total']['value'] += count
|
||||
|
||||
def increment_sync_files(self, count: int = 1):
|
||||
"""增加同步文件计数"""
|
||||
self._metrics['hyc_sync_files_total']['value'] += count
|
||||
|
||||
def generate_metrics(self) -> str:
|
||||
"""生成 Prometheus 格式的指标输出"""
|
||||
output = []
|
||||
output.append("# Prometheus metrics for HYC Mirror Server")
|
||||
output.append(f"# Generated at: {datetime.now().isoformat()}")
|
||||
output.append("")
|
||||
|
||||
for name, metric in self._metrics.items():
|
||||
desc = metric.get('description', '')
|
||||
mtype = metric.get('type', 'gauge')
|
||||
|
||||
output.append(f"# TYPE {name} {mtype}")
|
||||
output.append(f"# HELP {name} {desc}")
|
||||
|
||||
value = metric.get('value')
|
||||
|
||||
# 处理带标签的指标
|
||||
if 'labels' in metric and isinstance(value, dict):
|
||||
labels = metric['labels']
|
||||
for label_values, v in value.items():
|
||||
if isinstance(label_values, str):
|
||||
label_str = f'{",".join([f"{labels[0]}={label_values}"])}'
|
||||
else:
|
||||
label_str = ','.join([f"{l}={v}" for l, v in zip(labels, label_values)])
|
||||
output.append(f"{name}{{{label_str}}} {v}")
|
||||
# 处理普通指标
|
||||
elif isinstance(value, dict):
|
||||
# 旧格式，可能是直接存储
|
||||
for k, v in value.items():
|
||||
output.append(f"{name}{{type=\"{k}\"}} {v}")
|
||||
else:
|
||||
output.append(f"{name} {value}")
|
||||
|
||||
output.append("")
|
||||
|
||||
return '\n'.join(output)
|
||||
|
||||
|
||||
# ==================== 指标中间件 ====================
|
||||
|
||||
class MetricsMiddleware:
|
||||
"""HTTP 请求指标中间件"""
|
||||
|
||||
def __init__(self, metrics: PrometheusMetrics = None):
|
||||
self.metrics = metrics or PrometheusMetrics()
|
||||
self._request_count = 0
|
||||
self._request_duration_total = 0
|
||||
self._errors_4xx = 0
|
||||
self._errors_5xx = 0
|
||||
|
||||
def record_request(self, duration: float, status_code: int):
|
||||
"""记录请求"""
|
||||
self._request_count += 1
|
||||
self._request_duration_total += duration
|
||||
|
||||
if 400 <= status_code < 500:
|
||||
self._errors_4xx += 1
|
||||
elif status_code >= 500:
|
||||
self._errors_5xx += 1
|
||||
|
||||
def get_request_count(self) -> int:
|
||||
return self._request_count
|
||||
|
||||
def get_request_duration_total(self) -> float:
|
||||
return self._request_duration_total
|
||||
|
||||
def get_errors_4xx(self) -> int:
|
||||
return self._errors_4xx
|
||||
|
||||
def get_errors_5xx(self) -> int:
|
||||
return self._errors_5xx
|
||||
|
||||
def update_metrics(self):
|
||||
"""更新 Prometheus 指标"""
|
||||
# 添加请求相关指标
|
||||
if 'hyc_http_requests_total' not in self.metrics._metrics:
|
||||
self.metrics._metrics['hyc_http_requests_total'] = {
|
||||
'type': 'counter',
|
||||
'description': 'Total HTTP requests',
|
||||
'value': self._request_count
|
||||
}
|
||||
else:
|
||||
self.metrics._metrics['hyc_http_requests_total']['value'] = self._request_count
|
||||
|
||||
if 'hyc_http_request_duration_seconds_total' not in self.metrics._metrics:
|
||||
self.metrics._metrics['hyc_http_request_duration_seconds_total'] = {
|
||||
'type': 'counter',
|
||||
'description': 'Total HTTP request duration in seconds',
|
||||
'value': self._request_duration_total
|
||||
}
|
||||
else:
|
||||
self.metrics._metrics['hyc_http_request_duration_seconds_total']['value'] = self._request_duration_total
|
||||
|
||||
if 'hyc_http_requests_4xx_total' not in self.metrics._metrics:
|
||||
self.metrics._metrics['hyc_http_requests_4xx_total'] = {
|
||||
'type': 'counter',
|
||||
'description': 'Total HTTP 4xx errors',
|
||||
'value': self._errors_4xx
|
||||
}
|
||||
else:
|
||||
self.metrics._metrics['hyc_http_requests_4xx_total']['value'] = self._errors_4xx
|
||||
|
||||
if 'hyc_http_requests_5xx_total' not in self.metrics._metrics:
|
||||
self.metrics._metrics['hyc_http_requests_5xx_total'] = {
|
||||
'type': 'counter',
|
||||
'description': 'Total HTTP 5xx errors',
|
||||
'value': self._errors_5xx
|
||||
}
|
||||
else:
|
||||
self.metrics._metrics['hyc_http_requests_5xx_total']['value'] = self._errors_5xx
|
||||
@@ -0,0 +1,374 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
定时任务调度器
|
||||
支持 cron 表达式和简单间隔的定时任务
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Callable
|
||||
from enum import Enum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
"""任务状态"""
|
||||
IDLE = "idle"
|
||||
RUNNING = "running"
|
||||
ERROR = "error"
|
||||
DISABLED = "disabled"
|
||||
|
||||
|
||||
class ScheduledTask:
|
||||
"""定时任务"""
|
||||
|
||||
def __init__(self, name: str, task_type: str, config: dict,
|
||||
callback: Callable, logger=None):
|
||||
"""
|
||||
初始化定时任务
|
||||
|
||||
Args:
|
||||
name: 任务名称
|
||||
task_type: 任务类型 ('cron' 或 'interval')
|
||||
config: 任务配置
|
||||
callback: 回调函数
|
||||
logger: 日志器
|
||||
"""
|
||||
self.name = name
|
||||
self.task_type = task_type # 'cron' 或 'interval'
|
||||
self.config = config or {}
|
||||
self.callback = callback
|
||||
self.logger = logger or logging.getLogger(__name__)
|
||||
|
||||
# 状态
|
||||
self.status = TaskStatus.IDLE
|
||||
self.last_run: Optional[datetime] = None
|
||||
self.next_run: Optional[datetime] = None
|
||||
self.last_error: Optional[str] = None
|
||||
self.run_count = 0
|
||||
|
||||
# 配置解析
|
||||
self._parse_config()
|
||||
|
||||
def _parse_config(self):
|
||||
"""解析任务配置"""
|
||||
if self.task_type == 'cron':
|
||||
# Cron 表达式: "minute hour day month weekday"
|
||||
# 例如: "0 3 * * *" 每天凌晨3点
|
||||
cron = self.config.get('cron', '0 0 * * *')
|
||||
parts = cron.split()
|
||||
if len(parts) == 5:
|
||||
self.cron_parts = {
|
||||
'minute': self._parse_cron_part(parts[0], 0, 59),
|
||||
'hour': self._parse_cron_part(parts[1], 0, 23),
|
||||
'day': self._parse_cron_part(parts[2], 1, 31),
|
||||
'month': self._parse_cron_part(parts[3], 1, 12),
|
||||
'weekday': self._parse_cron_part(parts[4], 0, 6)
|
||||
}
|
||||
else:
|
||||
self.logger.warning(f"无效的 cron 表达式: {cron}")
|
||||
self.cron_parts = None
|
||||
|
||||
elif self.task_type == 'interval':
|
||||
# 间隔: seconds, minutes, hours
|
||||
interval = self.config.get('interval', {})
|
||||
self.interval_seconds = (
|
||||
interval.get('seconds', 0) +
|
||||
interval.get('minutes', 0) * 60 +
|
||||
interval.get('hours', 0) * 3600 +
|
||||
interval.get('days', 0) * 86400
|
||||
)
|
||||
if self.interval_seconds <= 0:
|
||||
self.interval_seconds = 3600 # 默认1小时
|
||||
|
||||
# 是否启用
|
||||
self.enabled = self.config.get('enabled', True)
|
||||
|
||||
def _parse_cron_part(self, part: str, min_val: int, max_val: int) -> List[int]:
|
||||
"""解析 cron 表达式的一部分"""
|
||||
result = []
|
||||
if part == '*':
|
||||
return list(range(min_val, max_val + 1))
|
||||
|
||||
# 处理列表: "1,2,3"
|
||||
if ',' in part:
|
||||
return self._parse_cron_part(part.replace(',', ' '), min_val, max_val)
|
||||
|
||||
# 处理范围: "1-5"
|
||||
if '-' in part:
|
||||
start, end = part.split('-')
|
||||
return list(range(int(start), int(end) + 1))
|
||||
|
||||
# 处理步进: "*/5"
|
||||
if '/' in part:
|
||||
base, step = part.split('/')
|
||||
base_list = self._parse_cron_part(base or '*', min_val, max_val)
|
||||
step = int(step)
|
||||
return base_list[::step]
|
||||
|
||||
# 单个值
|
||||
try:
|
||||
val = int(part)
|
||||
if min_val <= val <= max_val:
|
||||
return [val]
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return []
|
||||
|
||||
def should_run_now(self) -> bool:
|
||||
"""检查是否应该在当前时刻运行"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
if self.task_type == 'cron' and self.cron_parts:
|
||||
return self._matches_cron(now)
|
||||
elif self.task_type == 'interval':
|
||||
if self.last_run is None:
|
||||
return True
|
||||
elapsed = (now - self.last_run).total_seconds()
|
||||
return elapsed >= self.interval_seconds
|
||||
|
||||
return False
|
||||
|
||||
def _matches_cron(self, dt: datetime) -> bool:
|
||||
"""检查时间是否匹配 cron 表达式"""
|
||||
if not self.cron_parts:
|
||||
return False
|
||||
|
||||
return (
|
||||
dt.minute in self.cron_parts['minute'] and
|
||||
dt.hour in self.cron_parts['hour'] and
|
||||
dt.day in self.cron_parts['day'] and
|
||||
dt.month in self.cron_parts['month'] and
|
||||
dt.weekday() in self.cron_parts['weekday']
|
||||
)
|
||||
|
||||
def get_next_run_time(self) -> Optional[datetime]:
|
||||
"""计算下次运行时间"""
|
||||
if not self.enabled:
|
||||
return None
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
if self.task_type == 'cron' and self.cron_parts:
|
||||
# 找到下一个匹配的时间点
|
||||
for i in range(365 * 24 * 60): # 最多查找1年
|
||||
candidate = now + timedelta(minutes=i)
|
||||
if self._matches_cron(candidate):
|
||||
return candidate
|
||||
elif self.task_type == 'interval':
|
||||
if self.last_run:
|
||||
return self.last_run + timedelta(seconds=self.interval_seconds)
|
||||
return now
|
||||
|
||||
return None
|
||||
|
||||
def run(self) -> bool:
|
||||
"""执行任务"""
|
||||
if self.status == TaskStatus.RUNNING:
|
||||
self.logger.warning(f"任务 {self.name} 已在运行中")
|
||||
return False
|
||||
|
||||
self.status = TaskStatus.RUNNING
|
||||
self.last_run = datetime.now()
|
||||
self.last_error = None
|
||||
|
||||
try:
|
||||
self.logger.info(f"开始执行定时任务: {self.name}")
|
||||
result = self.callback(self.name, self.config)
|
||||
self.run_count += 1
|
||||
self.logger.info(f"定时任务 {self.name} 执行完成")
|
||||
return True
|
||||
except Exception as e:
|
||||
self.last_error = str(e)
|
||||
self.status = TaskStatus.ERROR
|
||||
self.logger.error(f"定时任务 {self.name} 执行失败: {e}")
|
||||
return False
|
||||
finally:
|
||||
if self.status != TaskStatus.ERROR:
|
||||
self.status = TaskStatus.IDLE
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
'name': self.name,
|
||||
'type': self.task_type,
|
||||
'enabled': self.enabled,
|
||||
'status': self.status.value,
|
||||
'config': self.config,
|
||||
'last_run': self.last_run.isoformat() if self.last_run else None,
|
||||
'next_run': self.next_run.isoformat() if self.next_run else None,
|
||||
'run_count': self.run_count,
|
||||
'last_error': self.last_error
|
||||
}
|
||||
|
||||
|
||||
class Scheduler:
|
||||
"""定时任务调度器"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.tasks: Dict[str, ScheduledTask] = {}
|
||||
self._running = False
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# 默认检查间隔
|
||||
self.check_interval = self.config.get('check_interval', 10)
|
||||
|
||||
# 事件回调
|
||||
self.on_task_start: Optional[Callable] = None
|
||||
self.on_task_complete: Optional[Callable] = None
|
||||
self.on_task_error: Optional[Callable] = None
|
||||
|
||||
def add_task(self, name: str, task_type: str, config: dict,
|
||||
callback: Callable) -> bool:
|
||||
"""
|
||||
添加定时任务
|
||||
|
||||
Args:
|
||||
name: 任务名称
|
||||
task_type: 任务类型 ('cron' 或 'interval')
|
||||
config: 任务配置
|
||||
callback: 回调函数
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
with self._lock:
|
||||
if name in self.tasks:
|
||||
logger.warning(f"任务 {name} 已存在，将被替换")
|
||||
self.tasks[name] = ScheduledTask(name, task_type, config, callback, logger)
|
||||
return True
|
||||
|
||||
def remove_task(self, name: str) -> bool:
|
||||
"""移除任务"""
|
||||
with self._lock:
|
||||
if name in self.tasks:
|
||||
del self.tasks[name]
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_task(self, name: str) -> Optional[ScheduledTask]:
|
||||
"""获取任务"""
|
||||
return self.tasks.get(name)
|
||||
|
||||
def get_all_tasks(self) -> List[dict]:
|
||||
"""获取所有任务状态"""
|
||||
with self._lock:
|
||||
for task in self.tasks.values():
|
||||
task.next_run = task.get_next_run_time()
|
||||
return [task.to_dict() for task in self.tasks.values()]
|
||||
|
||||
def start(self):
|
||||
"""启动调度器"""
|
||||
if self._running:
|
||||
logger.warning("调度器已在运行中")
|
||||
return
|
||||
|
||||
self._running = True
|
||||
self._thread = threading.Thread(target=self._run_loop, daemon=True)
|
||||
self._thread.start()
|
||||
logger.info("定时任务调度器已启动")
|
||||
|
||||
def stop(self):
|
||||
"""停止调度器"""
|
||||
self._running = False
|
||||
if self._thread:
|
||||
self._thread.join(timeout=5)
|
||||
logger.info("定时任务调度器已停止")
|
||||
|
||||
def _run_loop(self):
|
||||
"""运行循环"""
|
||||
while self._running:
|
||||
try:
|
||||
now = datetime.now()
|
||||
|
||||
with self._lock:
|
||||
for name, task in self.tasks.items():
|
||||
if task.should_run_now():
|
||||
# 使用线程池执行任务
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
executor.submit(task.run)
|
||||
|
||||
time.sleep(self.check_interval)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"调度器循环错误: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
def run_task_now(self, name: str) -> bool:
|
||||
"""立即运行指定任务"""
|
||||
task = self.get_task(name)
|
||||
if task:
|
||||
return task.run()
|
||||
return False
|
||||
|
||||
def enable_task(self, name: str, enabled: bool = True) -> bool:
|
||||
"""启用/禁用任务"""
|
||||
task = self.get_task(name)
|
||||
if task:
|
||||
task.enabled = enabled
|
||||
return True
|
||||
return False
|
||||
|
||||
def update_task_config(self, name: str, config: dict) -> bool:
|
||||
"""更新任务配置"""
|
||||
task = self.get_task(name)
|
||||
if task:
|
||||
task.config.update(config)
|
||||
task._parse_config()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ==================== 同步任务工厂 ====================
|
||||
|
||||
# def create_sync_task_callback(sync_manager):
|
||||
# """创建同步任务的回调函数"""
|
||||
# def sync_task_callback(task_name: str, config: dict):
|
||||
# """同步任务回调"""
|
||||
# sync_manager.start_sync(task_name)
|
||||
# return True
|
||||
# return sync_task_callback
|
||||
|
||||
|
||||
# ==================== 默认任务配置 ====================
|
||||
# DEFAULT_SCHEDULED_TASKS = { ... }
|
||||
|
||||
DEFAULT_SCHEDULED_TASKS = {
|
||||
# 数据库清理 - 每天凌晨2点
|
||||
'cleanup_db': {
|
||||
'type': 'cron',
|
||||
'config': {
|
||||
'cron': '0 2 * * *',
|
||||
'enabled': True
|
||||
}
|
||||
},
|
||||
# 缓存清理 - 每6小时
|
||||
'cleanup_cache': {
|
||||
'type': 'interval',
|
||||
'config': {
|
||||
'interval': {'hours': 6},
|
||||
'enabled': True
|
||||
}
|
||||
},
|
||||
# 健康检查 - 每5分钟
|
||||
'health_check': {
|
||||
'type': 'interval',
|
||||
'config': {
|
||||
'interval': {'minutes': 5},
|
||||
'enabled': True
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
安全增强模块
|
||||
提供 IP 白名单/黑名单、请求速率限制、HTTPS 支持
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import hashlib
|
||||
import threading
|
||||
import logging
|
||||
import ssl
|
||||
import ipaddress
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from dataclasses import dataclass, field
|
||||
from functools import wraps
|
||||
from collections import defaultdict
|
||||
from queue import Queue
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ==================== IP 管理器 ====================
|
||||
|
||||
class IPManager:
|
||||
"""IP 管理器 - 白名单/黑名单"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.whitelist: List[str] = []
|
||||
self.blacklist: List[str] = []
|
||||
self._load_lists()
|
||||
|
||||
def _load_lists(self):
|
||||
"""加载 IP 列表"""
|
||||
# 加载白名单
|
||||
whitelist_file = self.config.get('whitelist_file', 'whitelist.txt')
|
||||
if os.path.exists(whitelist_file):
|
||||
with open(whitelist_file, 'r') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#'):
|
||||
self.whitelist.append(line)
|
||||
|
||||
# 加载黑名单
|
||||
blacklist_file = self.config.get('blacklist_file', 'blacklist.txt')
|
||||
if os.path.exists(blacklist_file):
|
||||
with open(blacklist_file, 'r') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#'):
|
||||
self.blacklist.append(line)
|
||||
|
||||
def is_allowed(self, ip: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
检查 IP 是否允许访问
|
||||
返回: (是否允许, 原因)
|
||||
"""
|
||||
# 检查白名单
|
||||
if self.whitelist:
|
||||
for pattern in self.whitelist:
|
||||
if self._match_ip(ip, pattern):
|
||||
return True, "白名单"
|
||||
|
||||
# 检查黑名单
|
||||
for pattern in self.blacklist:
|
||||
if self._match_ip(ip, pattern):
|
||||
return False, "黑名单"
|
||||
|
||||
return True, "允许"
|
||||
|
||||
def _match_ip(self, ip: str, pattern: str) -> bool:
|
||||
"""匹配 IP"""
|
||||
try:
|
||||
# 单个 IP
|
||||
if pattern == ip:
|
||||
return True
|
||||
|
||||
# CIDR 范围
|
||||
if '/' in pattern:
|
||||
network = ipaddress.ip_network(pattern, strict=False)
|
||||
return ipaddress.ip_address(ip) in network
|
||||
|
||||
# 通配符 (例如: 192.168.1.*)
|
||||
if pattern.endswith('*'):
|
||||
prefix = pattern.rstrip('*').rstrip('.')
|
||||
return ip.startswith(prefix)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
def add_to_whitelist(self, ip: str):
|
||||
"""添加到白名单"""
|
||||
if ip not in self.whitelist:
|
||||
self.whitelist.append(ip)
|
||||
self._save_list('whitelist')
|
||||
|
||||
def add_to_blacklist(self, ip: str):
|
||||
"""添加到黑名单"""
|
||||
if ip not in self.blacklist:
|
||||
self.blacklist.append(ip)
|
||||
self._save_list('blacklist')
|
||||
|
||||
def remove_from_whitelist(self, ip: str):
|
||||
"""从白名单移除"""
|
||||
if ip in self.whitelist:
|
||||
self.whitelist.remove(ip)
|
||||
self._save_list('whitelist')
|
||||
|
||||
def remove_from_blacklist(self, ip: str):
|
||||
"""从黑名单移除"""
|
||||
if ip in self.blacklist:
|
||||
self.blacklist.remove(ip)
|
||||
self._save_list('blacklist')
|
||||
|
||||
def _save_list(self, list_type: str):
|
||||
"""保存列表到文件"""
|
||||
filename = f'{list_type}.txt'
|
||||
data = '\n'.join(self.whitelist if list_type == 'whitelist' else self.blacklist)
|
||||
with open(filename, 'w') as f:
|
||||
f.write(data)
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""获取状态"""
|
||||
return {
|
||||
'whitelist_count': len(self.whitelist),
|
||||
'blacklist_count': len(self.blacklist),
|
||||
'whitelist': self.whitelist[:10],
|
||||
'blacklist': self.blacklist[:10]
|
||||
}
|
||||
|
||||
|
||||
# ==================== 速率限制器 ====================
|
||||
|
||||
class RateLimiter:
|
||||
"""请求速率限制器"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.requests: Dict[str, List[float]] = defaultdict(list)
|
||||
self.lock = threading.Lock()
|
||||
|
||||
# 配置
|
||||
self.requests_per_minute = self.config.get('requests_per_minute', 100)
|
||||
self.burst_limit = self.config.get('burst_limit', 20)
|
||||
self.window_seconds = 60
|
||||
|
||||
def is_allowed(self, identifier: str) -> Tuple[bool, int]:
|
||||
"""
|
||||
检查请求是否允许
|
||||
返回: (是否允许, 剩余配额)
|
||||
"""
|
||||
now = time.time()
|
||||
window_start = now - self.window_seconds
|
||||
|
||||
with self.lock:
|
||||
# 清理过期记录
|
||||
self.requests[identifier] = [
|
||||
t for t in self.requests[identifier]
|
||||
if t > window_start
|
||||
]
|
||||
|
||||
# 检查限制
|
||||
if len(self.requests[identifier]) >= self.requests_per_minute:
|
||||
return False, 0
|
||||
|
||||
# 记录请求
|
||||
self.requests[identifier].append(now)
|
||||
|
||||
remaining = self.requests_per_minute - len(self.requests[identifier])
|
||||
return True, remaining
|
||||
|
||||
def get_usage(self, identifier: str) -> dict:
|
||||
"""获取使用情况"""
|
||||
now = time.time()
|
||||
window_start = now - self.window_seconds
|
||||
|
||||
with self.lock:
|
||||
requests = [
|
||||
t for t in self.requests[identifier]
|
||||
if t > window_start
|
||||
]
|
||||
|
||||
return {
|
||||
'requests': len(requests),
|
||||
'limit': self.requests_per_minute,
|
||||
'remaining': self.requests_per_minute - len(requests),
|
||||
'reset_in': int(self.window_seconds - (now - min(requests) if requests else now))
|
||||
}
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""获取全局状态"""
|
||||
total_requests = sum(len(v) for v in self.requests.values())
|
||||
return {
|
||||
'active_ips': len(self.requests),
|
||||
'total_requests': total_requests,
|
||||
'limit_per_minute': self.requests_per_minute,
|
||||
'burst_limit': self.burst_limit
|
||||
}
|
||||
|
||||
def reset(self):
|
||||
"""重置所有记录"""
|
||||
with self.lock:
|
||||
self.requests.clear()
|
||||
|
||||
|
||||
# ==================== HTTPS 管理器 ====================
|
||||
|
||||
class HTTPSManager:
|
||||
"""HTTPS 证书管理器"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.cert_file = self.config.get('ssl_cert')
|
||||
self.key_file = self.config.get('ssl_key')
|
||||
self.context = None
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
"""是否启用 HTTPS"""
|
||||
return bool(self.cert_file and self.key_file)
|
||||
|
||||
def create_context(self) -> Optional[ssl.SSLContext]:
|
||||
"""创建 SSL 上下文"""
|
||||
if not self.is_enabled():
|
||||
return None
|
||||
|
||||
try:
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
|
||||
# 加载证书
|
||||
context.load_cert_chain(self.cert_file, self.key_file)
|
||||
|
||||
# 安全配置
|
||||
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||
context.set_ciphers('ECDHE+AESGCM:DHE+AESGCM:ECDHE+CHACHA20')
|
||||
|
||||
self.context = context
|
||||
return context
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"创建 SSL 上下文失败: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def generate_self_signed_cert(cert_path: str, key_path: str, common_name: str = 'localhost') -> bool:
|
||||
"""生成自签名证书（用于测试）"""
|
||||
try:
|
||||
from cryptography import x509
|
||||
from cryptography.x509.oid import NameOID
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
import datetime as dt
|
||||
|
||||
# 生成私钥
|
||||
private_key = rsa.generate_private_key(
|
||||
public_exponent=65537,
|
||||
key_size=2048,
|
||||
backend=default_backend()
|
||||
)
|
||||
|
||||
# 生成证书
|
||||
subject = issuer = x509.Name([
|
||||
x509.NameAttribute(NameOID.COUNTRY_NAME, "CN"),
|
||||
x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Shanghai"),
|
||||
x509.NameAttribute(NameOID.LOCALITY_NAME, "Shanghai"),
|
||||
x509.NameAttribute(NameOID.ORGANIZATION_NAME, "HYC Download Station"),
|
||||
x509.NameAttribute(NameOID.COMMON_NAME, common_name),
|
||||
])
|
||||
|
||||
cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(subject)
|
||||
.issuer_name(issuer)
|
||||
.public_key(private_key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(dt.datetime.utcnow())
|
||||
.not_valid_after(dt.datetime.utcnow() + dt.timedelta(days=365))
|
||||
.add_extension(
|
||||
x509.SubjectAlternativeName([
|
||||
x509.DNSName(common_name),
|
||||
x509.DNSName("localhost"),
|
||||
]),
|
||||
critical=False,
|
||||
)
|
||||
.sign(private_key, hashes.SHA256(), default_backend())
|
||||
)
|
||||
|
||||
# 保存证书
|
||||
with open(cert_path, 'wb') as f:
|
||||
f.write(cert.public_bytes(serialization.Encoding.PEM))
|
||||
|
||||
# 保存私钥
|
||||
with open(key_path, 'wb') as f:
|
||||
f.write(private_key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption()
|
||||
))
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成自签名证书失败: {e}")
|
||||
return False
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""获取状态"""
|
||||
return {
|
||||
'enabled': self.is_enabled(),
|
||||
'cert_file': self.cert_file,
|
||||
'key_file': self.key_file
|
||||
}
|
||||
|
||||
|
||||
# ==================== 安全中间件 ====================
|
||||
|
||||
class SecurityMiddleware:
|
||||
"""安全中间件"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
|
||||
# 初始化各组件
|
||||
self.ip_manager = IPManager(self.config.get('ip', {}))
|
||||
self.rate_limiter = RateLimiter(self.config.get('rate_limit', {}))
|
||||
self.https_manager = HTTPSManager(self.config.get('ssl', {}))
|
||||
|
||||
def check_request(self, handler) -> Tuple[bool, str]:
|
||||
"""
|
||||
检查请求是否安全
|
||||
返回: (是否通过, 原因)
|
||||
"""
|
||||
client_ip = self._get_client_ip(handler)
|
||||
|
||||
# IP 检查
|
||||
allowed, reason = self.ip_manager.is_allowed(client_ip)
|
||||
if not allowed:
|
||||
return False, f"IP被阻止: {reason}"
|
||||
|
||||
# 速率限制
|
||||
allowed, _ = self.rate_limiter.is_allowed(client_ip)
|
||||
if not allowed:
|
||||
return False, "请求过于频繁"
|
||||
|
||||
return True, "通过"
|
||||
|
||||
def _get_client_ip(self, handler) -> str:
|
||||
"""获取客户端 IP"""
|
||||
# 检查代理头
|
||||
forwarded = handler.headers.get('X-Forwarded-For')
|
||||
if forwarded:
|
||||
return forwarded.split(',')[0].strip()
|
||||
|
||||
real_ip = handler.headers.get('X-Real-IP')
|
||||
if real_ip:
|
||||
return real_ip
|
||||
|
||||
return handler.client_address[0] if hasattr(handler, 'client_address') else 'unknown'
|
||||
|
||||
def get_security_headers(self) -> dict:
|
||||
"""获取安全响应头"""
|
||||
return {
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
'X-Frame-Options': 'SAMEORIGIN',
|
||||
'X-XSS-Protection': '1; mode=block',
|
||||
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
||||
'Content-Security-Policy': "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline';"
|
||||
}
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""获取安全状态"""
|
||||
return {
|
||||
'ip': self.ip_manager.get_status(),
|
||||
'rate_limit': self.rate_limiter.get_status(),
|
||||
'ssl': self.https_manager.get_status()
|
||||
}
|
||||
|
||||
|
||||
# ==================== 审计日志 ====================
|
||||
|
||||
class AuditLogger:
|
||||
"""审计日志记录器"""
|
||||
|
||||
def __init__(self, config: dict = None):
|
||||
self.config = config or {}
|
||||
self.logs: Queue = Queue(maxsize=10000)
|
||||
self.log_file = self.config.get('audit_log', 'audit.log')
|
||||
self.enabled = self.config.get('enabled', True)
|
||||
|
||||
# 启动日志写入线程
|
||||
if self.enabled:
|
||||
self._start_writer()
|
||||
|
||||
def log(self, event_type: str, data: dict):
|
||||
"""记录事件"""
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
entry = {
|
||||
'timestamp': time.time(),
|
||||
'type': event_type,
|
||||
**data
|
||||
}
|
||||
|
||||
if self.logs.full():
|
||||
self.logs.get() # 移除最旧的
|
||||
|
||||
self.logs.put(entry)
|
||||
|
||||
def _start_writer(self):
|
||||
"""启动日志写入线程"""
|
||||
def writer():
|
||||
while True:
|
||||
try:
|
||||
entry = self.logs.get(timeout=1)
|
||||
self._write_entry(entry)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
thread = threading.Thread(target=writer, daemon=True)
|
||||
thread.start()
|
||||
|
||||
def _write_entry(self, entry: dict):
|
||||
"""写入日志条目"""
|
||||
try:
|
||||
with open(self.log_file, 'a', encoding='utf-8') as f:
|
||||
line = json.dumps(entry, ensure_ascii=False)
|
||||
f.write(line + '\n')
|
||||
except Exception as e:
|
||||
logger.error(f"写入审计日志失败: {e}")
|
||||
|
||||
def get_recent_logs(self, event_type: str = None, limit: int = 100) -> List[dict]:
|
||||
"""获取最近的日志"""
|
||||
result = []
|
||||
with self.logs.mutex:
|
||||
for entry in list(self.logs.queue):
|
||||
if event_type and entry.get('type') != event_type:
|
||||
continue
|
||||
result.append(entry)
|
||||
if len(result) >= limit:
|
||||
break
|
||||
return result[-limit:]
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""获取状态"""
|
||||
with self.logs.mutex:
|
||||
return {
|
||||
'enabled': self.enabled,
|
||||
'log_file': self.log_file,
|
||||
'pending_logs': self.logs.qsize(),
|
||||
'max_size': self.logs.maxsize
|
||||
}
|
||||
|
||||
|
||||
# ==================== 便捷函数 ====================
|
||||
|
||||
def get_security_middleware(config: dict = None) -> SecurityMiddleware:
|
||||
"""获取安全中间件"""
|
||||
return SecurityMiddleware(config)
|
||||
|
||||
|
||||
def get_ip_manager(config: dict = None) -> IPManager:
|
||||
"""获取 IP 管理器"""
|
||||
return IPManager(config)
|
||||
|
||||
|
||||
def get_rate_limiter(config: dict = None) -> RateLimiter:
|
||||
"""获取速率限制器"""
|
||||
return RateLimiter(config)
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""服务器核心模块 - 线程池版本"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import ssl
|
||||
import signal
|
||||
import mimetypes
|
||||
import threading
|
||||
import socketserver
|
||||
from datetime import datetime
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from .config import ConfigManager
|
||||
from .mirror_sync import MirrorSyncManager
|
||||
|
||||
|
||||
class ThreadPoolHTTPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
|
||||
"""使用线程池的 HTTP 服务器"""
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True # 使用守护线程
|
||||
|
||||
def __init__(self, server_address, RequestHandlerClass, max_workers=50):
|
||||
self.max_workers = max_workers
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=max_workers,
|
||||
thread_name_prefix="http_handler"
|
||||
)
|
||||
super().__init__(server_address, RequestHandlerClass)
|
||||
|
||||
def process_request(self, request, client_address):
|
||||
"""使用线程池处理请求"""
|
||||
self._executor.submit(self._handle_request, request, client_address)
|
||||
|
||||
def _handle_request(self, request, client_address):
|
||||
"""实际处理请求"""
|
||||
try:
|
||||
self.finish_request(request, client_address)
|
||||
except Exception:
|
||||
self.handle_error(request, client_address)
|
||||
finally:
|
||||
self.shutdown_request(request)
|
||||
|
||||
def server_close(self):
|
||||
"""关闭服务器和线程池"""
|
||||
# Python 3.8 兼容处理
|
||||
import sys
|
||||
try:
|
||||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||||
except TypeError:
|
||||
# Python 3.8 不支持 cancel_futures 参数
|
||||
self._executor.shutdown(wait=False)
|
||||
super().server_close()
|
||||
|
||||
|
||||
class MirrorServer:
|
||||
"""镜像服务器主类"""
|
||||
|
||||
def __init__(self, config):
|
||||
if isinstance(config, dict):
|
||||
self.config_manager = ConfigManager(config)
|
||||
else:
|
||||
self.config_manager = config
|
||||
|
||||
self.config = self.config_manager.config
|
||||
self.server = None
|
||||
self.sync_manager = None
|
||||
self.is_running = False
|
||||
|
||||
def _validate_config(self, config):
|
||||
"""验证和修复配置"""
|
||||
return self.config_manager._validate_config(config)
|
||||
|
||||
def start(self):
|
||||
"""启动服务器"""
|
||||
try:
|
||||
# 创建下载目录
|
||||
base_dir = self.config['base_dir']
|
||||
if not os.path.exists(base_dir):
|
||||
os.makedirs(base_dir)
|
||||
print(f"创建下载目录: {os.path.abspath(base_dir)}")
|
||||
|
||||
# 初始化MIME类型
|
||||
mimetypes.init()
|
||||
|
||||
# 记录启动时间
|
||||
self.config['start_time'] = __import__('time').time()
|
||||
|
||||
# 创建同步管理器（仅当启用时）
|
||||
if self.config.get('enable_sync', True):
|
||||
self.sync_manager = MirrorSyncManager(self.config)
|
||||
self.sync_manager.start()
|
||||
|
||||
# 创建系统监控器（仅当启用时）
|
||||
self.monitor = None
|
||||
if self.config.get('enable_monitor', True):
|
||||
try:
|
||||
from .monitor import SystemMonitor
|
||||
self.monitor = SystemMonitor(self.config)
|
||||
print(f" ✓ 系统监控已启用 (间隔: {self.config.get('monitor_interval', 5)}秒)")
|
||||
except ImportError as e:
|
||||
print(f" ✗ 系统监控导入失败: {e}")
|
||||
except Exception as e:
|
||||
print(f" ✗ 系统监控初始化失败: {e}")
|
||||
|
||||
# 延迟导入 handler（避免循环导入）
|
||||
from handlers.http_handler import MirrorServerHandler
|
||||
|
||||
# 获取线程数配置
|
||||
max_workers = min(self.config.get('max_workers', 10), 10) # 限制最大线程数
|
||||
|
||||
# 创建服务器
|
||||
server_address = (self.config['host'], self.config['port'])
|
||||
self.server = ThreadPoolHTTPServer(
|
||||
server_address,
|
||||
MirrorServerHandler,
|
||||
max_workers=max_workers
|
||||
)
|
||||
|
||||
# 传递配置到处理器
|
||||
MirrorServerHandler.config = self.config
|
||||
MirrorServerHandler.sync_manager = self.sync_manager
|
||||
MirrorServerHandler.monitor = self.monitor
|
||||
# 设置调试模式
|
||||
MirrorServerHandler._setup_debug(self.config)
|
||||
|
||||
# 设置超时
|
||||
self.server.timeout = self.config.get('timeout', 30)
|
||||
|
||||
# 启用HTTPS
|
||||
if self.config.get('ssl_cert') and self.config.get('ssl_key'):
|
||||
if not self._setup_ssl():
|
||||
return False
|
||||
|
||||
self.is_running = True
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
print(f"服务器启动失败: {e}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
def _setup_ssl(self):
|
||||
"""设置SSL"""
|
||||
try:
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(self.config['ssl_cert'], self.config['ssl_key'])
|
||||
self.server.socket = context.wrap_socket(self.server.socket, server_side=True)
|
||||
print(f"启用HTTPS，证书: {self.config['ssl_cert']}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"启用HTTPS失败: {e}")
|
||||
return False
|
||||
|
||||
def stop(self):
|
||||
"""停止服务器"""
|
||||
print("正在停止服务器...")
|
||||
|
||||
if self.sync_manager:
|
||||
try:
|
||||
self.sync_manager.stop()
|
||||
except Exception as e:
|
||||
print(f"停止同步管理器时出错: {e}")
|
||||
|
||||
if self.server:
|
||||
try:
|
||||
self.server.server_close()
|
||||
except Exception as e:
|
||||
print(f"关闭服务器连接时出错: {e}")
|
||||
|
||||
self.is_running = False
|
||||
print("服务器已停止")
|
||||
|
||||
def serve_forever(self):
|
||||
"""运行服务器"""
|
||||
if not self.server:
|
||||
print("服务器未启动")
|
||||
return
|
||||
|
||||
# 打印服务器信息（已在 main.py 中显示，此处仅保留最简信息）
|
||||
protocol = "https" if self.config.get('ssl_cert') else "http"
|
||||
sync_count = len(self.sync_manager.sync_sources) if self.sync_manager else 0
|
||||
print(f"\n▶ 服务器运行于: {protocol}://{self.config['host']}:{self.config['port']}")
|
||||
print(f"▶ 同步源数: {sync_count} | 最大线程: {self.server.max_workers}")
|
||||
print("▶ 按 Ctrl+C 停止服务器")
|
||||
|
||||
try:
|
||||
self.server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\n正在关闭服务器...")
|
||||
finally:
|
||||
self.stop()
|
||||
|
||||
|
||||
# 全局变量，用于信号处理器访问服务器实例（预留）
|
||||
# _server_instance = None
|
||||
|
||||
# def signal_handler(signum, frame):
|
||||
# """处理退出信号"""
|
||||
# print(f"\n收到信号 {signum}，正在关闭服务器...")
|
||||
# import os
|
||||
# os._exit(0)
|
||||
+1031
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,475 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
定时同步调度器
|
||||
负责本地数据和数据库之间的定时同步
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import hashlib
|
||||
import threading
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
# 添加项目根目录到路径
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from core.database import DatabaseManager, get_db
|
||||
from core.scheduler import Scheduler, ScheduledTask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SyncScheduler:
|
||||
"""同步调度器"""
|
||||
|
||||
def __init__(self, config: dict, db: DatabaseManager = None):
|
||||
self.config = config
|
||||
self.db = db
|
||||
self._running = False
|
||||
self._executor = ThreadPoolExecutor(max_workers=4)
|
||||
|
||||
# 同步配置
|
||||
self.sync_interval = config.get('database', {}).get('sync_interval', 60)
|
||||
self.auto_scan = config.get('auto_scan', True)
|
||||
self.scan_interval = config.get('scan_interval', 300) # 5分钟扫描一次
|
||||
|
||||
# 同步状态
|
||||
self.last_sync_time = 0
|
||||
self.last_scan_time = 0
|
||||
self.sync_in_progress = False
|
||||
self.scan_in_progress = False
|
||||
|
||||
# 回调函数
|
||||
self.on_file_added: Optional[Callable] = None
|
||||
self.on_file_deleted: Optional[Callable] = None
|
||||
self.on_file_updated: Optional[Callable] = None
|
||||
self.on_sync_complete: Optional[Callable] = None
|
||||
|
||||
# 待同步队列
|
||||
self._pending_add = [] # 待添加的文件
|
||||
self._pending_update = [] # 待更新的文件
|
||||
self._pending_delete = [] # 待删除的文件
|
||||
|
||||
# 定时任务调度器
|
||||
self.task_scheduler = None
|
||||
self.scheduled_syncs: Dict[str, dict] = {}
|
||||
|
||||
def _init_scheduled_syncs(self):
|
||||
"""初始化定时同步任务"""
|
||||
if not self.config.get('enable_sync', True):
|
||||
return
|
||||
|
||||
# 从配置加载定时同步设置
|
||||
sync_sources = self.config.get('sync_sources', {})
|
||||
scheduled_sources = {}
|
||||
|
||||
for name, source_config in sync_sources.items():
|
||||
schedule = source_config.get('schedule', {})
|
||||
if schedule.get('enabled', False):
|
||||
scheduled_sources[name] = {
|
||||
'type': schedule.get('type', 'interval'), # 'cron' 或 'interval'
|
||||
'config': {
|
||||
'cron': schedule.get('cron'),
|
||||
'interval': schedule.get('interval', {}),
|
||||
'enabled': True
|
||||
}
|
||||
}
|
||||
|
||||
if scheduled_sources:
|
||||
self.task_scheduler = Scheduler()
|
||||
for name, sched_config in scheduled_sources.items():
|
||||
self.task_scheduler.add_task(
|
||||
name=f"sync_{name}",
|
||||
task_type=sched_config['type'],
|
||||
config=sched_config['config'],
|
||||
callback=self._create_sync_callback(name)
|
||||
)
|
||||
self.scheduled_syncs = scheduled_sources
|
||||
|
||||
def _create_sync_callback(self, source_name: str):
|
||||
"""创建同步回调函数"""
|
||||
def sync_callback(task_name: str, config: dict):
|
||||
logger.info(f"定时同步任务触发: {source_name}")
|
||||
self.start_sync(source_name)
|
||||
return True
|
||||
return sync_callback
|
||||
|
||||
def start(self):
|
||||
"""启动同步调度器"""
|
||||
if self._running:
|
||||
logger.warning("SyncScheduler 已经运行中")
|
||||
return
|
||||
|
||||
self._running = True
|
||||
self._executor.submit(self._sync_loop)
|
||||
self._executor.submit(self._scan_loop)
|
||||
|
||||
# 初始化并启动定时同步
|
||||
self._init_scheduled_syncs()
|
||||
if self.task_scheduler:
|
||||
self.task_scheduler.start()
|
||||
|
||||
logger.info(f"同步调度器已启动，间隔: {self.sync_interval}秒")
|
||||
|
||||
def stop(self):
|
||||
"""停止同步调度器"""
|
||||
self._running = False
|
||||
|
||||
# 停止定时任务调度器
|
||||
if self.task_scheduler:
|
||||
self.task_scheduler.stop()
|
||||
self.task_scheduler = None
|
||||
|
||||
self._executor.shutdown(wait=True)
|
||||
logger.info("同步调度器已停止")
|
||||
|
||||
def _sync_loop(self):
|
||||
"""同步循环"""
|
||||
while self._running:
|
||||
try:
|
||||
if time.time() - self.last_sync_time >= self.sync_interval:
|
||||
self.perform_sync()
|
||||
time.sleep(1)
|
||||
except Exception as e:
|
||||
logger.error(f"同步循环错误: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
def _scan_loop(self):
|
||||
"""扫描循环 - 检测本地文件变化"""
|
||||
while self._running:
|
||||
try:
|
||||
if self.auto_scan and time.time() - self.last_scan_time >= self.scan_interval:
|
||||
self.scan_local_files()
|
||||
time.sleep(5)
|
||||
except Exception as e:
|
||||
logger.error(f"扫描循环错误: {e}")
|
||||
time.sleep(10)
|
||||
|
||||
def queue_add(self, file_info: dict):
|
||||
"""队列添加文件"""
|
||||
self._pending_add.append(file_info)
|
||||
|
||||
def queue_update(self, file_info: dict):
|
||||
"""队列更新文件"""
|
||||
self._pending_update.append(file_info)
|
||||
|
||||
def queue_delete(self, file_id: str):
|
||||
"""队列删除文件"""
|
||||
self._pending_delete.append(file_id)
|
||||
|
||||
def perform_sync(self):
|
||||
"""执行同步"""
|
||||
if self.sync_in_progress:
|
||||
logger.warning("同步已在进行中，跳过")
|
||||
return
|
||||
|
||||
self.sync_in_progress = True
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
logger.info("开始执行数据库同步...")
|
||||
|
||||
# 同步待添加的文件
|
||||
added = 0
|
||||
for file_info in self._pending_add[:]:
|
||||
try:
|
||||
self._sync_add_file(file_info)
|
||||
self._pending_add.remove(file_info)
|
||||
added += 1
|
||||
except Exception as e:
|
||||
logger.error(f"同步添加文件失败: {e}")
|
||||
|
||||
# 同步待更新的文件
|
||||
updated = 0
|
||||
for file_info in self._pending_update[:]:
|
||||
try:
|
||||
self._sync_update_file(file_info)
|
||||
self._pending_update.remove(file_info)
|
||||
updated += 1
|
||||
except Exception as e:
|
||||
logger.error(f"同步更新文件失败: {e}")
|
||||
|
||||
# 同步待删除的文件
|
||||
deleted = 0
|
||||
for file_id in self._pending_delete[:]:
|
||||
try:
|
||||
self._sync_delete_file(file_id)
|
||||
self._pending_delete.remove(file_id)
|
||||
deleted += 1
|
||||
except Exception as e:
|
||||
logger.error(f"同步删除文件失败: {e}")
|
||||
|
||||
# 同步统计
|
||||
self.db.reset_pending_count()
|
||||
|
||||
self.last_sync_time = time.time()
|
||||
duration = time.time() - start_time
|
||||
|
||||
logger.info(f"同步完成: 添加{added}, 更新{updated}, 删除{deleted}, 耗时{duration:.2f}秒")
|
||||
|
||||
# 回调
|
||||
if self.on_sync_complete:
|
||||
self.on_sync_complete({
|
||||
'added': added,
|
||||
'updated': updated,
|
||||
'deleted': deleted,
|
||||
'duration': duration
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"同步过程错误: {e}")
|
||||
finally:
|
||||
self.sync_in_progress = False
|
||||
|
||||
def scan_local_files(self):
|
||||
"""扫描本地文件"""
|
||||
if self.scan_in_progress:
|
||||
return
|
||||
|
||||
self.scan_in_progress = True
|
||||
|
||||
try:
|
||||
base_dir = self.config.get('base_dir', './downloads')
|
||||
if not os.path.exists(base_dir):
|
||||
self.last_scan_time = time.time()
|
||||
return
|
||||
|
||||
# 扫描文件
|
||||
scanned_files = []
|
||||
for root, dirs, files in os.walk(base_dir):
|
||||
for filename in files:
|
||||
full_path = os.path.join(root, filename)
|
||||
rel_path = os.path.relpath(full_path, base_dir).replace("\\", "/")
|
||||
|
||||
stat = os.stat(full_path)
|
||||
file_info = {
|
||||
'path': rel_path,
|
||||
'name': filename,
|
||||
'size': stat.st_size,
|
||||
'mtime': stat.st_mtime,
|
||||
'ctime': stat.st_ctime
|
||||
}
|
||||
scanned_files.append(file_info)
|
||||
|
||||
# 与数据库对比
|
||||
db_files = self.db.list_files(limit=100000)
|
||||
db_paths = {f.path for f in db_files if not f.is_dir}
|
||||
|
||||
# 检测新增
|
||||
local_paths = {f['path'] for f in scanned_files}
|
||||
new_paths = local_paths - db_paths
|
||||
|
||||
for path in new_paths:
|
||||
file_info = next((f for f in scanned_files if f['path'] == path), None)
|
||||
if file_info:
|
||||
file_id = hashlib.md5(path.encode()).hexdigest()
|
||||
self._sync_add_file({
|
||||
'file_id': file_id,
|
||||
'path': path,
|
||||
'name': file_info['name'],
|
||||
'size': file_info['size'],
|
||||
'updated_at': file_info['mtime']
|
||||
})
|
||||
|
||||
# 检测删除
|
||||
deleted_paths = db_paths - local_paths
|
||||
for path in deleted_paths:
|
||||
record = self.db.get_file_by_path(path)
|
||||
if record:
|
||||
self.db.delete_file(record.file_id)
|
||||
|
||||
self.last_scan_time = time.time()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"扫描本地文件错误: {e}")
|
||||
finally:
|
||||
self.scan_in_progress = False
|
||||
|
||||
def _sync_add_file(self, file_info: dict):
|
||||
"""同步添加文件"""
|
||||
existing = self.db.get_file_by_path(file_info['path'])
|
||||
if existing:
|
||||
# 已存在，更新
|
||||
self.db.update_file(
|
||||
existing.file_id,
|
||||
size=file_info.get('size', 0),
|
||||
updated_at=file_info.get('updated_at', time.time()),
|
||||
hash=file_info.get('hash'),
|
||||
sync_status='synced'
|
||||
)
|
||||
else:
|
||||
# 新增
|
||||
file_id = file_info.get('file_id') or hashlib.md5(
|
||||
file_info['path'].encode()
|
||||
).hexdigest()
|
||||
|
||||
self.db.add_file(
|
||||
file_id=file_id,
|
||||
path=file_info['path'],
|
||||
name=file_info['name'],
|
||||
size=file_info.get('size', 0),
|
||||
hash=file_info.get('hash'),
|
||||
is_dir=False,
|
||||
created_at=file_info.get('created_at'),
|
||||
updated_at=file_info.get('updated_at', time.time())
|
||||
)
|
||||
|
||||
if self.on_file_added:
|
||||
self.on_file_added(file_info)
|
||||
|
||||
def _sync_update_file(self, file_info: dict):
|
||||
"""同步更新文件"""
|
||||
file_id = file_info.get('file_id')
|
||||
if file_id:
|
||||
self.db.update_file(
|
||||
file_id,
|
||||
size=file_info.get('size'),
|
||||
updated_at=file_info.get('updated_at', time.time()),
|
||||
hash=file_info.get('hash'),
|
||||
sync_status='synced'
|
||||
)
|
||||
|
||||
if self.on_file_updated:
|
||||
self.on_file_updated(file_info)
|
||||
|
||||
def _sync_delete_file(self, file_id: str):
|
||||
"""同步删除文件"""
|
||||
self.db.delete_file(file_id)
|
||||
|
||||
if self.on_file_deleted:
|
||||
self.on_file_deleted({'file_id': file_id})
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""获取同步状态"""
|
||||
return {
|
||||
'running': self._running,
|
||||
'last_sync_time': self.last_sync_time,
|
||||
'last_scan_time': self.last_scan_time,
|
||||
'sync_in_progress': self.sync_in_progress,
|
||||
'scan_in_progress': self.scan_in_progress,
|
||||
'pending_add': len(self._pending_add),
|
||||
'pending_update': len(self._pending_update),
|
||||
'pending_delete': len(self._pending_delete),
|
||||
'pending_operations': self.db.get_pending_operations() if self.db else 0
|
||||
}
|
||||
|
||||
def force_sync(self):
|
||||
"""强制立即同步"""
|
||||
self.last_sync_time = 0
|
||||
self.perform_sync()
|
||||
|
||||
|
||||
# ==================== 文件操作包装器 ====================
|
||||
|
||||
class DatabaseBackedFileOperations:
|
||||
"""数据库支持的文件操作"""
|
||||
|
||||
def __init__(self, config: dict, db: DatabaseManager, scheduler: SyncScheduler = None):
|
||||
self.config = config
|
||||
self.db = db
|
||||
self.scheduler = scheduler
|
||||
self.base_dir = config.get('base_dir', './downloads')
|
||||
|
||||
def add_file_record(self, path: str, name: str, size: int = 0,
|
||||
hash: str = None, is_dir: bool = False) -> dict:
|
||||
"""添加文件记录到数据库"""
|
||||
import hashlib
|
||||
|
||||
file_id = hashlib.md5(path.encode()).hexdigest()
|
||||
|
||||
file_info = {
|
||||
'file_id': file_id,
|
||||
'path': path,
|
||||
'name': name,
|
||||
'size': size,
|
||||
'hash': hash,
|
||||
'is_dir': is_dir,
|
||||
'created_at': time.time(),
|
||||
'updated_at': time.time()
|
||||
}
|
||||
|
||||
if self.scheduler:
|
||||
self.scheduler.queue_add(file_info)
|
||||
else:
|
||||
self.db.add_file(
|
||||
file_id=file_id,
|
||||
path=path,
|
||||
name=name,
|
||||
size=size,
|
||||
hash=hash,
|
||||
is_dir=is_dir,
|
||||
created_at=time.time(),
|
||||
updated_at=time.time()
|
||||
)
|
||||
|
||||
return file_info
|
||||
|
||||
def update_file_record(self, file_id: str, **kwargs):
|
||||
"""更新文件记录"""
|
||||
if self.scheduler:
|
||||
self.scheduler.queue_update({'file_id': file_id, **kwargs})
|
||||
else:
|
||||
self.db.update_file(file_id, **kwargs)
|
||||
|
||||
def delete_file_record(self, file_id: str, hard: bool = False):
|
||||
"""删除文件记录"""
|
||||
if self.scheduler:
|
||||
self.scheduler.queue_delete(file_id)
|
||||
else:
|
||||
self.db.delete_file(file_id, hard=hard)
|
||||
|
||||
def record_download(self, file_path: str, file_size: int = 0,
|
||||
client_ip: str = None, duration: float = 0,
|
||||
success: bool = True, error_message: str = None):
|
||||
"""记录下载"""
|
||||
self.db.add_download_record(
|
||||
file_path=file_path,
|
||||
file_size=file_size,
|
||||
client_ip=client_ip,
|
||||
duration=duration,
|
||||
success=success,
|
||||
error_message=error_message
|
||||
)
|
||||
|
||||
# 更新下载计数
|
||||
record = self.db.get_file_by_path(file_path)
|
||||
if record:
|
||||
self.db.increment_download_count(record.file_id)
|
||||
|
||||
def record_cache_hit(self, cache_key: str, cache_type: str):
|
||||
"""记录缓存命中"""
|
||||
record = self.db.get_cache_record(cache_key)
|
||||
if record:
|
||||
self.db.increment_cache_hits(cache_key)
|
||||
else:
|
||||
self.db.add_cache_record(
|
||||
cache_key=cache_key,
|
||||
cache_type=cache_type,
|
||||
hits=1,
|
||||
last_hit=time.time()
|
||||
)
|
||||
|
||||
|
||||
# ==================== 便捷函数 ====================
|
||||
|
||||
def get_sync_scheduler(config: dict) -> SyncScheduler:
|
||||
"""获取同步调度器"""
|
||||
db = get_db(config)
|
||||
return SyncScheduler(config, db)
|
||||
|
||||
|
||||
def init_database_sync(config: dict, db=None) -> tuple:
|
||||
"""初始化数据库和同步"""
|
||||
if db is None:
|
||||
db = get_db(config)
|
||||
scheduler = SyncScheduler(config, db)
|
||||
file_ops = DatabaseBackedFileOperations(config, db, scheduler)
|
||||
|
||||
return db, scheduler, file_ops
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""工具函数模块"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import hashlib
|
||||
|
||||
|
||||
def format_file_size(size_bytes: int) -> str:
|
||||
"""格式化文件大小为可读格式"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
size_names = ["B", "KB", "MB", "GB", "TB", "PB"]
|
||||
i = 0
|
||||
while size_bytes >= 1024 and i < len(size_names) - 1:
|
||||
size_bytes /= 1024.0
|
||||
i += 1
|
||||
|
||||
return f"{size_bytes:.2f} {size_names[i]}"
|
||||
|
||||
|
||||
def get_file_hash(filepath: str, algorithm: str = 'sha256') -> str:
|
||||
"""计算文件哈希值"""
|
||||
if not filepath or not isinstance(filepath, str):
|
||||
return "Error: Invalid filepath"
|
||||
|
||||
if not os.path.exists(filepath):
|
||||
return "Error: File not found"
|
||||
|
||||
try:
|
||||
hash_func = hashlib.new(algorithm)
|
||||
with open(filepath, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hash_func.update(chunk)
|
||||
return hash_func.hexdigest()
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}"
|
||||
|
||||
|
||||
def parse_size(size_str: str) -> int:
|
||||
"""解析文件大小字符串为字节数"""
|
||||
size_str = size_str.upper().strip()
|
||||
match = re.match(r'^(\d+(?:\.\d+)?)\s*([BKMGT]?)B?$', size_str)
|
||||
if not match:
|
||||
raise ValueError(f"无效的文件大小格式: {size_str}")
|
||||
|
||||
number = float(match.group(1))
|
||||
unit = match.group(2) or 'B'
|
||||
|
||||
units = {
|
||||
'B': 1,
|
||||
'K': 1024,
|
||||
'M': 1024 ** 2,
|
||||
'G': 1024 ** 3,
|
||||
'T': 1024 ** 4
|
||||
}
|
||||
|
||||
if unit not in units:
|
||||
raise ValueError(f"无效的单位: {unit}")
|
||||
|
||||
return int(number * units[unit])
|
||||
|
||||
|
||||
def sanitize_filename(filename: str) -> str:
|
||||
"""清理文件名，防止路径遍历和安全问题"""
|
||||
filename = os.path.basename(filename) # 移除路径分隔符
|
||||
|
||||
# 移除危险字符
|
||||
filename = re.sub(r'[<>:"|?*\\\x00-\x1f]', '_', filename)
|
||||
|
||||
# 限制长度
|
||||
if len(filename) > 255:
|
||||
name, ext = os.path.splitext(filename)
|
||||
filename = name[:255 - len(ext)] + ext
|
||||
|
||||
# 防止空文件名
|
||||
if not filename or filename in ('.', '..'):
|
||||
filename = 'unnamed_file'
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
def is_safe_path(base_dir: str, path: str) -> bool:
|
||||
"""检查路径是否安全（防止目录遍历）"""
|
||||
try:
|
||||
abs_path = os.path.abspath(path)
|
||||
abs_base = os.path.abspath(base_dir)
|
||||
common_path = os.path.commonpath([abs_path, abs_base])
|
||||
return common_path == abs_base
|
||||
except ValueError:
|
||||
return False
|
||||
@@ -0,0 +1,25 @@
|
||||
# ============================================
|
||||
# HYC下载站 v2.2 - 环境变量配置示例
|
||||
# ============================================
|
||||
|
||||
# 数据路径
|
||||
DATA_PATH=./data
|
||||
DOWNLOADS_PATH=./downloads
|
||||
DB_PATH=./postgres
|
||||
|
||||
# 数据库配置
|
||||
DB_TYPE=postgresql
|
||||
DB_HOST=hyc-db
|
||||
DB_PORT=5432
|
||||
DB_NAME=hyc
|
||||
DB_USER=postgres
|
||||
DB_PASS=your_password_here
|
||||
|
||||
# 外部数据库连接 (可选，覆盖上述配置)
|
||||
# DB_CONN_STR=postgresql://user:pass@host:5432/database
|
||||
|
||||
# Redis (可选)
|
||||
REDIS_PATH=./redis
|
||||
|
||||
# 时区
|
||||
TZ=Asia/Shanghai
|
||||
@@ -0,0 +1,73 @@
|
||||
# ============================================
|
||||
# HYC下载站 v2.2 - Docker 构建配置
|
||||
# ============================================
|
||||
|
||||
# 构建阶段
|
||||
FROM python:3.11-slim AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 安装构建依赖
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
musl-dev \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 安装 Python 依赖
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --prefix=/install -r requirements.txt
|
||||
|
||||
# ============================================
|
||||
# 运行阶段
|
||||
FROM python:3.11-slim
|
||||
|
||||
LABEL maintainer="HYC Download Station"
|
||||
LABEL description="HYC下载站 v2.2 - 镜像文件服务器 + 下载加速源"
|
||||
|
||||
# 环境变量
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV HYC_HOME=/app
|
||||
|
||||
# 创建工作目录
|
||||
WORKDIR ${HYC_HOME}
|
||||
|
||||
# 复制 Python 和依赖
|
||||
COPY --from=builder /install /usr/local
|
||||
|
||||
# 复制应用代码
|
||||
COPY main.py .
|
||||
COPY settings.json .
|
||||
COPY requirements.txt .
|
||||
COPY core/ ./core/
|
||||
COPY api/ ./api/
|
||||
COPY handlers/ ./handlers/
|
||||
COPY mirrors/ ./mirrors/
|
||||
COPY scripts/ ./scripts/
|
||||
|
||||
# 创建数据目录
|
||||
RUN mkdir -p \
|
||||
/data \
|
||||
/downloads \
|
||||
&& chown -R nobody:nogroup /data /downloads
|
||||
|
||||
# 设置工作用户
|
||||
USER nobody
|
||||
|
||||
# 挂载卷
|
||||
VOLUME ["/data", "/downloads"]
|
||||
|
||||
# 暴露端口
|
||||
EXPOSE 8080
|
||||
|
||||
# 健康检查
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/api/v1/health')" || exit 1
|
||||
|
||||
# 启动命令
|
||||
CMD ["python", "main.py", \
|
||||
"--host", "0.0.0.0", \
|
||||
"--port", "8080", \
|
||||
"--base-dir", "/downloads"]
|
||||
@@ -0,0 +1,87 @@
|
||||
# ============================================
|
||||
# HYC下载站 v2.2 - 轻量级镜像 (X86_32/ARMv7)
|
||||
# 适用于低端设备: 2CPU/1G 内存
|
||||
# ============================================
|
||||
|
||||
# 使用 Alpine Linux 作为基础镜像 (极简)
|
||||
FROM alpine:3.19 AS builder
|
||||
|
||||
# 安装 Python 和构建工具
|
||||
RUN apk add --no-cache \
|
||||
python3 \
|
||||
py3-pip \
|
||||
gcc \
|
||||
musl-dev \
|
||||
libffi-dev \
|
||||
openssl-dev \
|
||||
cargo \
|
||||
rust
|
||||
|
||||
# 创建虚拟环境
|
||||
ENV VENV=/opt/venv
|
||||
RUN python3 -m venv $VENV
|
||||
ENV PATH="$VENV/bin:$PATH"
|
||||
|
||||
# 安装依赖 (使用 --no-cache-dir 减少体积)
|
||||
COPY requirements.txt /tmp/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /tmp/requirements.txt
|
||||
|
||||
# ============================================
|
||||
# 运行阶段 - 极简镜像
|
||||
# ============================================
|
||||
FROM alpine:3.19
|
||||
|
||||
# 安装最小运行时依赖
|
||||
RUN apk add --no-cache \
|
||||
python3 \
|
||||
libffi \
|
||||
openssl \
|
||||
libstdc++ \
|
||||
tzdata \
|
||||
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
|
||||
&& echo "Asia/Shanghai" > /etc/timezone
|
||||
|
||||
# 创建非 root 用户
|
||||
RUN adduser -D -s /bin/sh hyc
|
||||
|
||||
# 设置工作目录
|
||||
WORKDIR /app
|
||||
|
||||
# 从构建阶段复制虚拟环境
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
# 复制应用代码
|
||||
COPY --chown=hyc:hyc main.py .
|
||||
COPY --chown=hyc:hyc core/ ./core/
|
||||
COPY --chown=hyc:hyc api/ ./api/
|
||||
COPY --chown=hyc:hyc handlers/ ./handlers/
|
||||
COPY --chown=hyc:hyc mirrors/ ./mirrors/
|
||||
|
||||
# 创建数据目录
|
||||
RUN mkdir -p /data /downloads && chown -R hyc:hyc /data /downloads
|
||||
|
||||
# 设置用户
|
||||
USER hyc
|
||||
|
||||
# 挂载卷
|
||||
VOLUME ["/data", "/downloads"]
|
||||
|
||||
# 暴露端口
|
||||
EXPOSE 8080
|
||||
|
||||
# 健康检查
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/api/v1/health')" || exit 1
|
||||
|
||||
# 启动命令 (使用虚拟环境中的 Python)
|
||||
CMD ["python", "main.py", \
|
||||
"--host", "0.0.0.0", \
|
||||
"--port", "8080", \
|
||||
"--base-dir", "/downloads", \
|
||||
"--config", "/data/config.json", \
|
||||
"--memory-limit", "512M"]
|
||||
|
||||
# 镜像元数据
|
||||
LABEL maintainer="HYC Download Station"
|
||||
LABEL description="HYC下载站 v2.2 - 轻量级版本"
|
||||
@@ -0,0 +1,125 @@
|
||||
# ============================================
|
||||
# HYC下载站 v2.2 - 多架构交叉编译 Dockerfile
|
||||
# 支持: amd64, arm64, arm/v7, i386
|
||||
# 使用: docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t hyc:v2.2 .
|
||||
# ============================================
|
||||
|
||||
# ============================================
|
||||
# 构建阶段 - 所有架构通用
|
||||
# ============================================
|
||||
FROM python:3.11-alpine AS builder
|
||||
|
||||
# 安装构建依赖 (按架构)
|
||||
ARG TARGETARCH
|
||||
RUN case "$TARGETARCH" in \
|
||||
amd64|x86_64) ARCH_OPTS="-march=x86-64" ;; \
|
||||
arm64|aarch64) ARCH_OPTS="-march=armv8-a" ;; \
|
||||
armv7|arm) ARCH_OPTS="-march=armv7-a" ;; \
|
||||
i386|i686) ARCH_OPTS="-march=i686" ;; \
|
||||
*) ARCH_OPTS="-march=native" ;; \
|
||||
esac && \
|
||||
apk add --no-cache \
|
||||
python3 \
|
||||
py3-pip \
|
||||
gcc \
|
||||
musl-dev \
|
||||
libffi-dev \
|
||||
openssl-dev \
|
||||
cargo \
|
||||
rust
|
||||
|
||||
# 创建优化编译的虚拟环境
|
||||
ENV VENV=/opt/venv
|
||||
ENV CFLAGS="$ARCH_OPTS"
|
||||
ENV CXXFLAGS="$ARCH_OPTS"
|
||||
ENV LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now"
|
||||
|
||||
RUN python3 -m venv $VENV && \
|
||||
$VENV/bin/pip install --upgrade pip wheel
|
||||
|
||||
# 安装依赖
|
||||
COPY requirements.txt /tmp/requirements.txt
|
||||
RUN $VENV/bin/pip install --no-cache-dir \
|
||||
--platform manylinux2014_${TARGETARCH} \
|
||||
--only-binary=:all: \
|
||||
-r /tmp/requirements.txt 2>/dev/null || \
|
||||
$VENV/bin/pip install --no-cache-dir -r /tmp/requirements.txt
|
||||
|
||||
# ============================================
|
||||
# 运行阶段 - 按架构选择基础镜像
|
||||
# ============================================
|
||||
# amd64
|
||||
FROM --platform=$TARGETOS/$TARGETARCH alpine:3.19 AS runner-amd64
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
COPY --from=builder /app /app
|
||||
CMD ["/opt/venv/bin/python", "/app/main.py"]
|
||||
|
||||
# arm64
|
||||
FROM --platform=$TARGETOS/$TARGETARCH alpine:3.19 AS runner-arm64
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
COPY --from=builder /app /app
|
||||
CMD ["/opt/venv/bin/python", "/app/main.py"]
|
||||
|
||||
# arm/v7
|
||||
FROM --platform=$TARGETOS/$TARGETARCH alpine:3.19 AS runner-armv7
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
COPY --from=builder /app /app
|
||||
CMD ["/opt/venv/bin/python", "/app/main.py"]
|
||||
|
||||
# i386
|
||||
FROM --platform=$TARGETOS/$TARGETARCH alpine:3.19 AS runner-i386
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
COPY --from=builder /app /app
|
||||
CMD ["/opt/venv/bin/python", "/app/main.py"]
|
||||
|
||||
# ============================================
|
||||
# 最终清单镜像
|
||||
# ============================================
|
||||
FROM --platform=$TARGETOS/$TARGETARCH alpine:3.19 AS final
|
||||
|
||||
# 安装运行时依赖
|
||||
RUN apk add --no-cache \
|
||||
libffi \
|
||||
openssl \
|
||||
libstdc++ \
|
||||
tzdata \
|
||||
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
|
||||
&& echo "Asia/Shanghai" > /etc/timezone
|
||||
|
||||
# 创建用户
|
||||
RUN adduser -D -s /bin/sh hyc
|
||||
|
||||
# 复制虚拟环境
|
||||
COPY --from=builder --chown=hyc:hyc /opt/venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
# 复制应用
|
||||
COPY --chown=hyc:hyc main.py .
|
||||
COPY --chown=hyc:hyc settings.json .
|
||||
COPY --chown=hyc:hyc requirements.txt .
|
||||
COPY --chown=hyc:hyc core/ ./core/
|
||||
COPY --chown=hyc:hyc api/ ./api/
|
||||
COPY --chown=hyc:hyc handlers/ ./handlers/
|
||||
COPY --chown=hyc:hyc mirrors/ ./mirrors/
|
||||
COPY --chown=hyc:hyc scripts/ ./scripts/
|
||||
|
||||
# 创建数据目录
|
||||
RUN mkdir -p /data /downloads && chown -R hyc:hyc /data /downloads
|
||||
|
||||
USER hyc
|
||||
VOLUME ["/data", "/downloads"]
|
||||
EXPOSE 8080
|
||||
|
||||
# 低内存模式
|
||||
ENV HYC_LOW_MEMORY=1
|
||||
ENV HYC_WORKERS=1
|
||||
ENV PYTHONOPTIMIZE=2
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/api/v1/health')" || exit 1
|
||||
|
||||
CMD ["python", "main.py", \
|
||||
"--host", "0.0.0.0", \
|
||||
"--port", "8080", \
|
||||
"--memory-limit", "256M"]
|
||||
@@ -0,0 +1,27 @@
|
||||
# ============================================
|
||||
# PyInstaller 多架构构建镜像
|
||||
# ============================================
|
||||
FROM python:3.11 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 安装构建依赖
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
musl-dev \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 复制整个项目
|
||||
COPY . .
|
||||
|
||||
# 安装 PyInstaller 和项目依赖
|
||||
RUN pip install --no-cache-dir pyinstaller -r requirements.txt
|
||||
|
||||
# 构建可执行文件（使用 .spec 文件生成单文件）
|
||||
RUN pyinstaller pyinstaller.spec
|
||||
|
||||
FROM scratch
|
||||
COPY --from=builder /app/dist/hyc-download /
|
||||
@@ -0,0 +1,52 @@
|
||||
# ============================================
|
||||
# HYC下载站 v2.3 - 轻量级 Docker Compose
|
||||
# 适用于: Raspberry Pi, 2CPU/1G 设备
|
||||
# ============================================
|
||||
|
||||
services:
|
||||
# 轻量级应用服务
|
||||
hyc-server:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.lite
|
||||
container_name: hyc-download-server
|
||||
hostname: hyc-server
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
- TZ=Asia/Shanghai
|
||||
# 低端设备优化
|
||||
- PRESET=ultra_low
|
||||
- PYTHONOPTIMIZE=2
|
||||
- HYC_LOW_MEMORY=1
|
||||
volumes:
|
||||
- ${DATA_PATH:-./data}:/data
|
||||
- ${DOWNLOADS_PATH:-./downloads}:/downloads
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
# 内存限制
|
||||
memory: 512M
|
||||
cpus: '1.0'
|
||||
reservations:
|
||||
memory: 256M
|
||||
cpus: '0.5'
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/api/v1/health')"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
labels:
|
||||
- "com.hyc.version=2.2"
|
||||
- "com.hyc.preset=ultra_low"
|
||||
|
||||
networks:
|
||||
default:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
data:
|
||||
downloads:
|
||||
caches:
|
||||
@@ -0,0 +1,57 @@
|
||||
# ============================================
|
||||
# HYC下载站 v2.3 - 树莓派专用配置
|
||||
# 适用于: Raspberry Pi 3/4/5 (ARMv7/Arm64)
|
||||
# ============================================
|
||||
|
||||
services:
|
||||
# ARM 优化版本
|
||||
hyc-server:
|
||||
# 使用 ARM 专用镜像
|
||||
image: ${REGISTRY:-hyc-download-station}:${TAG:-v2.3}-arm64
|
||||
container_name: hyc-download-server
|
||||
hostname: hyc-server
|
||||
restart: unless-stopped
|
||||
privileged: true # 树莓派可能需要
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
- TZ=Asia/Shanghai
|
||||
# 树莓派优化
|
||||
- PRESET=low
|
||||
- PYTHONOPTIMIZE=2
|
||||
- HYC_LOW_MEMORY=1
|
||||
# ARM 优化
|
||||
- PYTHONCPUCOUNT=2
|
||||
volumes:
|
||||
- ${DATA_PATH:-./data}:/data
|
||||
- ${DOWNLOADS_PATH:-./downloads}:/downloads
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
# Raspberry Pi 4: 4GB RAM
|
||||
memory: 1G
|
||||
cpus: '2.0'
|
||||
reservations:
|
||||
memory: 512M
|
||||
cpus: '1.0'
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/api/v1/health')"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
labels:
|
||||
- "com.hyc.platform=raspberry-pi"
|
||||
- "com.hyc.arch=arm64"
|
||||
|
||||
# 温度监控 (树莓派专用)
|
||||
devices:
|
||||
- /sys/class/thermal/thermal_zone0:/sys/class/thermal/thermal_zone0
|
||||
|
||||
networks:
|
||||
default:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
data:
|
||||
downloads:
|
||||
caches:
|
||||
@@ -0,0 +1,87 @@
|
||||
# ============================================
|
||||
# HYC下载站 v2.3 - Docker Compose 配置
|
||||
# ============================================
|
||||
|
||||
services:
|
||||
# 主应用服务
|
||||
hyc-server:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile
|
||||
container_name: hyc-download-server
|
||||
hostname: hyc-server
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
- TZ=Asia/Shanghai
|
||||
# 数据库配置
|
||||
- DB_TYPE=${DB_TYPE:-sqlite}
|
||||
- DB_PATH=/data/hyc.db
|
||||
# PostgreSQL (可选)
|
||||
- DB_HOST=${DB_HOST:-}
|
||||
- DB_PORT=${DB_PORT:-5432}
|
||||
- DB_NAME=${DB_NAME:-hyc}
|
||||
- DB_USER=${DB_USER:-postgres}
|
||||
- DB_PASS=${DB_PASS:-}
|
||||
# 外部数据库连接 (可选)
|
||||
- DB_CONN_STR=${DB_CONN_STR:-}
|
||||
volumes:
|
||||
- ${DATA_PATH:-./data}:/data
|
||||
- ${DOWNLOADS_PATH:-./downloads}:/downloads
|
||||
networks:
|
||||
- hyc-network
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/api/v1/health')"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
labels:
|
||||
- "com.hyc.version=2.2"
|
||||
|
||||
# PostgreSQL 数据库 (可选)
|
||||
hyc-db:
|
||||
image: postgres:15-alpine
|
||||
container_name: hyc-postgres
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- POSTGRES_DB=${DB_NAME:-hyc}
|
||||
- POSTGRES_USER=${DB_USER:-postgres}
|
||||
- POSTGRES_PASSWORD=${DB_PASS:-postgres}
|
||||
- TZ=Asia/Shanghai
|
||||
volumes:
|
||||
- ${DB_PATH:-./postgres}:/var/lib/postgresql/data
|
||||
networks:
|
||||
- hyc-network
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-hyc}"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# Redis 缓存 (可选，用于速率限制)
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: hyc-redis
|
||||
restart: unless-stopped
|
||||
command: redis-server --appendonly yes
|
||||
volumes:
|
||||
- ${REDIS_PATH:-./redis}:/data
|
||||
networks:
|
||||
- hyc-network
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
networks:
|
||||
hyc-network:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
data:
|
||||
downloads:
|
||||
caches:
|
||||
postgres:
|
||||
redis:
|
||||
@@ -0,0 +1,4 @@
|
||||
# HTTP处理模块初始化
|
||||
from .http_handler import MirrorServerHandler
|
||||
|
||||
__all__ = ['MirrorServerHandler']
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,44 @@
|
||||
# -*- mode: python ; coding: utf-8 -*-
|
||||
|
||||
|
||||
a = Analysis(
|
||||
['main.py'],
|
||||
pathex=[],
|
||||
binaries=[],
|
||||
datas=[('api', 'api'), ('core', 'core'), ('handlers', 'handlers'), ('mirrors', 'mirrors'), ('settings.json', '.')],
|
||||
hiddenimports=[],
|
||||
hookspath=[],
|
||||
hooksconfig={},
|
||||
runtime_hooks=[],
|
||||
excludes=[],
|
||||
noarchive=False,
|
||||
optimize=0,
|
||||
)
|
||||
pyz = PYZ(a.pure)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
[],
|
||||
exclude_binaries=True,
|
||||
name='hyc-download',
|
||||
debug=False,
|
||||
bootloader_ignore_signals=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
console=True,
|
||||
disable_windowed_traceback=False,
|
||||
argv_emulation=False,
|
||||
target_arch=None,
|
||||
codesign_identity=None,
|
||||
entitlements_file=None,
|
||||
)
|
||||
coll = COLLECT(
|
||||
exe,
|
||||
a.binaries,
|
||||
a.datas,
|
||||
strip=False,
|
||||
upx=True,
|
||||
upx_exclude=[],
|
||||
name='hyc-download',
|
||||
)
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
# ============================================
|
||||
# HYC下载站 v2.3 - Kubernetes 部署指南
|
||||
# ============================================
|
||||
|
||||
## 快速部署
|
||||
|
||||
### 1. 创建命名空间 (可选)
|
||||
```bash
|
||||
kubectl create namespace hyc
|
||||
kubectl config set-context --current --namespace=hyc
|
||||
```
|
||||
|
||||
### 2. 部署应用
|
||||
```bash
|
||||
# 使用内置 PostgreSQL 和 Redis (可选)
|
||||
kubectl apply -f deployment.yaml
|
||||
kubectl apply -f postgres.yaml
|
||||
kubectl apply -f redis.yaml
|
||||
|
||||
# 或只部署应用，使用外部数据库
|
||||
kubectl apply -f deployment.yaml
|
||||
```
|
||||
|
||||
### 3. 验证部署
|
||||
```bash
|
||||
# 查看 Pods
|
||||
kubectl get pods -l app=hyc-download-server
|
||||
|
||||
# 查看日志
|
||||
kubectl logs -l app=hyc-download-server -f
|
||||
|
||||
# 查看服务
|
||||
kubectl get svc hyc-server
|
||||
```
|
||||
|
||||
### 4. 访问应用
|
||||
```bash
|
||||
# Port Forward (开发环境)
|
||||
kubectl port-forward svc/hyc-server 8080:8080
|
||||
|
||||
# 浏览器访问
|
||||
# http://localhost:8080
|
||||
# 管理界面: http://localhost:8080/api/ui/
|
||||
```
|
||||
|
||||
## 生产环境部署
|
||||
|
||||
### 1. 构建并推送镜像
|
||||
```bash
|
||||
# 构建镜像
|
||||
docker build -t hx100cv/hyc-download-station:v2.3 .
|
||||
|
||||
# 推送镜像
|
||||
docker push hx100cv/hyc-download-station:v2.3
|
||||
|
||||
# 更新 deployment.yaml 中的镜像地址
|
||||
```
|
||||
|
||||
### 2. 配置域名和 TLS
|
||||
```bash
|
||||
# 编辑 deployment.yaml，修改 Ingress 配置
|
||||
# 添加 TLS secret
|
||||
kubectl create secret tls hyc-tls-secret --cert=certificate.crt --key=private.key
|
||||
|
||||
# 应用配置
|
||||
kubectl apply -f deployment.yaml
|
||||
```
|
||||
|
||||
### 3. 配置资源限制
|
||||
```yaml
|
||||
# 根据实际需求调整 deployment.yaml 中的资源限制
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 2000m
|
||||
memory: 2Gi
|
||||
```
|
||||
|
||||
### 4. 配置 HPA (自动扩缩容)
|
||||
```bash
|
||||
# HPA 已内置，查看状态
|
||||
kubectl get hpa hyc-hpa
|
||||
kubectl top pods
|
||||
```
|
||||
|
||||
## 使用外部数据库
|
||||
|
||||
### PostgreSQL
|
||||
```bash
|
||||
# 设置环境变量或 Secret
|
||||
export DB_TYPE=postgresql
|
||||
export DB_HOST=your-postgres-host
|
||||
export DB_PORT=5432
|
||||
export DB_NAME=hyc
|
||||
export DB_USER=postgres
|
||||
export DB_PASSWORD=your-password
|
||||
|
||||
# 或使用连接字符串
|
||||
export DB_CONN_STR=postgresql://user:pass@host:5432/database
|
||||
```
|
||||
|
||||
### MySQL
|
||||
```bash
|
||||
export DB_TYPE=mysql
|
||||
export DB_HOST=your-mysql-host
|
||||
export DB_PORT=3306
|
||||
export DB_NAME=hyc
|
||||
export DB_USER=root
|
||||
export DB_PASSWORD=your-password
|
||||
```
|
||||
|
||||
## 监控
|
||||
|
||||
### Prometheus + Grafana
|
||||
```bash
|
||||
# ServiceMonitor 已内置
|
||||
# 确保 Prometheus Operator 已安装
|
||||
kubectl get servicemonitor hyc-monitor
|
||||
|
||||
# Grafana Dashboard (导入 json/dashboard.json)
|
||||
```
|
||||
|
||||
### 日志
|
||||
```bash
|
||||
# 查看应用日志
|
||||
kubectl logs -l app=hyc-download-server --tail=100
|
||||
|
||||
# 实时日志
|
||||
kubectl logs -l app=hyc-download-server -f
|
||||
```
|
||||
|
||||
## 升级
|
||||
|
||||
```bash
|
||||
# 更新镜像版本
|
||||
kubectl set image deployment/hyc-server hyc-server=hx100cv/hyc-download-station:v2.4
|
||||
|
||||
# 查看滚动更新
|
||||
kubectl rollout status deployment/hyc-server
|
||||
|
||||
# 回滚 (如有问题)
|
||||
kubectl rollout undo deployment/hyc-server
|
||||
```
|
||||
|
||||
## 卸载
|
||||
|
||||
```bash
|
||||
# 删除所有资源
|
||||
kubectl delete -f deployment.yaml
|
||||
kubectl delete -f postgres.yaml
|
||||
kubectl delete -f redis.yaml
|
||||
|
||||
# 删除 PVC (数据将丢失)
|
||||
kubectl delete pvc hyc-data-pvc hyc-downloads-pvc hyc-cache-pvc
|
||||
```
|
||||
@@ -0,0 +1,237 @@
|
||||
# ============================================
|
||||
# HYC下载站 v2.3 - Kubernetes 部署配置
|
||||
# ============================================
|
||||
|
||||
---
|
||||
# ConfigMap - 应用配置
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: hyc-config
|
||||
labels:
|
||||
app: hyc-download-server
|
||||
data:
|
||||
HYC_HOST: "0.0.0.0"
|
||||
HYC_PORT: "8080"
|
||||
HYC_BASE_DIR: "/downloads"
|
||||
HYC_ENABLE_MONITOR: "true"
|
||||
HYC_ENABLE_SYNC: "true"
|
||||
HYC_ENABLE_MIRRORS: "true"
|
||||
HYC_ENABLE_WS: "true"
|
||||
HYC_ENABLE_SSE: "true"
|
||||
|
||||
---
|
||||
# Secret - 数据库凭据
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: hyc-secrets
|
||||
type: Opaque
|
||||
stringData:
|
||||
# PostgreSQL 凭据
|
||||
DB_HOST: "hyc-postgres"
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: "hyc"
|
||||
DB_USER: "postgres"
|
||||
DB_PASSWORD: "your-password-here"
|
||||
|
||||
# Redis 凭据 (可选)
|
||||
REDIS_HOST: "hyc-redis"
|
||||
REDIS_PORT: "6379"
|
||||
|
||||
---
|
||||
# PersistentVolumeClaim - 数据存储
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: hyc-data-pvc
|
||||
labels:
|
||||
app: hyc-download-server
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
|
||||
---
|
||||
# PersistentVolumeClaim - 下载目录
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: hyc-downloads-pvc
|
||||
labels:
|
||||
app: hyc-download-server
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Gi
|
||||
|
||||
---
|
||||
# PersistentVolumeClaim - 缓存目录
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: hyc-cache-pvc
|
||||
labels:
|
||||
app: hyc-download-server
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
|
||||
---
|
||||
# Deployment - 主应用
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hyc-server
|
||||
labels:
|
||||
app: hyc-download-server
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: hyc-download-server
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: hyc-download-server
|
||||
spec:
|
||||
containers:
|
||||
- name: hyc-server
|
||||
image: hyc-download-station:v2.3
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: hyc-config
|
||||
- secretRef:
|
||||
name: hyc-secrets
|
||||
volumeMounts:
|
||||
- name: data-volume
|
||||
mountPath: /data
|
||||
- name: downloads-volume
|
||||
mountPath: /downloads
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 2000m
|
||||
memory: 2Gi
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/v1/health
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/v1/health
|
||||
port: 8080
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
volumes:
|
||||
- name: data-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: hyc-data-pvc
|
||||
- name: downloads-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: hyc-downloads-pvc
|
||||
- name: cache-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: hyc-cache-pvc
|
||||
|
||||
---
|
||||
# Service - 内部服务
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: hyc-server
|
||||
labels:
|
||||
app: hyc-download-server
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
app: hyc-download-server
|
||||
|
||||
---
|
||||
# Ingress - 外部访问
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: hyc-ingress
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "2G"
|
||||
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: hyc.example.com
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: hyc-server
|
||||
port:
|
||||
number: 8080
|
||||
|
||||
---
|
||||
# HorizontalPodAutoscaler - 自动扩缩容
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: hyc-hpa
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hyc-server
|
||||
minReplicas: 2
|
||||
maxReplicas: 10
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
|
||||
---
|
||||
# ServiceMonitor - Prometheus 监控
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: hyc-monitor
|
||||
labels:
|
||||
release: prometheus
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: hyc-download-server
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /metrics
|
||||
interval: 15s
|
||||
@@ -0,0 +1,28 @@
|
||||
# ============================================
|
||||
# HYC下载站 Helm Chart
|
||||
# ============================================
|
||||
|
||||
apiVersion: v2
|
||||
name: hyc-download-station
|
||||
description: HYC下载站 v2.3 - 镜像文件服务器 + 下载加速源
|
||||
version: 2.2.0
|
||||
appVersion: "2.2.0"
|
||||
keywords:
|
||||
- download-station
|
||||
- mirror
|
||||
- docker-registry
|
||||
- apt-mirror
|
||||
- pypi-mirror
|
||||
home: https://github.com/hyc-download-station
|
||||
maintainers:
|
||||
- name: HYC Team
|
||||
email: [email protected]
|
||||
dependencies:
|
||||
- name: postgresql
|
||||
version: 12.x.x
|
||||
condition: postgresql.enabled
|
||||
repository: "https://charts.bitnami.com/bitnami"
|
||||
- name: redis
|
||||
version: 18.x.x
|
||||
condition: redis.enabled
|
||||
repository: "https://charts.bitnami.com/bitnami"
|
||||
@@ -0,0 +1,31 @@
|
||||
=============================================
|
||||
HYC下载站 v2.3 已成功部署!
|
||||
=============================================
|
||||
|
||||
1. 获取应用 URL:
|
||||
|
||||
{{- if .Values.ingress.enabled }}
|
||||
{{- range $host := .Values.ingress.hosts }}
|
||||
http://{{ $host.host }}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
# Port Forward (开发环境)
|
||||
kubectl port-forward svc/{{ include "hyc.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }}
|
||||
|
||||
# 访问地址
|
||||
http://localhost:{{ .Values.service.port }}
|
||||
{{- end }}
|
||||
|
||||
2. 管理界面:
|
||||
http://<your-url>/api/ui/
|
||||
|
||||
3. API 文档:
|
||||
http://<your-url>/api/docs/
|
||||
|
||||
4. 查看状态:
|
||||
kubectl get pods -l {{ include "hyc.selectorLabels" . | trimSuffix "-" | replace "=" "=" | replace " " "" }}
|
||||
|
||||
5. 查看日志:
|
||||
kubectl logs -l {{ include "hyc.selectorLabels" . | trimSuffix "-" | replace "=" "=" | replace " " "" }} -f
|
||||
|
||||
=============================================
|
||||
@@ -0,0 +1,59 @@
|
||||
{{/*
|
||||
创建完整名称
|
||||
*/}}
|
||||
{{- define "hyc.fullname" -}}
|
||||
{{- printf "%s-%s" .Release.Name (include "hyc.name" .) | trunc 63 -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
标签
|
||||
*/}}
|
||||
{{- define "hyc.labels" -}}
|
||||
helm.sh/chart: {{ include "hyc.chart" . }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/part-of: {{ .Chart.Name }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
选择器标签
|
||||
*/}}
|
||||
{{- define "hyc.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "hyc.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
图表名称
|
||||
*/}}
|
||||
{{- define "hyc.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
图表版本
|
||||
*/}}
|
||||
{{- define "hyc.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
PostgreSQL 主机
|
||||
*/}}
|
||||
{{- define "hyc.postgresql.host" -}}
|
||||
{{- if .Values.postgresql.enabled -}}
|
||||
{{- printf "%s-%s" .Release.Name "postgresql" -}}
|
||||
{{- else -}}
|
||||
{{- .Values.config.database.postgresql.host | default "localhost" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Redis 主机
|
||||
*/}}
|
||||
{{- define "hyc.redis.host" -}}
|
||||
{{- if .Values.redis.enabled -}}
|
||||
{{- printf "%s-%s" .Release.Name "redis" -}}
|
||||
{{- else -}}
|
||||
{{- .Values.config.database.redis.host | default "localhost" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@@ -0,0 +1,142 @@
|
||||
{{- if not .Values.postgresql.enabled }}
|
||||
{{- if not .Values.redis.enabled }}
|
||||
---
|
||||
# 独立部署 (无依赖)
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}-config
|
||||
data:
|
||||
HYC_HOST: {{ .Values.config.host | quote }}
|
||||
HYC_PORT: {{ .Values.config.port | quote }}
|
||||
HYC_BASE_DIR: {{ .Values.config.baseDir | quote }}
|
||||
HYC_CACHE_DIR: {{ .Values.config.cacheDir | quote }}
|
||||
HYC_ENABLE_MONITOR: {{ .Values.config.enableMonitor | quote }}
|
||||
HYC_ENABLE_SYNC: {{ .Values.config.enableSync | quote }}
|
||||
HYC_ENABLE_MIRRORS: {{ .Values.config.enableMirrors | quote }}
|
||||
HYC_ENABLE_WS: {{ .Values.config.enableWs | quote }}
|
||||
HYC_ENABLE_SSE: {{ .Values.config.enableSse | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}-secrets
|
||||
type: Opaque
|
||||
stringData:
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
DB_HOST: {{ include "hyc.postgresql.host" . }}
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: {{ .Values.postgresql.auth.database }}
|
||||
DB_USER: {{ .Values.postgresql.auth.username }}
|
||||
DB_PASSWORD: {{ .Values.postgresql.auth.password }}
|
||||
{{- else if .Values.config.database.external.enabled }}
|
||||
DB_CONN_STR: {{ .Values.config.database.external.connectionString }}
|
||||
{{- end }}
|
||||
{{- if .Values.redis.enabled }}
|
||||
REDIS_HOST: {{ include "hyc.redis.host" . }}
|
||||
REDIS_PORT: "6379"
|
||||
{{- end }}
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}
|
||||
labels:
|
||||
{{- include "hyc.labels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "hyc.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "hyc.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: {{ .Chart.Name }}
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.config.port }}
|
||||
name: http
|
||||
envFrom:
|
||||
{{- if not .Values.postgresql.enabled }}
|
||||
{{- if not .Values.config.database.external.enabled }}
|
||||
- configMapRef:
|
||||
name: {{ include "hyc.fullname" . }}-config
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- secretRef:
|
||||
name: {{ include "hyc.fullname" . }}-secrets
|
||||
volumeMounts:
|
||||
- name: data-volume
|
||||
mountPath: /data
|
||||
- name: downloads-volume
|
||||
mountPath: {{ .Values.config.baseDir }}
|
||||
- name: caches-volume
|
||||
mountPath: {{ .Values.config.cacheDir }}
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
{{- if .Values.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/v1/health
|
||||
port: {{ .Values.config.port }}
|
||||
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
{{- if .Values.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/v1/health
|
||||
port: {{ .Values.config.port }}
|
||||
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: data-volume
|
||||
{{- if .Values.persistence.data.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "hyc.fullname" . }}-data-pvc
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
- name: downloads-volume
|
||||
{{- if .Values.persistence.downloads.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "hyc.fullname" . }}-downloads-pvc
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
- name: caches-volume
|
||||
{{- if .Values.persistence.caches.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "hyc.fullname" . }}-caches-pvc
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,37 @@
|
||||
{{- if .Values.autoscaling.enabled }}
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}-hpa
|
||||
labels:
|
||||
{{- include "hyc.labels" . | nindent 4 }}
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: {{ include "hyc.fullname" . }}
|
||||
minReplicas: {{ .Values.autoscaling.minReplicas }}
|
||||
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
|
||||
metrics:
|
||||
{{- toYaml .Values.autoscaling.metrics | nindent 4 }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.metrics.enabled }}
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}-monitor
|
||||
labels:
|
||||
{{- include "hyc.labels" . | nindent 4 }}
|
||||
release: {{ .Values.metrics.release }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "hyc.selectorLabels" . | nindent 6 }}
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /metrics
|
||||
interval: {{ .Values.metrics.interval }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,56 @@
|
||||
{{- if .Values.persistence.data.enabled }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}-data-pvc
|
||||
labels:
|
||||
{{- include "hyc.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.data.size }}
|
||||
{{- if .Values.persistence.data.storageClass }}
|
||||
storageClassName: {{ .Values.persistence.data.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.persistence.downloads.enabled }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}-downloads-pvc
|
||||
labels:
|
||||
{{- include "hyc.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.downloads.size }}
|
||||
{{- if .Values.persistence.downloads.storageClass }}
|
||||
storageClassName: {{ .Values.persistence.downloads.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.persistence.caches.enabled }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}-caches-pvc
|
||||
labels:
|
||||
{{- include "hyc.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.caches.size }}
|
||||
{{- if .Values.persistence.caches.storageClass }}
|
||||
storageClassName: {{ .Values.persistence.caches.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}
|
||||
labels:
|
||||
{{- include "hyc.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "hyc.selectorLabels" . | nindent 4 }}
|
||||
|
||||
{{- if .Values.ingress.enabled }}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "hyc.fullname" . }}-ingress
|
||||
labels:
|
||||
{{- include "hyc.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "2G"
|
||||
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
|
||||
spec:
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
{{- if .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- toYaml .Values.ingress.tls | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- toYaml .Values.ingress.hosts | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,169 @@
|
||||
# ============================================
|
||||
# HYC下载站 Helm Chart - 默认配置
|
||||
# ============================================
|
||||
|
||||
# 副本数
|
||||
replicaCount: 2
|
||||
|
||||
# 镜像配置
|
||||
image:
|
||||
repository: hyc-download-station
|
||||
tag: v2.3
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# 镜像拉取密钥 (如需要私有仓库)
|
||||
imagePullSecrets: []
|
||||
|
||||
# 服务配置
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
|
||||
# 入口配置 (Ingress)
|
||||
ingress:
|
||||
enabled: true
|
||||
className: nginx
|
||||
hosts:
|
||||
- host: hyc.example.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
tls: []
|
||||
# - secretName: hyc-tls
|
||||
# hosts:
|
||||
# - hyc.example.com
|
||||
|
||||
# 资源配置
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 2000m
|
||||
memory: 2Gi
|
||||
|
||||
# 持久化配置
|
||||
persistence:
|
||||
data:
|
||||
enabled: true
|
||||
storageClass: ""
|
||||
size: 10Gi
|
||||
downloads:
|
||||
enabled: true
|
||||
storageClass: ""
|
||||
size: 100Gi
|
||||
caches:
|
||||
enabled: true
|
||||
storageClass: ""
|
||||
size: 50Gi
|
||||
|
||||
# PostgreSQL (可选)
|
||||
postgresql:
|
||||
enabled: false
|
||||
auth:
|
||||
database: hyc
|
||||
username: postgres
|
||||
password: "postgres-password"
|
||||
|
||||
# Redis (可选)
|
||||
redis:
|
||||
enabled: false
|
||||
auth:
|
||||
enabled: true
|
||||
password: "redis-password"
|
||||
|
||||
# 应用配置
|
||||
config:
|
||||
# 基础配置
|
||||
host: "0.0.0.0"
|
||||
port: 8080
|
||||
baseDir: "/downloads"
|
||||
apiVersion: "v2"
|
||||
|
||||
# 功能开关
|
||||
enableMonitor: true
|
||||
enableSync: true
|
||||
enableMirrors: true
|
||||
enableWs: true
|
||||
enableSse: true
|
||||
|
||||
# 数据库配置
|
||||
database:
|
||||
enabled: true
|
||||
type: "postgresql"
|
||||
syncInterval: 60
|
||||
|
||||
# 速率限制
|
||||
rateLimit:
|
||||
requestsPerMinute: 100
|
||||
burstLimit: 20
|
||||
|
||||
# 安全配置
|
||||
security:
|
||||
# IP 白名单/黑名单
|
||||
ip:
|
||||
whitelist: []
|
||||
blacklist: []
|
||||
|
||||
# SSL/TLS
|
||||
ssl:
|
||||
enabled: false
|
||||
cert: ""
|
||||
key: ""
|
||||
|
||||
# 节点选择器
|
||||
nodeSelector: {}
|
||||
|
||||
# 容忍
|
||||
tolerations: []
|
||||
|
||||
# 亲和性
|
||||
affinity: {}
|
||||
|
||||
# 探针
|
||||
livenessProbe:
|
||||
enabled: true
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
|
||||
readinessProbe:
|
||||
enabled: true
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
|
||||
# 滚动更新策略
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
|
||||
# 拓扑分布 (可选)
|
||||
topologySpreadConstraints: []
|
||||
|
||||
# 自动扩缩容
|
||||
autoscaling:
|
||||
enabled: true
|
||||
minReplicas: 2
|
||||
maxReplicas: 10
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
|
||||
# 监控指标
|
||||
metrics:
|
||||
enabled: false
|
||||
release: prometheus
|
||||
interval: 15s
|
||||
@@ -0,0 +1,113 @@
|
||||
# ============================================
|
||||
# PostgreSQL 数据库部署 (可选)
|
||||
# ============================================
|
||||
|
||||
---
|
||||
# StorageClass (如果需要动态供应)
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: hyc-storage
|
||||
provisioner: kubernetes.io/no-provisioner
|
||||
volumeBindingMode: WaitForFirstConsumer
|
||||
|
||||
---
|
||||
# PersistentVolume - PostgreSQL 数据
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: hyc-postgres-pv
|
||||
spec:
|
||||
capacity:
|
||||
storage: 20Gi
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: hyc-storage
|
||||
hostPath:
|
||||
path: /data/hyc-postgres
|
||||
type: DirectoryOrCreate
|
||||
|
||||
---
|
||||
# PersistentVolumeClaim - PostgreSQL
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: hyc-postgres-pvc
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
storageClassName: hyc-storage
|
||||
|
||||
---
|
||||
# Secret - PostgreSQL 凭据
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: hyc-postgres-secret
|
||||
type: Opaque
|
||||
stringData:
|
||||
POSTGRES_DB: hyc
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: your-secure-password
|
||||
|
||||
---
|
||||
# Service - PostgreSQL
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: hyc-postgres
|
||||
spec:
|
||||
selector:
|
||||
app: hyc-postgres
|
||||
ports:
|
||||
- port: 5432
|
||||
targetPort: 5432
|
||||
clusterIP: None
|
||||
|
||||
---
|
||||
# Deployment - PostgreSQL
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hyc-postgres
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: hyc-postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: hyc-postgres
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:15-alpine
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: hyc-postgres-secret
|
||||
volumeMounts:
|
||||
- name: postgres-data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- pg_isready
|
||||
- -U
|
||||
- postgres
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: postgres-data
|
||||
persistentVolumeClaim:
|
||||
claimName: hyc-postgres-pvc
|
||||
@@ -0,0 +1,93 @@
|
||||
# ============================================
|
||||
# Redis 缓存部署 (可选，用于速率限制)
|
||||
# ============================================
|
||||
|
||||
---
|
||||
# PersistentVolumeClaim - Redis
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: hyc-redis-pvc
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
|
||||
---
|
||||
# Secret - Redis 凭据 (可选)
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: hyc-redis-secret
|
||||
type: Opaque
|
||||
stringData:
|
||||
REDIS_PASSWORD: your-redis-password
|
||||
|
||||
---
|
||||
# Service - Redis
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: hyc-redis
|
||||
spec:
|
||||
selector:
|
||||
app: hyc-redis
|
||||
ports:
|
||||
- port: 6379
|
||||
targetPort: 6379
|
||||
clusterIP: None
|
||||
|
||||
---
|
||||
# Deployment - Redis
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hyc-redis
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: hyc-redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: hyc-redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7-alpine
|
||||
command:
|
||||
- redis-server
|
||||
- --appendonly
|
||||
- "yes"
|
||||
- --maxmemory
|
||||
- "256mb"
|
||||
- --maxmemory-policy
|
||||
- allkeys-lru
|
||||
volumeMounts:
|
||||
- name: redis-data
|
||||
mountPath: /data
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- redis-cli
|
||||
- ping
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: redis-data
|
||||
persistentVolumeClaim:
|
||||
claimName: hyc-redis-pvc
|
||||
|
||||
---
|
||||
# RedisCluster (可选，集群模式)
|
||||
# 如果需要高可用，可以改用 Redis Operator
|
||||
@@ -0,0 +1,606 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
HYC下载站 v2.3 - 完整增强版
|
||||
主入口文件
|
||||
支持镜像同步、下载加速源、系统监控、实时通信等
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import argparse
|
||||
import time
|
||||
|
||||
# PyInstaller 资源路径处理
|
||||
def get_resource_path(relative_path):
|
||||
"""获取打包后的资源路径"""
|
||||
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
|
||||
# 打包后的路径
|
||||
return os.path.join(sys._MEIPASS, relative_path)
|
||||
return os.path.join(os.path.dirname(os.path.abspath(__file__)), relative_path)
|
||||
|
||||
# 添加项目根目录到Python路径
|
||||
base_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, base_dir)
|
||||
|
||||
from core.config import ConfigManager, load_config_file, deep_merge, load_settings_with_override
|
||||
from core.server import MirrorServer
|
||||
from core.utils import parse_size
|
||||
from core.database import init_database, load_db_config_from_env, merge_config
|
||||
from core.sync_scheduler import init_database_sync
|
||||
from core.optimization import (
|
||||
MemoryManager, LowMemoryConfig, ArchitectureDetector, check_compatibility
|
||||
)
|
||||
|
||||
|
||||
def signal_handler(signum, _frame):
|
||||
"""处理退出信号"""
|
||||
print(f"\n收到信号 {signum}，正在关闭服务器...")
|
||||
import os
|
||||
os._exit(0)
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
"""解析命令行参数"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='HYC下载站 v2.3 - 镜像文件服务器 + 下载加速源',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog='''
|
||||
示例:
|
||||
# 基本启动
|
||||
python main.py -p 8080 -d ./downloads
|
||||
|
||||
# 使用配置文件
|
||||
python main.py --settings settings.json --config custom.json
|
||||
|
||||
# 认证配置
|
||||
python main.py --auth-type token --auth-token your_token_here
|
||||
|
||||
# 功能开关
|
||||
python main.py --enable-monitor --enable-sync --enable-mirrors
|
||||
python main.py --disable-ws --disable-sse # 适合低端设备
|
||||
|
||||
# 镜像加速源 (通过 settings.json 配置各个镜像的启用/禁用)
|
||||
python main.py --enable-mirrors
|
||||
|
||||
# 低端设备优化
|
||||
python main.py --preset ultra_low
|
||||
python main.py --memory-limit 256M --workers 2
|
||||
|
||||
# 调试
|
||||
python main.py --debug
|
||||
python main.py --debug-types http auth api v2 error
|
||||
|
||||
# 检查系统兼容性
|
||||
python main.py --check-compat
|
||||
'''
|
||||
)
|
||||
|
||||
# 服务器配置
|
||||
parser.add_argument('--host', default='0.0.0.0', help='监听地址 (默认: 0.0.0.0)')
|
||||
parser.add_argument('-p', '--port', type=int, default=8080, help='监听端口 (默认: 8080)')
|
||||
parser.add_argument('-d', '--base-dir', default='./downloads', help='文件存储目录 (默认: ./downloads)')
|
||||
parser.add_argument('--server-name', default='HYC下载站', help='服务器名称')
|
||||
|
||||
# HTTPS配置
|
||||
parser.add_argument('--ssl-cert', help='SSL证书文件路径')
|
||||
parser.add_argument('--ssl-key', help='SSL私钥文件路径')
|
||||
|
||||
# 认证配置
|
||||
auth_group = parser.add_argument_group('认证配置')
|
||||
auth_group.add_argument('--auth-type', choices=['none', 'basic', 'token'], default=None,
|
||||
help='认证类型: none(无), basic(基本认证), token(令牌认证)')
|
||||
auth_group.add_argument('--auth-user', default='admin', help='基本认证用户名')
|
||||
auth_group.add_argument('--auth-pass', help='基本认证密码')
|
||||
auth_group.add_argument('--auth-token', help='令牌认证密钥')
|
||||
|
||||
# 功能配置
|
||||
func_group = parser.add_argument_group('功能配置')
|
||||
func_group.add_argument('--directory-listing', type=bool, default=True,
|
||||
help='启用目录浏览 (默认: True)')
|
||||
func_group.add_argument('--enable-stats', type=bool, default=True,
|
||||
help='启用下载统计 (默认: True)')
|
||||
func_group.add_argument('--show-hash', action='store_true', help='显示文件哈希值')
|
||||
func_group.add_argument('--ignore-hidden', action='store_true', default=True,
|
||||
help='忽略隐藏文件')
|
||||
func_group.add_argument('--max-upload-size', default='1G', help='最大上传文件大小')
|
||||
func_group.add_argument('--api-version', choices=['v1', 'v2'], default='v2',
|
||||
help='API版本 (默认: v2)')
|
||||
|
||||
# 实时通信配置
|
||||
realtime_group = parser.add_argument_group('实时通信配置')
|
||||
realtime_group.add_argument('--enable-ws', action='store_true', default=True,
|
||||
help='启用WebSocket (默认: True)')
|
||||
realtime_group.add_argument('--enable-sse', action='store_true', default=True,
|
||||
help='启用SSE (默认: True)')
|
||||
|
||||
# 系统监控配置
|
||||
monitor_group = parser.add_argument_group('系统监控配置')
|
||||
monitor_group.add_argument('--enable-monitor', action='store_true', default=True,
|
||||
help='启用系统监控 (默认: True)')
|
||||
monitor_group.add_argument('--monitor-interval', type=int, default=5,
|
||||
help='监控数据采集间隔(秒) (默认: 5)')
|
||||
|
||||
# 同步配置
|
||||
sync_group = parser.add_argument_group('同步配置')
|
||||
sync_group.add_argument('--enable-sync', action='store_true', default=True,
|
||||
help='启用镜像同步 (默认: True)')
|
||||
sync_group.add_argument('--sync-config', help='同步配置文件路径')
|
||||
|
||||
# 镜像加速源配置
|
||||
mirror_group = parser.add_argument_group('镜像加速源配置')
|
||||
mirror_group.add_argument('--enable-mirrors', action='store_true', default=True,
|
||||
help='启用下载加速源 (默认: True)')
|
||||
# 注意: 各个镜像的启用/禁用通过 settings.json 中的 mirrors.xxx.enabled 配置
|
||||
|
||||
# 下载限速
|
||||
rate_group = parser.add_argument_group('下载限速配置')
|
||||
rate_group.add_argument('--rate-limit', type=int, default=0,
|
||||
help='全局下载限速(字节/秒) (默认: 0=不限速)')
|
||||
|
||||
# 低端设备优化配置
|
||||
optimize_group = parser.add_argument_group('低端设备优化配置')
|
||||
optimize_group.add_argument('--preset', choices=['ultra_low', 'low', 'medium', 'high', 'auto'],
|
||||
default='auto', help='设备预设 (默认: auto)')
|
||||
optimize_group.add_argument('--memory-limit', default='512M',
|
||||
help='内存限制 (例如: 256M, 512M, 1G)')
|
||||
optimize_group.add_argument('--workers', type=int, default=0,
|
||||
help='工作进程数 (0=自动)')
|
||||
optimize_group.add_argument('--chunk-size', default='128K',
|
||||
help='文件传输块大小 (默认: 128K)')
|
||||
optimize_group.add_argument('--buffer-size', default='256K',
|
||||
help='缓冲区大小 (默认: 256K)')
|
||||
optimize_group.add_argument('--disable-ws', action='store_true',
|
||||
help='禁用WebSocket (低端设备)')
|
||||
optimize_group.add_argument('--disable-sse', action='store_true',
|
||||
help='禁用SSE (低端设备)')
|
||||
optimize_group.add_argument('--disable-hash', action='store_true',
|
||||
help='禁用文件哈希计算 (低端设备)')
|
||||
optimize_group.add_argument('--check-compat', action='store_true',
|
||||
help='检查系统兼容性后退出')
|
||||
|
||||
# 日志配置
|
||||
log_group = parser.add_argument_group('日志配置')
|
||||
log_group.add_argument('--access-log', help='访问日志文件路径')
|
||||
log_group.add_argument('--verbose', '-v', action='count', default=0, help='详细输出')
|
||||
log_group.add_argument('--quiet', '-q', action='store_true', help='静默模式')
|
||||
|
||||
# 细粒度调试开关
|
||||
debug_group = parser.add_argument_group('调试选项')
|
||||
debug_group.add_argument('--debug', '-D', action='store_true',
|
||||
help='启用所有调试输出')
|
||||
debug_group.add_argument('--debug-log', dest='debug_log_file',
|
||||
help='调试日志文件路径 (debug 输出将写入此文件)')
|
||||
debug_group.add_argument('--debug-http', action='store_true',
|
||||
help='调试 HTTP 请求')
|
||||
debug_group.add_argument('--debug-auth', action='store_true',
|
||||
help='调试 认证检查')
|
||||
debug_group.add_argument('--debug-api', action='store_true',
|
||||
help='调试 API 路由')
|
||||
debug_group.add_argument('--debug-v2', action='store_true',
|
||||
help='调试 V2 API')
|
||||
debug_group.add_argument('--debug-error', action='store_true',
|
||||
help='调试 错误堆栈')
|
||||
debug_group.add_argument('--debug-download', action='store_true',
|
||||
help='调试 下载记录')
|
||||
debug_group.add_argument('--debug-types', '--debug-list',
|
||||
nargs='+', metavar='TYPE',
|
||||
choices=['http', 'auth', 'api', 'v2', 'error', 'download'],
|
||||
help='指定调试类型列表 (http auth api v2 error download)')
|
||||
|
||||
# 配置文件
|
||||
parser.add_argument('--settings', '--default-config', dest='settings',
|
||||
help='默认配置文件路径 (settings.json)')
|
||||
parser.add_argument('--config', help='覆盖配置文件路径 (JSON格式，会覆盖默认配置)')
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_config_from_args(args):
|
||||
"""从命令行参数构建配置"""
|
||||
cmd_config = {}
|
||||
|
||||
# 服务器基本配置
|
||||
basic_args = ['host', 'port', 'server_name', 'ssl_cert', 'ssl_key',
|
||||
'auth_type', 'auth_user', 'auth_pass', 'auth_token',
|
||||
'access_log', 'verbose',
|
||||
'api_version', 'directory_listing',
|
||||
'enable_stats', 'show_hash', 'ignore_hidden', 'max_upload_size']
|
||||
|
||||
for arg_name in basic_args:
|
||||
arg_value = getattr(args, arg_name, None)
|
||||
if arg_value is not None:
|
||||
cmd_config[arg_name] = arg_value
|
||||
|
||||
# Debug 配置处理
|
||||
# 优先级: --debug > --debug-types > individual --debug-xxx
|
||||
debug_types = []
|
||||
if args.debug_types:
|
||||
# 用户指定了类型列表
|
||||
debug_types = list(args.debug_types)
|
||||
cmd_config['debug'] = debug_types
|
||||
elif args.debug_http or args.debug_auth or args.debug_api or args.debug_v2 or args.debug_error or args.debug_download:
|
||||
# 用户指定了单个类型
|
||||
if args.debug_http:
|
||||
debug_types.append('http')
|
||||
if args.debug_auth:
|
||||
debug_types.append('auth')
|
||||
if args.debug_api:
|
||||
debug_types.append('api')
|
||||
if args.debug_v2:
|
||||
debug_types.append('v2')
|
||||
if args.debug_error:
|
||||
debug_types.append('error')
|
||||
if args.debug_download:
|
||||
debug_types.append('download')
|
||||
cmd_config['debug'] = debug_types
|
||||
elif args.debug:
|
||||
# 开启所有
|
||||
cmd_config['debug'] = True
|
||||
else:
|
||||
# 未开启任何 debug
|
||||
cmd_config['debug'] = False
|
||||
|
||||
# Debug 日志文件
|
||||
if args.debug_log_file:
|
||||
cmd_config['debug_log_file'] = args.debug_log_file
|
||||
|
||||
# 路径处理
|
||||
if args.base_dir:
|
||||
cmd_config['base_dir'] = os.path.abspath(args.base_dir)
|
||||
|
||||
# 实时通信
|
||||
cmd_config['enable_ws'] = args.enable_ws
|
||||
cmd_config['enable_sse'] = args.enable_sse
|
||||
|
||||
# 系统监控
|
||||
cmd_config['enable_monitor'] = args.enable_monitor
|
||||
cmd_config['monitor_interval'] = args.monitor_interval
|
||||
|
||||
# 同步
|
||||
cmd_config['enable_sync'] = args.enable_sync
|
||||
|
||||
# 镜像加速源
|
||||
cmd_config['enable_mirrors'] = args.enable_mirrors
|
||||
# 各个镜像的启用/禁用通过 settings.json 中的 mirrors.xxx.enabled 配置
|
||||
|
||||
# 下载限速
|
||||
cmd_config['rate_limit'] = args.rate_limit
|
||||
|
||||
# 静默模式
|
||||
if args.quiet:
|
||||
cmd_config['verbose'] = -1
|
||||
|
||||
return cmd_config
|
||||
|
||||
|
||||
def main():
|
||||
"""主入口函数"""
|
||||
args = parse_arguments()
|
||||
|
||||
# 确定默认配置文件路径
|
||||
project_root = os.path.dirname(os.path.abspath(__file__))
|
||||
settings_path = args.settings or os.path.join(project_root, 'settings.json')
|
||||
|
||||
# 打印配置来源信息
|
||||
print(f"[配置加载]")
|
||||
print(f" 默认配置: {settings_path}")
|
||||
|
||||
# 加载配置（优先级从低到高：默认配置 -> 覆盖配置 -> 环境变量 -> 命令行参数）
|
||||
# 使用深度合并，只替换覆盖配置中有的字段
|
||||
|
||||
# 1. 加载默认配置 (settings.json)
|
||||
from core.config import load_json_config
|
||||
default_config = load_json_config(settings_path) or {}
|
||||
print(f" 默认配置加载: {'成功' if default_config else '使用内联默认'}")
|
||||
|
||||
# 2. 从覆盖配置文件加载
|
||||
override_config = {}
|
||||
if args.config:
|
||||
override_config = load_json_config(args.config) or {}
|
||||
print(f" 覆盖配置: {args.config} ({len(override_config)} 个顶层键)")
|
||||
else:
|
||||
print(f" 覆盖配置: 未指定")
|
||||
|
||||
# 3. 深度合并默认配置和覆盖配置
|
||||
config = deep_merge(default_config, override_config)
|
||||
|
||||
# 4. 从环境变量加载数据库配置
|
||||
env_db_config = load_db_config_from_env()
|
||||
if env_db_config:
|
||||
config = deep_merge(config, env_db_config)
|
||||
|
||||
# 5. 同步配置
|
||||
if args.sync_config:
|
||||
sync_config = load_json_config(args.sync_config)
|
||||
if sync_config:
|
||||
if 'sync_sources' in sync_config:
|
||||
config['sync_sources'] = deep_merge(
|
||||
config.get('sync_sources', {}),
|
||||
sync_config['sync_sources']
|
||||
)
|
||||
if 'mirrors' in sync_config:
|
||||
config['mirrors'] = deep_merge(
|
||||
config.get('mirrors', {}),
|
||||
sync_config['mirrors']
|
||||
)
|
||||
|
||||
# 6. 命令行参数覆盖（最高优先级）
|
||||
cmd_config = build_config_from_args(args)
|
||||
config = deep_merge(config, cmd_config)
|
||||
|
||||
# 7. 处理 auth_token（优先从文件读取，否则生成新的）
|
||||
import secrets
|
||||
|
||||
# 确定 token 文件路径
|
||||
if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
|
||||
# 打包环境：使用 _MEIPASS 目录
|
||||
token_file = os.path.join(sys._MEIPASS, 'auth_token.txt')
|
||||
else:
|
||||
# 非打包：使用脚本所在目录
|
||||
token_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'auth_token.txt')
|
||||
|
||||
# 尝试从文件读取
|
||||
existing_token = None
|
||||
if os.path.exists(token_file):
|
||||
try:
|
||||
with open(token_file, 'r') as f:
|
||||
existing_token = f.read().strip()
|
||||
if existing_token:
|
||||
config['auth_token'] = existing_token
|
||||
print(f" 已从文件加载 auth_token: {config['auth_token'][:16]}...")
|
||||
except Exception as e:
|
||||
print(f" 警告: 读取 auth_token 失败: {e}")
|
||||
|
||||
# 如果没有现有 token，生成新的
|
||||
if not existing_token:
|
||||
config['auth_token'] = secrets.token_hex(32)
|
||||
print(f" 已生成新的 auth_token: {config['auth_token'][:16]}...")
|
||||
|
||||
# 保存新的 token 到文件
|
||||
try:
|
||||
with open(token_file, 'w') as f:
|
||||
f.write(config['auth_token'])
|
||||
print(f" 已保存 auth_token 到: {token_file}")
|
||||
except Exception as e:
|
||||
print(f" 警告: 保存 auth_token 失败: {e}")
|
||||
|
||||
print(f" 最终配置: {len(config)} 个顶层配置项")
|
||||
|
||||
# ==================== 系统兼容性检查 ====================
|
||||
if args.check_compat:
|
||||
print("\n[系统兼容性检查]")
|
||||
compat = check_compatibility()
|
||||
print(f"\n兼容状态: {'✓ 通过' if compat['compatible'] else '✗ 存在问题'}")
|
||||
|
||||
arch = ArchitectureDetector.get_architecture()
|
||||
print(f"\n架构信息:")
|
||||
print(f" - 平台: {arch['platform']}")
|
||||
print(f" - 机器: {arch['machine']}")
|
||||
print(f" - 架构: {arch['architecture']}")
|
||||
|
||||
recommended = ArchitectureDetector.get_recommended_config()
|
||||
print(f"\n推荐配置:")
|
||||
for k, v in recommended.items():
|
||||
print(f" - {k}: {v}")
|
||||
|
||||
if compat['warnings']:
|
||||
print(f"\n警告:")
|
||||
for w in compat['warnings']:
|
||||
print(f" ⚠ {w}")
|
||||
|
||||
if compat['errors']:
|
||||
print(f"\n错误:")
|
||||
for e in compat['errors']:
|
||||
print(f" ✗ {e}")
|
||||
|
||||
sys.exit(0 if compat['compatible'] else 1)
|
||||
|
||||
# ==================== 低端设备优化 ====================
|
||||
print("\n[设备检测与优化]")
|
||||
|
||||
# 检测架构
|
||||
arch = ArchitectureDetector.get_architecture()
|
||||
|
||||
# 应用低端设备配置
|
||||
preset = config.get('preset', 'auto')
|
||||
low_mem_config = LowMemoryConfig(preset)
|
||||
|
||||
# 获取设备信息
|
||||
device_info = low_mem_config.get_device_info()
|
||||
|
||||
# 显示设备信息
|
||||
if device_info['total_ram_mb'] > 0:
|
||||
print(f" [硬件配置]")
|
||||
print(f" - 总内存: {device_info['total_ram_mb']:.0f} MB")
|
||||
print(f" - 可用内存: {device_info['available_ram_mb']:.0f} MB ({100-device_info['percent_used']:.1f}% 可用)")
|
||||
print(f" - CPU核心: {device_info['cpu_count']} 核心")
|
||||
print(f" - 系统架构: {arch['machine']} ({arch['architecture']})")
|
||||
else:
|
||||
print(f" - 系统架构: {arch['machine']} ({arch['architecture']})")
|
||||
|
||||
print(f"\n [性能优化]")
|
||||
print(f" - 优化模式: {preset}")
|
||||
|
||||
# 应用配置
|
||||
if preset != 'auto':
|
||||
status = low_mem_config.get_status()
|
||||
print(f" - 预设方案: {status.get('description', preset)}")
|
||||
|
||||
config = low_mem_config.apply_to_config(config)
|
||||
|
||||
# 覆盖命令行参数
|
||||
if args.memory_limit:
|
||||
config['memory_limit'] = parse_size(args.memory_limit)
|
||||
if args.workers > 0:
|
||||
config['workers'] = args.workers
|
||||
if args.chunk_size:
|
||||
config['chunk_size'] = parse_size(args.chunk_size)
|
||||
if args.buffer_size:
|
||||
config['buffer_size'] = parse_size(args.buffer_size)
|
||||
|
||||
# 禁用可选功能
|
||||
if args.disable_ws:
|
||||
config['enable_ws'] = False
|
||||
if args.disable_sse:
|
||||
config['enable_sse'] = False
|
||||
if args.disable_hash:
|
||||
config['calculate_hash'] = False
|
||||
|
||||
# 启动内存管理器
|
||||
memory_manager = MemoryManager({
|
||||
'enabled': True,
|
||||
'memory_limit': config.get('memory_limit', 512 * 1024 * 1024),
|
||||
'gc_interval': config.get('gc_interval', 300), # 定时GC间隔
|
||||
'enable_scheduled_gc': config.get('enable_scheduled_gc', True)
|
||||
})
|
||||
memory_manager.start()
|
||||
config['_memory_manager'] = memory_manager
|
||||
|
||||
mem_status = memory_manager.get_status()
|
||||
print(f" - 内存限制: {mem_status['memory_limit_mb']} MB")
|
||||
print(f" - 工作进程: {config.get('workers', 1)}")
|
||||
print(f" - 传输块: {config.get('chunk_size', 128 * 1024) // 1024} KB")
|
||||
print(f" - GC间隔: {config.get('gc_interval', 300)}秒")
|
||||
print(f" - 缓存大小: {config.get('max_cache_size', 0) // (1024*1024)} MB")
|
||||
|
||||
# 设置信号处理
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
# 打印启动信息
|
||||
print()
|
||||
print("╔" + "═" * 58 + "╗")
|
||||
print("║" + " " * 22 + "HYC下载站 v2.3" + " " * 22 + "║")
|
||||
print("╚" + "═" * 58 + "╝")
|
||||
|
||||
print()
|
||||
print("【服务器配置】")
|
||||
print(f" ▶ 监听地址: {config.get('host')}:{config.get('port')}")
|
||||
print(f" ▶ 文件目录: {os.path.abspath(config.get('base_dir', './downloads'))}")
|
||||
print(f" ▶ API版本: {config.get('api_version', 'v2')}")
|
||||
print(f" ▶ 认证方式: {config.get('auth_type', 'none')}")
|
||||
max_upload = config.get('max_upload_size', '1G')
|
||||
if isinstance(max_upload, str):
|
||||
max_upload = parse_size(max_upload)
|
||||
else:
|
||||
max_upload = int(max_upload)
|
||||
print(f" ▶ 最大上传: {max_upload // (1024*1024)} MB")
|
||||
|
||||
# 数据库状态
|
||||
db_config = config.get('database', {})
|
||||
db_enabled = db_config.get('enabled', True)
|
||||
|
||||
print()
|
||||
print("【功能模块】")
|
||||
print(f" {'●' if config.get('enable_monitor') else '○'} 系统监控", end='')
|
||||
print(f" {'●' if config.get('enable_sync') else '○'} 镜像同步", end='')
|
||||
print(f" {'●' if config.get('enable_mirrors') else '○'} 加速源")
|
||||
print(f" {'●' if config.get('enable_ws') else '○'} WebSocket", end='')
|
||||
print(f" {'●' if config.get('enable_sse') else '○'} SSE", end='')
|
||||
print(f" {'●' if db_enabled else '○'} 数据库")
|
||||
|
||||
if db_enabled:
|
||||
print(f" └── 类型: {db_config.get('type', 'sqlite')} | 同步间隔: {db_config.get('sync_interval', 60)}s")
|
||||
|
||||
print()
|
||||
print("【性能参数】")
|
||||
print(f" ▶ 最大线程: {config.get('max_workers', 10)}")
|
||||
print(f" ▶ 工作进程: {config.get('workers', 1)}")
|
||||
print(f" ▶ 内存限制: {config.get('memory_limit', 512*1024*1024) // (1024*1024)} MB")
|
||||
print(f" ▶ 连接超时: {config.get('timeout', 30)}s")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
|
||||
# 初始化数据库
|
||||
db = None
|
||||
scheduler = None
|
||||
if db_enabled:
|
||||
print("\n[初始化数据库...]")
|
||||
try:
|
||||
db = init_database(config)
|
||||
|
||||
# 健康检查
|
||||
health = db.health_check()
|
||||
if health.get('healthy'):
|
||||
print(f" ✓ 数据库连接成功 ({db_config.get('type', 'sqlite')})")
|
||||
print(f" ✓ 表数量: {health.get('table_count', 0)}")
|
||||
else:
|
||||
print(f" ✗ 数据库健康检查失败: {health.get('error')}")
|
||||
|
||||
# 检查并更新Schema
|
||||
if db.check_schema_version():
|
||||
print(f" ✓ 数据库结构版本检查通过")
|
||||
|
||||
# 获取统计
|
||||
db_stats = db.get_stats()
|
||||
print(f"\n 数据库统计:")
|
||||
print(f" - 文件记录: {db_stats['total_files']}")
|
||||
print(f" - 同步记录: {db_stats['total_sync_records']}")
|
||||
print(f" - 缓存记录: {db_stats['total_cache_records']}")
|
||||
print(f" - 下载记录: {db_stats['total_download_records']}")
|
||||
|
||||
# 将数据库实例添加到配置中
|
||||
config['_db_instance'] = db
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n ✗ 数据库初始化失败: {e}")
|
||||
print(" ⚠ 服务器将继续运行，但不使用数据库功能")
|
||||
|
||||
# 初始化同步调度器（不使用数据库）
|
||||
if config.get('enable_sync'):
|
||||
_, scheduler, _ = init_database_sync(config)
|
||||
scheduler.start()
|
||||
print(" - 同步调度器已启动")
|
||||
|
||||
# ==================== 优雅关闭处理 ====================
|
||||
import atexit
|
||||
|
||||
def cleanup():
|
||||
"""服务器关闭时清理资源"""
|
||||
print("\n正在关闭服务器...")
|
||||
|
||||
# 关闭数据库连接池
|
||||
if db:
|
||||
try:
|
||||
db.engine.dispose()
|
||||
print("✓ 数据库连接已关闭")
|
||||
except Exception as e:
|
||||
print(f"✗ 关闭数据库连接时出错: {e}")
|
||||
|
||||
# 停止同步调度器
|
||||
if scheduler:
|
||||
try:
|
||||
scheduler.stop()
|
||||
print("✓ 同步调度器已停止")
|
||||
except Exception as e:
|
||||
print(f"✗ 停止同步调度器时出错: {e}")
|
||||
|
||||
print("服务器已关闭")
|
||||
|
||||
# 注册关闭处理函数
|
||||
atexit.register(cleanup)
|
||||
|
||||
# 设置服务器启动时间（用于计算运行时间）
|
||||
config['start_time'] = time.time()
|
||||
|
||||
# 创建并启动服务器
|
||||
try:
|
||||
server = MirrorServer(config)
|
||||
if server.start():
|
||||
server.serve_forever()
|
||||
else:
|
||||
print("服务器启动失败")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"错误: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,583 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
镜像源处理器包
|
||||
支持 Docker、APT、YUM、PyPI、npm、Go、Maven、Gradle 等镜像源代理
|
||||
"""
|
||||
|
||||
from .docker import DockerMirror
|
||||
from .apt import APTMirror
|
||||
from .yum import YUMMirror
|
||||
from .pypi import PyPIMirror
|
||||
from .npm import NpmMirror
|
||||
from .go import GoProxy
|
||||
from .http import HttpMirror
|
||||
|
||||
# 镜像处理器映射
|
||||
# 特殊类型使用专用处理器，其他使用通用HTTP处理器
|
||||
MIRROR_HANDLERS = {
|
||||
# 专用处理器
|
||||
'docker': DockerMirror,
|
||||
'apt': APTMirror,
|
||||
'yum': YUMMirror,
|
||||
'pypi': PyPIMirror,
|
||||
'npm': NpmMirror,
|
||||
'go': GoProxy,
|
||||
|
||||
# ======== 包管理器 ========
|
||||
# Python
|
||||
'pip': HttpMirror,
|
||||
'pipenv': HttpMirror,
|
||||
'poetry': HttpMirror,
|
||||
'conda': HttpMirror,
|
||||
'anaconda': HttpMirror,
|
||||
'bandersnatch': HttpMirror,
|
||||
'twine': HttpMirror,
|
||||
|
||||
# Node.js
|
||||
'yarn': HttpMirror,
|
||||
'pnpm': HttpMirror,
|
||||
'bower': HttpMirror,
|
||||
|
||||
# Java
|
||||
'maven': HttpMirror,
|
||||
'gradle': HttpMirror,
|
||||
|
||||
# .NET
|
||||
'nuget': HttpMirror,
|
||||
|
||||
# Ruby
|
||||
'gem': HttpMirror,
|
||||
'rubygems': HttpMirror,
|
||||
|
||||
# Rust
|
||||
'cargo': HttpMirror,
|
||||
'rustup': HttpMirror,
|
||||
|
||||
# PHP
|
||||
'composer': HttpMirror,
|
||||
'packagist': HttpMirror,
|
||||
|
||||
# Swift/Apple
|
||||
'cocoapods': HttpMirror,
|
||||
|
||||
# C/C++
|
||||
'conan': HttpMirror,
|
||||
'vcpkg': HttpMirror,
|
||||
|
||||
# Dart/Flutter
|
||||
'pub': HttpMirror,
|
||||
'dart': HttpMirror,
|
||||
'flutter': HttpMirror,
|
||||
|
||||
# Haskell
|
||||
'hackage': HttpMirror,
|
||||
'stackage': HttpMirror,
|
||||
|
||||
# OCaml
|
||||
'opam': HttpMirror,
|
||||
|
||||
# D语言
|
||||
'dub': HttpMirror,
|
||||
|
||||
# Nim
|
||||
'nimble': HttpMirror,
|
||||
|
||||
# V语言
|
||||
'v': HttpMirror,
|
||||
|
||||
# Julia
|
||||
'julia': HttpMirror,
|
||||
|
||||
# Lua
|
||||
'lua': HttpMirror,
|
||||
'luarocks': HttpMirror,
|
||||
|
||||
# Elm
|
||||
'elm': HttpMirror,
|
||||
|
||||
# Perl
|
||||
'cpan': HttpMirror,
|
||||
'cpanm': HttpMirror,
|
||||
|
||||
# R
|
||||
'cran': HttpMirror,
|
||||
|
||||
# LaTeX
|
||||
'ctan': HttpMirror,
|
||||
|
||||
# Haskell
|
||||
'haskell': HttpMirror,
|
||||
|
||||
# ======== 容器/云原生 ========
|
||||
'helm': HttpMirror,
|
||||
'kubernetes': HttpMirror,
|
||||
' Quay.io': HttpMirror,
|
||||
'quay': HttpMirror,
|
||||
'ghcr': HttpMirror,
|
||||
'gcr': HttpMirror,
|
||||
'harbor': HttpMirror,
|
||||
|
||||
# ======== 开发工具 ========
|
||||
'jetbrains': HttpMirror,
|
||||
'vscode': HttpMirror,
|
||||
'cuda': HttpMirror,
|
||||
|
||||
# ======== 语言运行时 ========
|
||||
'node': HttpMirror,
|
||||
'python': HttpMirror,
|
||||
'ruby': HttpMirror,
|
||||
'php': HttpMirror,
|
||||
'java': HttpMirror,
|
||||
'dotnet': HttpMirror,
|
||||
|
||||
# ======== Linux 发行版 ========
|
||||
'alpine': HttpMirror,
|
||||
'arch': HttpMirror,
|
||||
'aur': HttpMirror,
|
||||
'centos': HttpMirror,
|
||||
'debian': HttpMirror,
|
||||
'fedora': HttpMirror,
|
||||
'gentoo': HttpMirror,
|
||||
'opensuse': HttpMirror,
|
||||
'void': HttpMirror,
|
||||
'freebsd': HttpMirror,
|
||||
'netbsd': HttpMirror,
|
||||
'openbsd': HttpMirror,
|
||||
'rocky': HttpMirror,
|
||||
'alma': HttpMirror,
|
||||
'kali': HttpMirror,
|
||||
'ubuntu': HttpMirror,
|
||||
'mint': HttpMirror,
|
||||
'manjaro': HttpMirror,
|
||||
'slackware': HttpMirror,
|
||||
'mageia': HttpMirror,
|
||||
'openmandriva': HttpMirror,
|
||||
|
||||
# ======== 系统包管理器 ========
|
||||
'homebrew': HttpMirror,
|
||||
'brew': HttpMirror,
|
||||
'chocolatey': HttpMirror,
|
||||
'snap': HttpMirror,
|
||||
'flatpak': HttpMirror,
|
||||
'appimage': HttpMirror,
|
||||
'winget': HttpMirror,
|
||||
'scoop': HttpMirror,
|
||||
|
||||
# ======== 数据库/运维 ========
|
||||
'postgresql': HttpMirror,
|
||||
'mysql': HttpMirror,
|
||||
'mariadb': HttpMirror,
|
||||
'mongodb': HttpMirror,
|
||||
'redis': HttpMirror,
|
||||
'influxdata': HttpMirror,
|
||||
'grafana': HttpMirror,
|
||||
'prometheus': HttpMirror,
|
||||
'elastic': HttpMirror,
|
||||
'bitnami': HttpMirror,
|
||||
|
||||
# ======== 代码托管/源码 ========
|
||||
'git': HttpMirror,
|
||||
'github': HttpMirror,
|
||||
'gitlab': HttpMirror,
|
||||
'bitbucket': HttpMirror,
|
||||
'sourceforge': HttpMirror,
|
||||
|
||||
# ======== 其他 ========
|
||||
'pacman': HttpMirror,
|
||||
'nix': HttpMirror,
|
||||
'guix': HttpMirror,
|
||||
'termux': HttpMirror,
|
||||
'msys2': HttpMirror,
|
||||
'google-fonts': HttpMirror,
|
||||
'aurora': HttpMirror,
|
||||
'cloudflare': HttpMirror,
|
||||
'fastly': HttpMirror,
|
||||
|
||||
# ======== 自定义 ========
|
||||
'http': HttpMirror,
|
||||
'custom': HttpMirror,
|
||||
}
|
||||
|
||||
|
||||
def get_mirror_handler(mirror_type: str):
|
||||
"""获取镜像处理器类"""
|
||||
return MIRROR_HANDLERS.get(mirror_type)
|
||||
|
||||
|
||||
# 各镜像类型的默认上游URL
|
||||
DEFAULT_UPSTREAM_URLS = {
|
||||
# ======== 包管理器 ========
|
||||
# Python
|
||||
'pip': 'https://pypi.org/simple',
|
||||
'pipenv': 'https://pypi.org/simple',
|
||||
'poetry': 'https://pypi.org/simple',
|
||||
'conda': 'https://repo.anaconda.com',
|
||||
'anaconda': 'https://repo.anaconda.com',
|
||||
'bandersnatch': 'https://pypi.org/simple',
|
||||
'twine': 'https://pypi.org/simple',
|
||||
|
||||
# Node.js
|
||||
'yarn': 'https://registry.yarnpkg.com',
|
||||
'pnpm': 'https://registry.npmjs.org',
|
||||
'bower': 'https://registry.bower.io',
|
||||
|
||||
# Java
|
||||
'maven': 'https://repo1.maven.org/maven2',
|
||||
'gradle': 'https://services.gradle.org/distributions',
|
||||
|
||||
# .NET
|
||||
'nuget': 'https://api.nuget.org/v3',
|
||||
|
||||
# Ruby
|
||||
'gem': 'https://rubygems.org',
|
||||
'rubygems': 'https://rubygems.org',
|
||||
|
||||
# Rust
|
||||
'cargo': 'https://crates.io',
|
||||
'rustup': 'https://static.rust-lang.org',
|
||||
|
||||
# PHP
|
||||
'composer': 'https://repo.packagist.org',
|
||||
'packagist': 'https://repo.packagist.org',
|
||||
|
||||
# Swift/Apple
|
||||
'cocoapods': 'https://cdn.cocoapods.org',
|
||||
|
||||
# C/C++
|
||||
'conan': 'https://center.conan.io',
|
||||
'vcpkg': 'https://github.com/microsoft/vcpkg/archive/refs/heads/main.tar.gz',
|
||||
|
||||
# Dart/Flutter
|
||||
'pub': 'https://pub.dev',
|
||||
'dart': 'https://storage.googleapis.com/flutter_infra_release',
|
||||
'flutter': 'https://storage.flutter-io.cn',
|
||||
|
||||
# Haskell
|
||||
'hackage': 'https://hackage.haskell.org',
|
||||
'stackage': 'https://www.stackage.org',
|
||||
|
||||
# OCaml
|
||||
'opam': 'https://opam.ocaml.org',
|
||||
|
||||
# D语言
|
||||
'dub': 'https://code.dlang.org',
|
||||
|
||||
# Nim
|
||||
'nimble': 'https://nimble.directory',
|
||||
|
||||
# V语言
|
||||
'v': 'https://vpm.itsgriffin.com',
|
||||
|
||||
# Julia
|
||||
'julia': 'https://pkg.julialang.org',
|
||||
|
||||
# Lua
|
||||
'lua': 'https://luarocks.org',
|
||||
'luarocks': 'https://luarocks.org',
|
||||
|
||||
# Elm
|
||||
'elm': 'https://package.elm-lang.org',
|
||||
|
||||
# Perl
|
||||
'cpan': 'https://www.cpan.org',
|
||||
'cpanm': 'https://www.cpan.org',
|
||||
|
||||
# R
|
||||
'cran': 'https://cran.r-project.org',
|
||||
|
||||
# LaTeX
|
||||
'ctan': 'https://ctan.math.illinois.edu',
|
||||
|
||||
# Haskell
|
||||
'haskell': 'https://haskell.org',
|
||||
|
||||
# ======== 容器/云原生 ========
|
||||
'helm': 'https://charts.helm.sh',
|
||||
'kubernetes': 'https://dl.k8s.io/release',
|
||||
'quay': 'https://quay.io',
|
||||
'quay.io': 'https://quay.io',
|
||||
'ghcr': 'https://ghcr.io',
|
||||
'gcr': 'https://gcr.io',
|
||||
'harbor': 'https://harbor.io',
|
||||
|
||||
# ======== 开发工具 ========
|
||||
'jetbrains': 'https://www.jetbrains.com',
|
||||
'vscode': 'https://code.visualstudio.com',
|
||||
'cuda': 'https://developer.download.nvidia.com/compute/cuda/repos',
|
||||
|
||||
# ======== 语言运行时 ========
|
||||
'node': 'https://nodejs.org/dist',
|
||||
'python': 'https://www.python.org/ftp/python',
|
||||
'ruby': 'https://www.ruby-lang.org',
|
||||
'php': 'https://www.php.net/distributions',
|
||||
'java': 'https://download.oracle.com/java',
|
||||
'dotnet': 'https://dotnetcli.azureedge.net',
|
||||
|
||||
# ======== Linux 发行版 ========
|
||||
'alpine': 'https://dl-cdn.alpinelinux.org',
|
||||
'arch': 'https://mirror.archlinux.org',
|
||||
'aur': 'https://aur.archlinux.org',
|
||||
'centos': 'https://mirrors.aliyun.com/centos',
|
||||
'debian': 'https://deb.debian.org/debian',
|
||||
'fedora': 'https://mirrors.fedoraproject.org',
|
||||
'gentoo': 'https://distfiles.gentoo.org',
|
||||
'opensuse': 'https://download.opensuse.org',
|
||||
'void': 'https://repo.voidlinux.org',
|
||||
'freebsd': 'https://pkg.freebsd.org',
|
||||
'netbsd': 'https://cdn.netbsd.org',
|
||||
'openbsd': 'https://cdn.openbsd.org',
|
||||
'rocky': 'https://download.rockylinux.org',
|
||||
'alma': 'https://repo.almalinux.org',
|
||||
'kali': 'http://.kali.org',
|
||||
'ubuntu': 'https://releases.ubuntu.com',
|
||||
'mint': 'https://packages.linuxmint.com',
|
||||
'manjaro': 'https://repo.manjaro.org',
|
||||
'slackware': 'https://mirrors.slackware.com',
|
||||
'mageia': 'https://www.mageia.org',
|
||||
'openmandriva': 'https://download.openmandriva.org',
|
||||
|
||||
# ======== 系统包管理器 ========
|
||||
'homebrew': 'https://github.com/Homebrew/brew',
|
||||
'brew': 'https://github.com/Homebrew/brew',
|
||||
'chocolatey': 'https://chocolatey.org/api/v2',
|
||||
'snap': 'https://snapcraft.io',
|
||||
'flatpak': 'https://flathub.org',
|
||||
'appimage': 'https://github.com/AppImage/AppImageHub/releases',
|
||||
'winget': 'https://github.com/microsoft/winget-pkgs',
|
||||
'scoop': 'https://scoop.sh',
|
||||
|
||||
# ======== 数据库/运维 ========
|
||||
'postgresql': 'https://www.postgresql.org',
|
||||
'mysql': 'https://dev.mysql.com',
|
||||
'mariadb': 'https://mariadb.org',
|
||||
'mongodb': 'https://www.mongodb.org',
|
||||
'redis': 'https://redis.io',
|
||||
'influxdata': 'https://portal.influxdata.com',
|
||||
'grafana': 'https://grafana.com',
|
||||
'prometheus': 'https://prometheus.io',
|
||||
'elastic': 'https://www.elastic.co',
|
||||
'bitnami': 'https://bitnami.com',
|
||||
|
||||
# ======== 代码托管/源码 ========
|
||||
'git': 'https://github.com',
|
||||
'github': 'https://github.com',
|
||||
'gitlab': 'https://gitlab.com',
|
||||
'bitbucket': 'https://bitbucket.org',
|
||||
'sourceforge': 'https://sourceforge.net',
|
||||
|
||||
# ======== 其他 ========
|
||||
'pacman': 'https://mirror.archlinux.org',
|
||||
'nix': 'https://nix-community.org',
|
||||
'guix': 'https://guix.gnu.org',
|
||||
'termux': 'https://termux.net',
|
||||
'msys2': 'https://repo.msys2.org',
|
||||
'google-fonts': 'https://fonts.google.com',
|
||||
'aurora': 'https://auroralinux.org',
|
||||
'cloudflare': 'https://cloudflare.com',
|
||||
'fastly': 'https://fastly.com',
|
||||
}
|
||||
|
||||
|
||||
def get_default_upstream(mirror_type: str) -> str:
|
||||
"""获取镜像类型的默认上游URL"""
|
||||
return DEFAULT_UPSTREAM_URLS.get(mirror_type, 'https://mirror.example.com')
|
||||
|
||||
|
||||
def list_available_mirrors() -> list:
|
||||
"""列出可用的镜像类型"""
|
||||
return [
|
||||
# ======== 专用处理器 ========
|
||||
{
|
||||
'type': 'docker',
|
||||
'name': 'Docker Registry',
|
||||
'description': 'Docker镜像代理'
|
||||
},
|
||||
{
|
||||
'type': 'apt',
|
||||
'name': 'APT (Debian/Ubuntu)',
|
||||
'description': 'APT软件源代理'
|
||||
},
|
||||
{
|
||||
'type': 'yum',
|
||||
'name': 'YUM/DNF (RHEL/CentOS)',
|
||||
'description': 'YUM/DNF软件源代理'
|
||||
},
|
||||
{
|
||||
'type': 'pypi',
|
||||
'name': 'PyPI',
|
||||
'description': 'Python包索引代理'
|
||||
},
|
||||
{
|
||||
'type': 'npm',
|
||||
'name': 'npm Registry',
|
||||
'description': 'Node.js包管理器代理'
|
||||
},
|
||||
{
|
||||
'type': 'go',
|
||||
'name': 'Go Modules',
|
||||
'description': 'Go模块代理'
|
||||
},
|
||||
|
||||
# ======== 包管理器 ========
|
||||
# Python
|
||||
{'type': 'pip', 'name': 'pip', 'description': 'Python pip包管理器'},
|
||||
{'type': 'pipenv', 'name': 'Pipenv', 'description': 'Python Pipenv代理'},
|
||||
{'type': 'poetry', 'name': 'Poetry', 'description': 'Python Poetry包管理器'},
|
||||
{'type': 'conda', 'name': 'Conda', 'description': 'Python Conda包管理器'},
|
||||
{'type': 'anaconda', 'name': 'Anaconda', 'description': 'Anaconda Python发行版'},
|
||||
|
||||
# Node.js
|
||||
{'type': 'yarn', 'name': 'Yarn', 'description': 'Node.js Yarn包管理器'},
|
||||
{'type': 'pnpm', 'name': 'pnpm', 'description': 'Node.js pnpm包管理器'},
|
||||
{'type': 'bower', 'name': 'Bower', 'description': '前端包管理器'},
|
||||
|
||||
# Java
|
||||
{'type': 'maven', 'name': 'Maven Central', 'description': 'Java/Maven包管理器'},
|
||||
{'type': 'gradle', 'name': 'Gradle', 'description': 'Gradle构建工具分发'},
|
||||
|
||||
# .NET
|
||||
{'type': 'nuget', 'name': 'NuGet', 'description': '.NET包管理器'},
|
||||
|
||||
# Ruby
|
||||
{'type': 'gem', 'name': 'RubyGems', 'description': 'Ruby包管理器'},
|
||||
{'type': 'rubygems', 'name': 'RubyGems', 'description': 'Ruby官方仓库'},
|
||||
|
||||
# Rust
|
||||
{'type': 'cargo', 'name': 'Crates.io', 'description': 'Rust语言包管理器'},
|
||||
{'type': 'rustup', 'name': 'Rustup', 'description': 'Rust工具链'},
|
||||
|
||||
# PHP
|
||||
{'type': 'composer', 'name': 'Composer', 'description': 'PHP包管理器'},
|
||||
{'type': 'packagist', 'name': 'Packagist', 'description': 'PHP官方包仓库'},
|
||||
|
||||
# Swift/Apple
|
||||
{'type': 'cocoapods', 'name': 'CocoaPods', 'description': 'iOS/macOS包管理器'},
|
||||
|
||||
# C/C++
|
||||
{'type': 'conan', 'name': 'Conan', 'description': 'C/C++包管理器'},
|
||||
{'type': 'vcpkg', 'name': 'vcpkg', 'description': 'C/C++包管理器'},
|
||||
|
||||
# Dart/Flutter
|
||||
{'type': 'pub', 'name': 'Pub.dev', 'description': 'Dart/Flutter包管理器'},
|
||||
{'type': 'dart', 'name': 'Dart SDK', 'description': 'Dart SDK'},
|
||||
{'type': 'flutter', 'name': 'Flutter', 'description': 'Flutter SDK'},
|
||||
|
||||
# Haskell
|
||||
{'type': 'hackage', 'name': 'Hackage', 'description': 'Haskell包管理器'},
|
||||
{'type': 'stackage', 'name': 'Stackage', 'description': 'Haskell Stackage'},
|
||||
|
||||
# OCaml
|
||||
{'type': 'opam', 'name': 'OPAM', 'description': 'OCaml包管理器'},
|
||||
|
||||
# D语言
|
||||
{'type': 'dub', 'name': 'Dub', 'description': 'D语言包管理器'},
|
||||
|
||||
# Nim
|
||||
{'type': 'nimble', 'name': 'Nimble', 'description': 'Nim语言包管理器'},
|
||||
|
||||
# V语言
|
||||
{'type': 'v', 'name': 'V PM', 'description': 'V语言包管理器'},
|
||||
|
||||
# Julia
|
||||
{'type': 'julia', 'name': 'Julia Packages', 'description': 'Julia语言包管理器'},
|
||||
|
||||
# Lua
|
||||
{'type': 'lua', 'name': 'Lua', 'description': 'Lua语言'},
|
||||
{'type': 'luarocks', 'name': 'LuaRocks', 'description': 'Lua包管理器'},
|
||||
|
||||
# Elm
|
||||
{'type': 'elm', 'name': 'Elm Packages', 'description': 'Elm语言包管理器'},
|
||||
|
||||
# Perl
|
||||
{'type': 'cpan', 'name': 'CPAN', 'description': 'Perl包管理器'},
|
||||
{'type': 'cpanm', 'name': 'cpanm', 'description': 'Perl cpanminus'},
|
||||
|
||||
# R
|
||||
{'type': 'cran', 'name': 'CRAN', 'description': 'R语言包仓库'},
|
||||
|
||||
# LaTeX
|
||||
{'type': 'ctan', 'name': 'CTAN', 'description': 'LaTeX包管理器'},
|
||||
|
||||
# ======== 容器/云原生 ========
|
||||
{'type': 'helm', 'name': 'Helm Charts', 'description': 'Kubernetes Helm包'},
|
||||
{'type': 'kubernetes', 'name': 'Kubernetes', 'description': 'Kubernetes发行版'},
|
||||
{'type': 'quay', 'name': 'Quay.io', 'description': 'Quay容器仓库'},
|
||||
{'type': 'ghcr', 'name': 'GitHub Container Registry', 'description': 'GitHub容器仓库'},
|
||||
{'type': 'gcr', 'name': 'Google Container Registry', 'description': 'Google容器仓库'},
|
||||
{'type': 'harbor', 'name': 'Harbor', 'description': 'Harbor容器仓库'},
|
||||
|
||||
# ======== 开发工具 ========
|
||||
{'type': 'jetbrains', 'name': 'JetBrains', 'description': 'JetBrains IDE'},
|
||||
{'type': 'vscode', 'name': 'VS Code', 'description': 'Visual Studio Code'},
|
||||
{'type': 'cuda', 'name': 'CUDA', 'description': 'NVIDIA CUDA工具包'},
|
||||
|
||||
# ======== 语言运行时 ========
|
||||
{'type': 'node', 'name': 'Node.js', 'description': 'Node.js官方'},
|
||||
{'type': 'python', 'name': 'Python', 'description': 'Python官方'},
|
||||
{'type': 'ruby', 'name': 'Ruby', 'description': 'Ruby官方'},
|
||||
{'type': 'php', 'name': 'PHP', 'description': 'PHP官方'},
|
||||
{'type': 'java', 'name': 'Java JDK', 'description': 'Oracle/OpenJDK'},
|
||||
{'type': 'dotnet', 'name': '.NET SDK', 'description': '.NET SDK'},
|
||||
|
||||
# ======== Linux 发行版 ========
|
||||
{'type': 'alpine', 'name': 'Alpine Linux', 'description': 'Alpine Linux'},
|
||||
{'type': 'arch', 'name': 'Arch Linux', 'description': 'Arch Linux'},
|
||||
{'type': 'aur', 'name': 'AUR', 'description': 'Arch Linux AUR'},
|
||||
{'type': 'centos', 'name': 'CentOS', 'description': 'CentOS'},
|
||||
{'type': 'debian', 'name': 'Debian', 'description': 'Debian'},
|
||||
{'type': 'fedora', 'name': 'Fedora', 'description': 'Fedora'},
|
||||
{'type': 'gentoo', 'name': 'Gentoo', 'description': 'Gentoo'},
|
||||
{'type': 'opensuse', 'name': 'openSUSE', 'description': 'openSUSE'},
|
||||
{'type': 'void', 'name': 'Void Linux', 'description': 'Void Linux'},
|
||||
{'type': 'freebsd', 'name': 'FreeBSD', 'description': 'FreeBSD'},
|
||||
{'type': 'netbsd', 'name': 'NetBSD', 'description': 'NetBSD'},
|
||||
{'type': 'openbsd', 'name': 'OpenBSD', 'description': 'OpenBSD'},
|
||||
{'type': 'rocky', 'name': 'Rocky Linux', 'description': 'Rocky Linux'},
|
||||
{'type': 'alma', 'name': 'AlmaLinux', 'description': 'AlmaLinux'},
|
||||
{'type': 'kali', 'name': 'Kali Linux', 'description': 'Kali Linux'},
|
||||
{'type': 'ubuntu', 'name': 'Ubuntu', 'description': 'Ubuntu'},
|
||||
{'type': 'mint', 'name': 'Linux Mint', 'description': 'Linux Mint'},
|
||||
{'type': 'manjaro', 'name': 'Manjaro', 'description': 'Manjaro Linux'},
|
||||
{'type': 'slackware', 'name': 'Slackware', 'description': 'Slackware'},
|
||||
{'type': 'mageia', 'name': 'Mageia', 'description': 'Mageia Linux'},
|
||||
{'type': 'openmandriva', 'name': 'OpenMandriva', 'description': 'OpenMandriva'},
|
||||
|
||||
# ======== 系统包管理器 ========
|
||||
{'type': 'homebrew', 'name': 'Homebrew', 'description': 'macOS Homebrew'},
|
||||
{'type': 'brew', 'name': 'Brew', 'description': 'Homebrew'},
|
||||
{'type': 'chocolatey', 'name': 'Chocolatey', 'description': 'Windows Chocolatey'},
|
||||
{'type': 'snap', 'name': 'Snap Store', 'description': 'Snap应用商店'},
|
||||
{'type': 'flatpak', 'name': 'Flatpak', 'description': 'Flatpak应用'},
|
||||
{'type': 'appimage', 'name': 'AppImage', 'description': 'AppImage应用'},
|
||||
{'type': 'winget', 'name': 'winget', 'description': 'Windows winget'},
|
||||
{'type': 'scoop', 'name': 'Scoop', 'description': 'Windows Scoop'},
|
||||
|
||||
# ======== 数据库/运维 ========
|
||||
{'type': 'postgresql', 'name': 'PostgreSQL', 'description': 'PostgreSQL数据库'},
|
||||
{'type': 'mysql', 'name': 'MySQL', 'description': 'MySQL数据库'},
|
||||
{'type': 'mariadb', 'name': 'MariaDB', 'description': 'MariaDB数据库'},
|
||||
{'type': 'mongodb', 'name': 'MongoDB', 'description': 'MongoDB数据库'},
|
||||
{'type': 'redis', 'name': 'Redis', 'description': 'Redis数据库'},
|
||||
{'type': 'influxdata', 'name': 'InfluxData', 'description': 'InfluxDB时序数据库'},
|
||||
{'type': 'grafana', 'name': 'Grafana', 'description': 'Grafana可视化'},
|
||||
{'type': 'prometheus', 'name': 'Prometheus', 'description': 'Prometheus监控'},
|
||||
{'type': 'elastic', 'name': 'Elastic', 'description': 'Elasticsearch'},
|
||||
|
||||
# ======== 代码托管 ========
|
||||
{'type': 'github', 'name': 'GitHub', 'description': 'GitHub代码托管'},
|
||||
{'type': 'gitlab', 'name': 'GitLab', 'description': 'GitLab代码托管'},
|
||||
{'type': 'bitbucket', 'name': 'Bitbucket', 'description': 'Bitbucket代码托管'},
|
||||
{'type': 'sourceforge', 'name': 'SourceForge', 'description': 'SourceForge开源托管'},
|
||||
|
||||
# ======== 其他 ========
|
||||
{'type': 'nix', 'name': 'Nix/NixOS', 'description': 'Nix包管理器'},
|
||||
{'type': 'guix', 'name': 'Guix', 'description': 'Guix包管理器'},
|
||||
{'type': 'termux', 'name': 'Termux', 'description': 'Termux包管理器'},
|
||||
{'type': 'msys2', 'name': 'MSYS2', 'description': 'MSYS2包管理器'},
|
||||
{'type': 'google-fonts', 'name': 'Google Fonts', 'description': 'Google字体库'},
|
||||
|
||||
# ======== 自定义 ========
|
||||
{'type': 'custom', 'name': '自定义HTTP', 'description': '自定义HTTP镜像源'}
|
||||
]
|
||||
+432
@@ -0,0 +1,432 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
APT镜像代理处理器
|
||||
支持Debian/Ubuntu软件源
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import gzip
|
||||
import re
|
||||
import urllib.request
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class APTMirror:
|
||||
"""APT镜像代理"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
|
||||
# 配置 - 使用 storage_dir（基于 base_dir）
|
||||
self.mirrors = config.get('mirrors', [
|
||||
'http://archive.ubuntu.com/ubuntu',
|
||||
'http://security.ubuntu.com/ubuntu'
|
||||
])
|
||||
self.storage_dir = config.get('storage_dir', './downloads/apt')
|
||||
self.base_dir = config.get('base_dir', './downloads')
|
||||
self.default_suite = config.get('suite', 'jammy')
|
||||
self.default_components = config.get('components', ['main', 'restricted', 'universe', 'multiverse'])
|
||||
self.default_arch = config.get('arch', 'amd64')
|
||||
|
||||
# 确保存储目录存在
|
||||
os.makedirs(self.storage_dir, exist_ok=True)
|
||||
|
||||
def handle_request(self, handler, path: str) -> bool:
|
||||
"""
|
||||
处理APT请求
|
||||
路径格式: /ubuntu/dists/jammy/main/binary-amd64/Packages.gz
|
||||
"""
|
||||
try:
|
||||
# 解析路径
|
||||
parts = path.strip('/').split('/')
|
||||
|
||||
if len(parts) < 5:
|
||||
# 返回镜像列表或帮助信息
|
||||
return self._handle_index(handler)
|
||||
|
||||
# 提取组件
|
||||
distro = parts[0] # ubuntu, debian 等
|
||||
dist_type = parts[1] # dists
|
||||
suite = parts[2] # jammy, focal 等
|
||||
component = parts[3] # main, updates 等
|
||||
rest = '/'.join(parts[4:])
|
||||
|
||||
# 确定请求类型
|
||||
if rest.endswith('Packages.gz'):
|
||||
return self._handle_packages(handler, distro, suite, component, rest)
|
||||
elif rest.endswith('Packages'):
|
||||
return self._handle_packages_uncompressed(handler, distro, suite, component, rest)
|
||||
elif rest.endswith('Release'):
|
||||
return self._handle_release(handler, distro, suite, component, rest)
|
||||
elif rest.endswith('Release.gpg'):
|
||||
return self._handle_release_gpg(handler, distro, suite, component, rest)
|
||||
elif rest.endswith('InRelease'):
|
||||
return self._handle_inrelease(handler, distro, suite, component, rest)
|
||||
else:
|
||||
# 其他文件（源码包等）
|
||||
return self._handle_file(handler, distro, suite, rest)
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, str(e))
|
||||
return False
|
||||
|
||||
def _handle_index(self, handler) -> bool:
|
||||
"""处理索引请求"""
|
||||
handler.send_json_response({
|
||||
'mirrors': self.mirrors,
|
||||
'default_suite': self.default_suite,
|
||||
'default_components': self.default_components,
|
||||
'cache_stats': self.get_cache_stats()
|
||||
})
|
||||
return True
|
||||
|
||||
def _handle_packages(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
|
||||
"""处理Packages.gz请求"""
|
||||
cache_key = f"packages:{distro}:{suite}:{component}:{self.default_arch}"
|
||||
|
||||
# 检查架构
|
||||
if 'binary-' in path:
|
||||
arch = path.split('binary-')[1].split('/')[0]
|
||||
else:
|
||||
arch = self.default_arch
|
||||
|
||||
cache_key = f"packages:{distro}:{suite}:{component}:{arch}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/x-gzip')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取
|
||||
for mirror in self.mirrors:
|
||||
url = f"{mirror}/{path}"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
if data:
|
||||
# 缓存
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/x-gzip')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
continue
|
||||
|
||||
handler.send_error(502, "Failed to fetch from all mirrors")
|
||||
return False
|
||||
|
||||
def _handle_packages_uncompressed(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
|
||||
"""处理未压缩的Packages文件"""
|
||||
# 先获取gz版本
|
||||
gz_path = path + '.gz'
|
||||
|
||||
for mirror in self.mirrors:
|
||||
url = f"{mirror}/{gz_path}"
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
if data:
|
||||
# 解压
|
||||
packages_data = gzip.decompress(data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain')
|
||||
handler.send_header('Content-Length', str(len(packages_data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(packages_data)
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
handler.send_error(502, "Failed to fetch packages")
|
||||
return False
|
||||
|
||||
def _handle_release(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
|
||||
"""处理Release文件"""
|
||||
cache_key = f"release:{distro}:{suite}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 获取Release文件
|
||||
release_path = f"/{distro}/dists/{suite}/Release"
|
||||
|
||||
for mirror in self.mirrors:
|
||||
url = mirror + release_path
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
if data:
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
handler.send_error(502, "Failed to fetch Release")
|
||||
return False
|
||||
|
||||
def _handle_release_gpg(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
|
||||
"""处理Release.gpg文件"""
|
||||
cache_key = f"release_gpg:{distro}:{suite}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/pgp-signature')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 尝试获取
|
||||
gpg_path = f"/{distro}/dists/{suite}/Release.gpg"
|
||||
|
||||
for mirror in self.mirrors:
|
||||
url = mirror + gpg_path
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
if data:
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/pgp-signature')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
handler.send_error(404, "Release.gpg not found")
|
||||
return False
|
||||
|
||||
def _handle_inrelease(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
|
||||
"""处理InRelease文件 - 获取或生成签名后的Release信息"""
|
||||
cache_key = f"inrelease:{distro}:{suite}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 尝试从上游获取 InRelease
|
||||
inrelease_path = f"/{distro}/dists/{suite}/InRelease"
|
||||
|
||||
for mirror in self.mirrors:
|
||||
url = mirror + inrelease_path
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
if data:
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# 如果没有 InRelease，尝试生成一个（基于 Release + 方括号注释）
|
||||
# 注意：这不是有效的签名，但可以用于不验证签名的客户端
|
||||
release_cache_key = f"release:{distro}:{suite}"
|
||||
release_data = self._get_cache(release_cache_key)
|
||||
|
||||
if not release_data:
|
||||
# 尝试获取 Release
|
||||
release_path = f"/{distro}/dists/{suite}/Release"
|
||||
for mirror in self.mirrors:
|
||||
url = mirror + release_path
|
||||
try:
|
||||
release_data = self._fetch(url)
|
||||
if release_data:
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if release_data:
|
||||
# 添加注释说明这是未签名的 Release
|
||||
comment = f"# Note: This is a synthesized InRelease (original InRelease not available)\n"
|
||||
inrelease_data = comment + release_data.decode('utf-8', errors='replace')
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, inrelease_data.encode())
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain')
|
||||
handler.send_header('Content-Length', str(len(inrelease_data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(inrelease_data.encode())
|
||||
return True
|
||||
|
||||
handler.send_error(502, "Failed to fetch InRelease")
|
||||
return False
|
||||
|
||||
def _handle_file(self, handler, distro: str, suite: str, path: str) -> bool:
|
||||
"""处理普通文件请求（如源码包）"""
|
||||
cache_key = f"file:{distro}:{path.replace('/', ':')}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取
|
||||
for mirror in self.mirrors:
|
||||
url = f"{mirror}/{path}"
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
if data:
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
handler.send_error(404, "File not found")
|
||||
return False
|
||||
|
||||
def _fetch(self, url: str) -> Optional[bytes]:
|
||||
"""从URL获取数据"""
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('User-Agent', 'APT-Mirror/1.0')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return response.read()
|
||||
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _get_cache(self, cache_key: str) -> Optional[bytes]:
|
||||
"""获取缓存"""
|
||||
if not self.cache_enabled:
|
||||
return None
|
||||
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
if not os.path.exists(cache_path):
|
||||
return None
|
||||
|
||||
# 检查过期
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f)
|
||||
if time.time() > meta.get('expires', 0):
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
with open(cache_path, 'rb') as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _set_cache(self, cache_key: str, data: bytes):
|
||||
"""设置缓存"""
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
||||
|
||||
try:
|
||||
with open(cache_path, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
meta = {
|
||||
'cached_at': time.time(),
|
||||
'expires': time.time() + self.cache_ttl,
|
||||
'size': len(data)
|
||||
}
|
||||
|
||||
with open(meta_path, 'w') as f:
|
||||
json.dump(meta, f)
|
||||
|
||||
except Exception as e:
|
||||
print(f"APT缓存写入失败: {e}")
|
||||
|
||||
def _get_cache_path(self, cache_key: str) -> str:
|
||||
"""获取缓存路径"""
|
||||
subdir = cache_key[:2]
|
||||
return os.path.join(self.storage_dir, subdir, cache_key)
|
||||
|
||||
def get_cache_stats(self) -> dict:
|
||||
"""获取缓存统计"""
|
||||
if not os.path.exists(self.storage_dir):
|
||||
return {'files': 0, 'size': 0}
|
||||
|
||||
total_size = 0
|
||||
file_count = 0
|
||||
|
||||
for root, dirs, files in os.walk(self.storage_dir):
|
||||
for f in files:
|
||||
if not f.endswith('.meta'):
|
||||
file_count += 1
|
||||
total_size += os.path.getsize(os.path.join(root, f))
|
||||
|
||||
return {
|
||||
'files': file_count,
|
||||
'size': total_size,
|
||||
'size_formatted': self._format_size(total_size)
|
||||
}
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
"""格式化文件大小"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
units = ["B", "KB", "MB", "GB"]
|
||||
i = 0
|
||||
while size_bytes >= 1024 and i < len(units) - 1:
|
||||
size_bytes /= 1024.0
|
||||
i += 1
|
||||
|
||||
return f"{size_bytes:.2f} {units[i]}"
|
||||
@@ -0,0 +1,359 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
Docker镜像代理处理器
|
||||
支持Docker Registry API v2
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
import urllib.request
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class DockerMirror:
|
||||
"""Docker镜像代理"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
|
||||
# 配置 - 使用 storage_dir（基于 base_dir）
|
||||
self.registry_url = config.get('registry_url', 'https://registry-1.docker.io')
|
||||
self.mirror_url = config.get('mirror_url', '')
|
||||
self.storage_dir = config.get('storage_dir', './downloads/docker')
|
||||
self.base_dir = config.get('base_dir', './downloads')
|
||||
|
||||
# 认证（可选）
|
||||
self.username = config.get('username')
|
||||
self.password = config.get('password')
|
||||
|
||||
# 确保存储目录存在
|
||||
os.makedirs(self.storage_dir, exist_ok=True)
|
||||
|
||||
def handle_request(self, handler, path: str) -> bool:
|
||||
"""
|
||||
处理Docker镜像请求
|
||||
路径格式: /v2/library/ubuntu/tags/list 或 /v2/library/ubuntu/manifests/latest
|
||||
"""
|
||||
try:
|
||||
# 解析路径
|
||||
parts = path.strip('/').split('/')
|
||||
|
||||
if len(parts) < 2 or parts[0] != 'v2':
|
||||
handler.send_error(400, "Invalid Docker API path")
|
||||
return False
|
||||
|
||||
# 提取组件
|
||||
if parts[1] == 'library':
|
||||
# 官方镜像
|
||||
image = 'library/' + '/'.join(parts[2:-2]) if len(parts) > 4 else 'library/' + parts[2]
|
||||
action = parts[-2] # tags 或 manifests
|
||||
reference = parts[-1]
|
||||
else:
|
||||
# 非官方镜像
|
||||
image = '/'.join(parts[1:-2])
|
||||
action = parts[-2]
|
||||
reference = parts[-1]
|
||||
|
||||
# 根据操作类型处理
|
||||
if action == 'tags' and reference == 'list':
|
||||
return self._handle_tag_list(handler, image.rstrip('/tags'))
|
||||
elif action == 'manifests':
|
||||
return self._handle_manifest(handler, image, reference)
|
||||
elif action == 'blobs':
|
||||
return self._handle_blob(handler, image, reference)
|
||||
elif action == 'token':
|
||||
return self._handle_token(handler)
|
||||
else:
|
||||
handler.send_error(404, "Unknown action")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, str(e))
|
||||
return False
|
||||
|
||||
def _handle_tag_list(self, handler, image: str) -> bool:
|
||||
"""处理标签列表请求"""
|
||||
cache_key = f"tags:{image}"
|
||||
cached = self._get_cache(handler, cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/json')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.registry_url}/v2/{image}/tags/list"
|
||||
|
||||
try:
|
||||
data = self._fetch_from_upstream(url)
|
||||
|
||||
# 缓存
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/json')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, f"Failed to fetch tags: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_manifest(self, handler, image: str, reference: str) -> bool:
|
||||
"""处理清单请求"""
|
||||
cache_key = f"manifest:{image}:{reference}"
|
||||
cached = self._get_cache(handler, cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/vnd.docker.distribution.manifest.v2+json')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.send_header('Docker-Content-Digest', f"sha256:{hashlib.sha256(cached).hexdigest()}")
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.registry_url}/v2/{image}/manifests/{reference}"
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('Accept', 'application/vnd.docker.distribution.manifest.v2+json')
|
||||
|
||||
if self.username and self.password:
|
||||
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
|
||||
req.add_header('Authorization', f"Basic {auth}")
|
||||
|
||||
with urllib.request.urlopen(req) as response:
|
||||
data = response.read()
|
||||
|
||||
# 缓存
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
digest = f"sha256:{hashlib.sha256(data).hexdigest()}"
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/vnd.docker.distribution.manifest.v2+json')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.send_header('Docker-Content-Digest', digest)
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, f"Failed to fetch manifest: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_blob(self, handler, image: str, digest: str) -> bool:
|
||||
"""处理Blob层下载"""
|
||||
# 移除 sha256: 前缀
|
||||
if digest.startswith('sha256:'):
|
||||
digest = digest[7:]
|
||||
|
||||
cache_key = f"blob:{digest}"
|
||||
cached = self._get_cache(handler, cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.send_header('Docker-Content-Digest', f"sha256:{digest}")
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.registry_url}/v2/{image}/blobs/sha256:{digest}"
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
|
||||
if self.username and self.password:
|
||||
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
|
||||
req.add_header('Authorization', f"Basic {auth}")
|
||||
|
||||
with urllib.request.urlopen(req) as response:
|
||||
data = response.read()
|
||||
|
||||
# 缓存
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.send_header('Docker-Content-Digest', f"sha256:{digest}")
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, f"Failed to fetch blob: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_token(self, handler) -> bool:
|
||||
"""处理Token请求 - 生成真实的访问令牌"""
|
||||
# 解析认证信息
|
||||
auth_header = handler.headers.get('Authorization', '')
|
||||
username = None
|
||||
password = None
|
||||
|
||||
if auth_header.startswith('Basic '):
|
||||
try:
|
||||
decoded = base64.b64decode(auth_header[6:]).decode('utf-8')
|
||||
username, password = decoded.split(':', 1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 验证凭据（如果有）
|
||||
if self.username and self.password:
|
||||
if username != self.username or password != self.password:
|
||||
handler.send_error(401, "Invalid credentials")
|
||||
return False
|
||||
|
||||
# 生成唯一的访问令牌
|
||||
token_id = str(uuid.uuid4())
|
||||
issued_at = int(time.time())
|
||||
expires_in = 300 # 5分钟
|
||||
expires_at = issued_at + expires_in
|
||||
|
||||
# 创建令牌信息（简化版 JWT 结构）
|
||||
token_data = {
|
||||
"iss": "hyc-mirror",
|
||||
"sub": username or "anonymous",
|
||||
"aud": self.registry_url,
|
||||
"iat": issued_at,
|
||||
"exp": expires_at,
|
||||
"access": [
|
||||
{"type": "repository", "actions": ["pull"]},
|
||||
{"type": "registry", "actions": ["catalog"]}
|
||||
]
|
||||
}
|
||||
|
||||
# 使用 HMAC-SHA256 对令牌进行简单签名
|
||||
secret_key = f"hyc-mirror-{self.registry_url}".encode()
|
||||
signature = hmac.new(
|
||||
secret_key,
|
||||
f"{token_id}:{issued_at}".encode(),
|
||||
hashlib.sha256
|
||||
).hexdigest()[:32]
|
||||
|
||||
full_token = f"{token_id}-{signature}"
|
||||
|
||||
handler.send_json_response({
|
||||
"token": full_token,
|
||||
"expires_in": expires_in,
|
||||
"issued_at": issued_at
|
||||
})
|
||||
return True
|
||||
|
||||
def _fetch_from_upstream(self, url: str) -> bytes:
|
||||
"""从上游获取数据"""
|
||||
req = urllib.request.Request(url)
|
||||
|
||||
if self.username and self.password:
|
||||
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
|
||||
req.add_header('Authorization', f"Basic {auth}")
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return response.read()
|
||||
|
||||
def _get_cache(self, handler, cache_key: str) -> Optional[bytes]:
|
||||
"""获取缓存"""
|
||||
if not self.cache_enabled:
|
||||
return None
|
||||
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
if not os.path.exists(cache_path):
|
||||
return None
|
||||
|
||||
# 检查是否过期
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f)
|
||||
if time.time() > meta.get('expires', 0):
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
with open(cache_path, 'rb') as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _set_cache(self, cache_key: str, data: bytes):
|
||||
"""设置缓存"""
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
||||
|
||||
try:
|
||||
with open(cache_path, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
meta = {
|
||||
'cached_at': time.time(),
|
||||
'expires': time.time() + self.cache_ttl,
|
||||
'size': len(data)
|
||||
}
|
||||
|
||||
with open(meta_path, 'w') as f:
|
||||
json.dump(meta, f)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Docker缓存写入失败: {e}")
|
||||
|
||||
def _get_cache_path(self, cache_key: str) -> str:
|
||||
"""获取缓存路径"""
|
||||
subdir = cache_key[:2]
|
||||
return os.path.join(self.storage_dir, subdir, cache_key)
|
||||
|
||||
def get_cache_stats(self) -> dict:
|
||||
"""获取缓存统计"""
|
||||
if not os.path.exists(self.storage_dir):
|
||||
return {'files': 0, 'size': 0}
|
||||
|
||||
total_size = 0
|
||||
file_count = 0
|
||||
|
||||
for root, dirs, files in os.walk(self.storage_dir):
|
||||
for f in files:
|
||||
if not f.endswith('.meta'):
|
||||
file_count += 1
|
||||
total_size += os.path.getsize(os.path.join(root, f))
|
||||
|
||||
return {
|
||||
'files': file_count,
|
||||
'size': total_size,
|
||||
'size_formatted': self._format_size(total_size)
|
||||
}
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
"""格式化文件大小"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
units = ["B", "KB", "MB", "GB"]
|
||||
i = 0
|
||||
while size_bytes >= 1024 and i < len(units) - 1:
|
||||
size_bytes /= 1024.0
|
||||
i += 1
|
||||
|
||||
return f"{size_bytes:.2f} {units[i]}"
|
||||
+525
@@ -0,0 +1,525 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
Go模块代理处理器
|
||||
支持Go模块代理协议
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
from typing import Dict, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class GoProxy:
|
||||
"""Go模块代理"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
|
||||
# 配置 - 使用 storage_dir（基于 base_dir）
|
||||
self.upstream_url = config.get('upstream_url', 'https://proxy.golang.org')
|
||||
self.storage_dir = config.get('storage_dir', './downloads/go')
|
||||
self.base_dir = config.get('base_dir', './downloads')
|
||||
self.mode = config.get('mode', 'proxy') # proxy | direct
|
||||
|
||||
# 确保存储目录存在
|
||||
os.makedirs(self.storage_dir, exist_ok=True)
|
||||
|
||||
def handle_request(self, handler, path: str) -> bool:
|
||||
"""
|
||||
处理Go模块请求
|
||||
路径格式: /golang.org/x/net/@v/list
|
||||
/golang.org/x/net/@v/v1.0.0.info
|
||||
/golang.org/x/net/@v/v1.0.0.zip
|
||||
/golang.org/x/net/@latest
|
||||
"""
|
||||
try:
|
||||
parts = path.strip('/').split('/')
|
||||
|
||||
if len(parts) < 2:
|
||||
return self._handle_index(handler)
|
||||
|
||||
# 解析模块路径和操作
|
||||
module_parts = []
|
||||
|
||||
for i, part in enumerate(parts):
|
||||
if part.startswith('@'):
|
||||
# 找到操作部分
|
||||
module_path = '/'.join(parts[:i])
|
||||
action = parts[i:]
|
||||
break
|
||||
else:
|
||||
# 没有找到操作符
|
||||
module_path = '/'.join(parts)
|
||||
action = []
|
||||
|
||||
if not action:
|
||||
handler.send_error(400, "Invalid Go module path")
|
||||
return False
|
||||
|
||||
action_type = action[0]
|
||||
|
||||
if action_type == '@v':
|
||||
# 版本相关操作
|
||||
if len(action) >= 3:
|
||||
version = action[2]
|
||||
return self._handle_version(handler, module_path, version)
|
||||
elif len(action) == 2:
|
||||
# /@v/list
|
||||
return self._handle_version_list(handler, module_path)
|
||||
|
||||
elif action_type == '@latest':
|
||||
# /@latest
|
||||
return self._handle_latest(handler, module_path)
|
||||
|
||||
elif action_type == '@all':
|
||||
# /@all
|
||||
return self._handle_all(handler, module_path)
|
||||
|
||||
elif action_type == '@list':
|
||||
# /@list
|
||||
return self._handle_module_list(handler, module_path)
|
||||
|
||||
else:
|
||||
handler.send_error(400, f"Unknown action: {action_type}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, str(e))
|
||||
return False
|
||||
|
||||
def _handle_index(self, handler) -> bool:
|
||||
"""处理索引请求"""
|
||||
handler.send_json_response({
|
||||
'proxy_url': self.upstream_url,
|
||||
'mode': self.mode,
|
||||
'cache_stats': self.get_cache_stats()
|
||||
})
|
||||
return True
|
||||
|
||||
def _handle_version_list(self, handler, module: str) -> bool:
|
||||
"""处理版本列表请求 /@v/list"""
|
||||
cache_key = f"vlist:{module}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{module}/@v/list"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
handler.send_error(404, f"Module not found: {module}")
|
||||
else:
|
||||
handler.send_error(502, f"Failed to fetch: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_version_info(self, handler, module: str, version: str) -> bool:
|
||||
"""处理版本信息请求 /@v/version.info"""
|
||||
cache_key = f"info:{module}:{version}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/json')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
url = f"{self.upstream_url}/{module}/@v/{version}.info"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/json')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
handler.send_error(502, f"Failed to fetch info: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_version(self, handler, module: str, suffix: str) -> bool:
|
||||
"""处理版本相关请求"""
|
||||
if suffix.endswith('.info'):
|
||||
version = suffix[:-5]
|
||||
return self._handle_version_info(handler, module, version)
|
||||
|
||||
elif suffix.endswith('.zip'):
|
||||
version = suffix[:-4]
|
||||
return self._handle_zip(handler, module, version)
|
||||
|
||||
elif suffix.endswith('.mod'):
|
||||
version = suffix[:-4]
|
||||
return self._handle_mod(handler, module, version)
|
||||
|
||||
elif suffix.endswith('.sum'):
|
||||
version = suffix[:-4]
|
||||
return self._handle_sum(handler, module, version)
|
||||
|
||||
else:
|
||||
handler.send_error(400, f"Unknown suffix: {suffix}")
|
||||
return False
|
||||
|
||||
def _handle_latest(self, handler, module: str) -> bool:
|
||||
"""处理最新版本请求 /@latest"""
|
||||
cache_key = f"latest:{module}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/json')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
url = f"{self.upstream_url}/{module}/@latest"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/json')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
handler.send_error(502, f"Failed to fetch latest: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_zip(self, handler, module: str, version: str) -> bool:
|
||||
"""处理zip下载"""
|
||||
cache_key = f"zip:{module}:{version}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/zip')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
url = f"{self.upstream_url}/{module}/@v/{version}.zip"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/zip')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
handler.send_error(502, f"Failed to fetch zip: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_mod(self, handler, module: str, version: str) -> bool:
|
||||
"""处理mod文件"""
|
||||
cache_key = f"mod:{module}:{version}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
url = f"{self.upstream_url}/{module}/@v/{version}.mod"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
handler.send_error(502, f"Failed to fetch mod: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_sum(self, handler, module: str, version: str) -> bool:
|
||||
"""处理sum文件"""
|
||||
cache_key = f"sum:{module}:{version}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
url = f"{self.upstream_url}/{module}/@v/{version}.sum"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
# 没有sum文件时返回空
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(b'')
|
||||
else:
|
||||
handler.send_error(502, f"Failed to fetch sum: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_all(self, handler, module: str) -> bool:
|
||||
"""处理/@all请求 - 返回模块及其所有依赖的zip包"""
|
||||
cache_key = f"all:{module}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/zip')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取所有依赖的zip
|
||||
url = f"{self.upstream_url}/{module}/@all.zip"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if not data:
|
||||
handler.send_error(404, f"Module not found: {module}")
|
||||
return False
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/zip')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
handler.send_error(404, f"Module not found: {module}")
|
||||
else:
|
||||
handler.send_error(502, f"Failed to fetch @all: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_module_list(self, handler, module: str) -> bool:
|
||||
"""处理/@list请求 - 返回模块及其依赖的路径列表"""
|
||||
cache_key = f"list:{module}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 首先获取模块的 go.mod 文件以提取依赖
|
||||
mod_url = f"{self.upstream_url}/{module}/@v/{module}.mod"
|
||||
|
||||
try:
|
||||
mod_data = self._fetch(mod_url)
|
||||
if not mod_data:
|
||||
handler.send_error(404, f"Module not found: {module}")
|
||||
return False
|
||||
|
||||
# 解析go.mod获取依赖
|
||||
modules = [module]
|
||||
mod_content = mod_data.decode('utf-8', errors='replace')
|
||||
|
||||
# 提取require语句中的依赖
|
||||
import re
|
||||
require_pattern = r'require\s+\(([^\)]+)\)'
|
||||
inline_require_pattern = r'require\s+([^\s]+)\s+([^\s]+)'
|
||||
|
||||
# 处理多行require
|
||||
matches = re.findall(require_pattern, mod_content, re.DOTALL)
|
||||
for match in matches:
|
||||
for line in match.strip().split('\n'):
|
||||
line = line.strip()
|
||||
if line and not line.startswith('//'):
|
||||
parts = line.split()
|
||||
if parts:
|
||||
modules.append(parts[0])
|
||||
|
||||
# 处理单行require
|
||||
matches = re.findall(inline_require_pattern, mod_content)
|
||||
for match in matches:
|
||||
if match[0] not in modules:
|
||||
modules.append(match[0])
|
||||
|
||||
# 生成列表输出
|
||||
list_output = '\n'.join(sorted(set(modules))) + '\n'
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, list_output.encode())
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(list_output.encode())
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
handler.send_error(404, f"Module not found: {module}")
|
||||
else:
|
||||
handler.send_error(502, f"Failed to fetch @list: {str(e)}")
|
||||
return False
|
||||
|
||||
def _fetch(self, url: str) -> Optional[bytes]:
|
||||
"""从URL获取数据"""
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('User-Agent', 'Go-Mirror/1.0')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=60) as response:
|
||||
return response.read()
|
||||
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _get_cache(self, cache_key: str) -> Optional[bytes]:
|
||||
"""获取缓存"""
|
||||
if not self.cache_enabled:
|
||||
return None
|
||||
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
if not os.path.exists(cache_path):
|
||||
return None
|
||||
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f)
|
||||
if time.time() > meta.get('expires', 0):
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
with open(cache_path, 'rb') as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _set_cache(self, cache_key: str, data: bytes):
|
||||
"""设置缓存"""
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
||||
|
||||
try:
|
||||
with open(cache_path, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
meta = {
|
||||
'cached_at': time.time(),
|
||||
'expires': time.time() + self.cache_ttl,
|
||||
'size': len(data)
|
||||
}
|
||||
|
||||
with open(meta_path, 'w') as f:
|
||||
json.dump(meta, f)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Go缓存写入失败: {e}")
|
||||
|
||||
def _get_cache_path(self, cache_key: str) -> str:
|
||||
"""获取缓存路径"""
|
||||
subdir = cache_key[:2]
|
||||
return os.path.join(self.storage_dir, subdir, cache_key)
|
||||
|
||||
def get_cache_stats(self) -> dict:
|
||||
"""获取缓存统计"""
|
||||
if not os.path.exists(self.storage_dir):
|
||||
return {'files': 0, 'size': 0}
|
||||
|
||||
total_size = 0
|
||||
file_count = 0
|
||||
|
||||
for root, dirs, files in os.walk(self.storage_dir):
|
||||
for f in files:
|
||||
if not f.endswith('.meta'):
|
||||
file_count += 1
|
||||
total_size += os.path.getsize(os.path.join(root, f))
|
||||
|
||||
return {
|
||||
'files': file_count,
|
||||
'size': total_size,
|
||||
'size_formatted': self._format_size(total_size)
|
||||
}
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
"""格式化文件大小"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
units = ["B", "KB", "MB", "GB"]
|
||||
i = 0
|
||||
while size_bytes >= 1024 and i < len(units) - 1:
|
||||
size_bytes /= 1024.0
|
||||
i += 1
|
||||
|
||||
return f"{size_bytes:.2f} {units[i]}"
|
||||
+485
@@ -0,0 +1,485 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
通用HTTP镜像代理处理器
|
||||
支持Maven、Gradle、RubyGems、Cargo、NuGet、CocoaPods、CRAN、CTAN、CUDA、Pacman等基于HTTP的镜像
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
from typing import Dict, List, Optional
|
||||
from datetime import datetime
|
||||
from html import escape
|
||||
|
||||
|
||||
class HttpMirror:
|
||||
"""通用HTTP镜像代理 - 支持多种包管理器"""
|
||||
|
||||
# 不同包管理器的默认上游URL
|
||||
DEFAULT_UPSTREAM = {
|
||||
'maven': 'https://repo1.maven.org/maven2',
|
||||
'gradle': 'https://services.gradle.org/distributions',
|
||||
'gem': 'https://rubygems.org',
|
||||
'cargo': 'https://crates.io',
|
||||
'nuget': 'https://api.nuget.org/v3',
|
||||
'cocoapods': 'https://cdn.cocoapods.org',
|
||||
'cran': 'https://cran.r-project.org',
|
||||
'ctan': 'https://ctan.math.illinois.edu',
|
||||
'cuda': 'https://developer.download.nvidia.com/compute/cuda/repos',
|
||||
'pacman': 'https://mirror.archlinux.org',
|
||||
}
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
self.mirror_type = config.get('type', 'http')
|
||||
|
||||
# 配置
|
||||
self.upstream_url = config.get('upstream_url', self.DEFAULT_UPSTREAM.get(self.mirror_type, 'https://mirror.example.com'))
|
||||
self.storage_dir = config.get('storage_dir', f'./downloads/{self.mirror_type}')
|
||||
self.base_dir = config.get('base_dir', './downloads')
|
||||
self.cache_enabled = config.get('cache_enabled', True)
|
||||
self.cache_ttl = config.get('cache_ttl', 3600) # 默认1小时
|
||||
|
||||
# 确保存储目录存在
|
||||
os.makedirs(self.storage_dir, exist_ok=True)
|
||||
|
||||
def handle_request(self, handler, path: str) -> bool:
|
||||
"""处理HTTP镜像请求"""
|
||||
try:
|
||||
# 移除前导斜杠
|
||||
path = path.strip('/')
|
||||
|
||||
if not path or path == self.mirror_type:
|
||||
return self._handle_index(handler)
|
||||
|
||||
# 根据镜像类型分发请求
|
||||
if self.mirror_type == 'maven':
|
||||
return self._handle_maven(handler, path)
|
||||
elif self.mirror_type == 'gradle':
|
||||
return self._handle_gradle(handler, path)
|
||||
elif self.mirror_type == 'gem':
|
||||
return self._handle_gem(handler, path)
|
||||
elif self.mirror_type == 'cargo':
|
||||
return self._handle_cargo(handler, path)
|
||||
elif self.mirror_type == 'nuget':
|
||||
return self._handle_nuget(handler, path)
|
||||
elif self.mirror_type == 'cocoapods':
|
||||
return self._handle_cocoapods(handler, path)
|
||||
elif self.mirror_type == 'cran':
|
||||
return self._handle_cran(handler, path)
|
||||
elif self.mirror_type == 'ctan':
|
||||
return self._handle_ctan(handler, path)
|
||||
elif self.mirror_type == 'cuda':
|
||||
return self._handle_cuda(handler, path)
|
||||
elif self.mirror_type == 'pacman':
|
||||
return self._handle_pacman(handler, path)
|
||||
else:
|
||||
# 默认作为普通HTTP文件处理
|
||||
return self._handle_generic(handler, path)
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, str(e))
|
||||
return False
|
||||
|
||||
def _handle_index(self, handler) -> bool:
|
||||
"""返回镜像索引信息"""
|
||||
handler.send_json_response({
|
||||
'type': self.mirror_type,
|
||||
'upstream_url': self.upstream_url,
|
||||
'storage_dir': self.storage_dir,
|
||||
'cache_enabled': self.cache_enabled,
|
||||
'cache_stats': self.get_cache_stats()
|
||||
})
|
||||
return True
|
||||
|
||||
# ========== Maven ==========
|
||||
def _handle_maven(self, handler, path: str) -> bool:
|
||||
"""处理Maven请求: /groupId/artifactId/version/artifactId-version.jar"""
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== Gradle ==========
|
||||
def _handle_gradle(self, handler, path: str) -> bool:
|
||||
"""处理Gradle请求: /gradle-x.x-bin.zip"""
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== RubyGems ==========
|
||||
def _handle_gem(self, handler, path: str) -> bool:
|
||||
"""处理RubyGems请求"""
|
||||
# API请求: /api/v1/gems/xxx.json
|
||||
# Spec请求: /quick/Marshal.4.8/xxx-x.x.0.gemspec.rz
|
||||
# 下载请求: /gems/xxx-x.x.0.gem
|
||||
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== Cargo ==========
|
||||
def _handle_cargo(self, handler, path: str) -> bool:
|
||||
"""处理Cargo请求"""
|
||||
# API: /api/v1/crates
|
||||
# 下载: /crates/xxx/xxx-x.x.x.crate
|
||||
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== NuGet ==========
|
||||
def _handle_nuget(self, handler, path: str) -> bool:
|
||||
"""处理NuGet请求"""
|
||||
# API v3: /v3-flatcontainer/xxx.nupkg
|
||||
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== CocoaPods ==========
|
||||
def _handle_cocoapods(self, handler, path: str) -> bool:
|
||||
"""处理CocoaPods请求"""
|
||||
# Specs: /Specs/xxx.podspec.json
|
||||
# Pods: /Pods/xxx/xxx.pod
|
||||
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== CRAN ==========
|
||||
def _handle_cran(self, handler, path: str) -> bool:
|
||||
"""处理CRAN请求"""
|
||||
# /src/contrib/xxx.tar.gz
|
||||
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== CTAN ==========
|
||||
def _handle_ctan(self, handler, path: str) -> bool:
|
||||
"""处理CTAN请求"""
|
||||
# /macros/latex/xxx.zip
|
||||
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== CUDA ==========
|
||||
def _handle_cuda(self, handler, path: str) -> bool:
|
||||
"""处理CUDA请求"""
|
||||
# /xxx/xxx.deb
|
||||
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== Pacman ==========
|
||||
def _handle_pacman(self, handler, path: str) -> bool:
|
||||
"""处理Pacman请求"""
|
||||
# /os/x86_64/xxx.db.tar.gz
|
||||
# /os/x86_64/xxx-x.x.x-x-x86_64.pkg.tar.zst
|
||||
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== Generic HTTP ==========
|
||||
def _handle_generic(self, handler, path: str) -> bool:
|
||||
"""通用的HTTP文件代理"""
|
||||
# 先检查本地
|
||||
local_path = os.path.join(self.storage_dir, path)
|
||||
if os.path.exists(local_path) and os.path.isfile(local_path):
|
||||
return self._serve_local_file(handler, local_path)
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
return self._proxy_request(handler, url)
|
||||
|
||||
# ========== 辅助方法 ==========
|
||||
def _serve_local_file(self, handler, local_path: str) -> bool:
|
||||
"""服务本地文件"""
|
||||
try:
|
||||
file_size = os.path.getsize(local_path)
|
||||
|
||||
# 尝试确定内容类型
|
||||
content_type = self._guess_content_type(local_path)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', content_type)
|
||||
handler.send_header('Content-Length', str(file_size))
|
||||
handler.send_header('X-Local', 'true')
|
||||
handler.end_headers()
|
||||
|
||||
with open(local_path, 'rb') as f:
|
||||
handler.wfile.write(f.read())
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
handler.send_error(500, f"Error serving file: {str(e)}")
|
||||
return False
|
||||
|
||||
def _proxy_request(self, handler, url: str) -> bool:
|
||||
"""代理请求到上游"""
|
||||
try:
|
||||
# 检查缓存
|
||||
cache_key = self._get_cache_key(url)
|
||||
if self.cache_enabled:
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
return self._serve_cached(handler, cached, url)
|
||||
|
||||
# 从上游获取
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('User-Agent', f'HTTP-Mirror/1.0 ({self.mirror_type})')
|
||||
|
||||
# 处理Range请求
|
||||
range_header = handler.headers.get('Range')
|
||||
if range_header:
|
||||
req.add_header('Range', range_header)
|
||||
|
||||
try:
|
||||
response = urllib.request.urlopen(req, timeout=60)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
handler.send_error(404, f"File not found: {url}")
|
||||
else:
|
||||
handler.send_error(502, f"Upstream error: {str(e)}")
|
||||
return False
|
||||
except Exception as e:
|
||||
handler.send_error(502, f"Failed to connect upstream: {str(e)}")
|
||||
return False
|
||||
|
||||
# 获取响应头
|
||||
content_type = response.headers.get('Content-Type', 'application/octet-stream')
|
||||
content_length = response.headers.get('Content-Length')
|
||||
content_range = response.headers.get('Content-Range')
|
||||
|
||||
# 处理部分内容响应
|
||||
if content_range:
|
||||
handler.send_response(206)
|
||||
handler.send_header('Content-Range', content_range)
|
||||
else:
|
||||
handler.send_response(200)
|
||||
|
||||
handler.send_header('Content-Type', content_type)
|
||||
if content_length:
|
||||
handler.send_header('Content-Length', content_length)
|
||||
handler.send_header('X-Upstream', self.upstream_url)
|
||||
handler.end_headers()
|
||||
|
||||
# 读取并转发内容，同时缓存
|
||||
data = response.read()
|
||||
|
||||
if self.cache_enabled and response.status == 200:
|
||||
self._set_cache(cache_key, data, content_type)
|
||||
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(502, f"Proxy error: {str(e)}")
|
||||
return False
|
||||
|
||||
def _serve_cached(self, handler, cached: dict, url: str):
|
||||
"""服务缓存的文件"""
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', cached.get('content_type', 'application/octet-stream'))
|
||||
handler.send_header('Content-Length', len(cached.get('data', b'')))
|
||||
handler.send_header('X-Cached', 'true')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached.get('data', b''))
|
||||
return True
|
||||
|
||||
def _get_cache_key(self, url: str) -> str:
|
||||
"""生成缓存键"""
|
||||
return urllib.parse.quote(url, safe='')
|
||||
|
||||
def _get_cache_path(self, cache_key: str) -> str:
|
||||
"""获取缓存文件路径"""
|
||||
# 使用前两个字符作为子目录
|
||||
subdir = cache_key[:2] if len(cache_key) >= 2 else 'cache'
|
||||
return os.path.join(self.storage_dir, '.cache', subdir, cache_key)
|
||||
|
||||
def _get_cache(self, cache_key: str) -> Optional[dict]:
|
||||
"""获取缓存"""
|
||||
if not self.cache_enabled:
|
||||
return None
|
||||
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
if not os.path.exists(cache_path):
|
||||
return None
|
||||
|
||||
# 检查是否过期
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f)
|
||||
if time.time() > meta.get('expires', 0):
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 读取缓存数据
|
||||
try:
|
||||
with open(cache_path, 'rb') as f:
|
||||
data = f.read()
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f)
|
||||
return {
|
||||
'data': data,
|
||||
'content_type': meta.get('content_type', 'application/octet-stream')
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _set_cache(self, cache_key: str, data: bytes, content_type: str = 'application/octet-stream'):
|
||||
"""设置缓存"""
|
||||
try:
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
||||
|
||||
# 写入数据
|
||||
with open(cache_path, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
# 写入元数据
|
||||
meta = {
|
||||
'cached_at': time.time(),
|
||||
'expires': time.time() + self.cache_ttl,
|
||||
'size': len(data),
|
||||
'content_type': content_type,
|
||||
'url': cache_key
|
||||
}
|
||||
with open(meta_path, 'w') as f:
|
||||
json.dump(meta, f)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[{self.mirror_type}] Cache write failed: {e}")
|
||||
|
||||
def _guess_content_type(self, path: str) -> str:
|
||||
"""根据文件扩展名猜测内容类型"""
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
|
||||
content_types = {
|
||||
'.jar': 'application/java-archive',
|
||||
'.war': 'application/java-archive',
|
||||
'.ear': 'application/java-archive',
|
||||
'.pom': 'application/xml',
|
||||
'.xml': 'application/xml',
|
||||
'.json': 'application/json',
|
||||
'.gem': 'application/octet-stream',
|
||||
'.crate': 'application/octet-stream',
|
||||
'.nupkg': 'application/zip',
|
||||
'.tar.gz': 'application/gzip',
|
||||
'.tgz': 'application/gzip',
|
||||
'.zip': 'application/zip',
|
||||
'.deb': 'application/deb',
|
||||
'.rpm': 'application/x-rpm',
|
||||
'.pkg.tar.zst': 'application/zstd',
|
||||
'.podspec': 'text/plain',
|
||||
'.tar': 'application/x-tar',
|
||||
'.pdf': 'application/pdf',
|
||||
'.html': 'text/html',
|
||||
'.css': 'text/css',
|
||||
'.js': 'application/javascript',
|
||||
}
|
||||
|
||||
return content_types.get(ext, 'application/octet-stream')
|
||||
|
||||
def get_cache_stats(self) -> dict:
|
||||
"""获取缓存统计"""
|
||||
cache_dir = os.path.join(self.storage_dir, '.cache')
|
||||
if not os.path.exists(cache_dir):
|
||||
return {'files': 0, 'size': 0, 'size_formatted': '0 B'}
|
||||
|
||||
total_size = 0
|
||||
file_count = 0
|
||||
|
||||
try:
|
||||
for root, dirs, files in os.walk(cache_dir):
|
||||
for f in files:
|
||||
if not f.endswith('.meta'):
|
||||
file_count += 1
|
||||
total_size += os.path.getsize(os.path.join(root, f))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
'files': file_count,
|
||||
'size': total_size,
|
||||
'size_formatted': self._format_size(total_size)
|
||||
}
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
"""格式化文件大小"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
units = ["B", "KB", "MB", "GB", "TB"]
|
||||
i = 0
|
||||
size = float(size_bytes)
|
||||
while size >= 1024 and i < len(units) - 1:
|
||||
size /= 1024.0
|
||||
i += 1
|
||||
|
||||
return f"{size:.2f} {units[i]}"
|
||||
+280
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
npm镜像代理处理器
|
||||
支持Node.js包管理器
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
from typing import Dict, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class NpmMirror:
|
||||
"""npm镜像代理"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
|
||||
# 配置 - 使用 storage_dir（基于 base_dir）
|
||||
self.upstream_url = config.get('upstream_url', 'https://registry.npmjs.org')
|
||||
self.storage_dir = config.get('storage_dir', './downloads/npm')
|
||||
self.base_dir = config.get('base_dir', './downloads')
|
||||
|
||||
# 确保存储目录存在
|
||||
os.makedirs(self.storage_dir, exist_ok=True)
|
||||
|
||||
def handle_request(self, handler, path: str) -> bool:
|
||||
"""
|
||||
处理npm请求
|
||||
路径格式: /lodash 或 /-/package/lodash/dist
|
||||
"""
|
||||
try:
|
||||
parts = path.strip('/').split('/')
|
||||
|
||||
if not parts:
|
||||
return self._handle_index(handler)
|
||||
|
||||
if parts[0] == '-':
|
||||
# Scoped package 或其他特殊请求
|
||||
if len(parts) >= 4 and parts[1] == 'package':
|
||||
return self._handleScopedPackage(handler, parts[2], parts[3] if len(parts) > 3 else None)
|
||||
elif len(parts) >= 3 and parts[1] == 'package':
|
||||
return self._handle_package(handler, parts[2], None)
|
||||
else:
|
||||
handler.send_error(400, "Invalid npm API path")
|
||||
return False
|
||||
|
||||
elif parts[0] == '@':
|
||||
# Scoped package
|
||||
if len(parts) >= 2:
|
||||
scope = parts[0]
|
||||
package = '/'.join(parts[1:])
|
||||
return self._handle_scoped_package(handler, scope, package)
|
||||
else:
|
||||
handler.send_error(400, "Invalid scoped package")
|
||||
return False
|
||||
|
||||
elif parts[0] == '-/':
|
||||
# npm特殊路径
|
||||
return self._handle_special(handler, '/'.join(parts))
|
||||
|
||||
elif len(parts) == 1:
|
||||
# 单个包名
|
||||
return self._handle_package(handler, parts[0], None)
|
||||
|
||||
else:
|
||||
# 其他请求
|
||||
return self._handle_package(handler, parts[0], parts[1] if len(parts) > 1 else None)
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, str(e))
|
||||
return False
|
||||
|
||||
def _handle_index(self, handler) -> bool:
|
||||
"""处理索引请求"""
|
||||
handler.send_json_response({
|
||||
'registry_url': self.upstream_url,
|
||||
'cache_stats': self.get_cache_stats()
|
||||
})
|
||||
return True
|
||||
|
||||
def _handle_package(self, handler, package: str, version: str = None) -> bool:
|
||||
"""处理包元数据请求"""
|
||||
cache_key = f"package:{package}:{version or 'latest'}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/json')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取
|
||||
if version:
|
||||
url = f"{self.upstream_url}/{package}/{version}"
|
||||
else:
|
||||
url = f"{self.upstream_url}/{package}/latest"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/json')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
handler.send_error(404, f"Package not found: {package}")
|
||||
return False
|
||||
|
||||
def _handle_scoped_package(self, handler, scope: str, package: str) -> bool:
|
||||
"""处理scoped包"""
|
||||
full_name = f"{scope}/{package}"
|
||||
return self._handle_package(handler, full_name, None)
|
||||
|
||||
def _handleScopedPackage(self, handler, scope: str, package: str) -> bool:
|
||||
"""处理特殊路径的scoped包"""
|
||||
full_name = f"{scope}/{package}"
|
||||
return self._handle_package(handler, full_name, None)
|
||||
|
||||
def _handle_special(self, handler, path: str) -> bool:
|
||||
"""处理特殊npm路径"""
|
||||
# 简化实现：转发到上游
|
||||
url = f"{self.upstream_url}/{path}"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/json')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(502, f"Failed to fetch: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_tarball(self, handler, package: str, filename: str) -> bool:
|
||||
"""处理tarball下载"""
|
||||
cache_key = f"tarball:{package}:{filename}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.upstream_url}/{package}/-/{filename}"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(502, f"Failed to fetch tarball: {str(e)}")
|
||||
return False
|
||||
|
||||
def _fetch(self, url: str) -> Optional[bytes]:
|
||||
"""从URL获取数据"""
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('User-Agent', 'npm-Mirror/1.0')
|
||||
req.add_header('Accept', 'application/json')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return response.read()
|
||||
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _get_cache(self, cache_key: str) -> Optional[bytes]:
|
||||
"""获取缓存"""
|
||||
if not self.cache_enabled:
|
||||
return None
|
||||
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
if not os.path.exists(cache_path):
|
||||
return None
|
||||
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f)
|
||||
if time.time() > meta.get('expires', 0):
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
with open(cache_path, 'rb') as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _set_cache(self, cache_key: str, data: bytes):
|
||||
"""设置缓存"""
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
||||
|
||||
try:
|
||||
with open(cache_path, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
meta = {
|
||||
'cached_at': time.time(),
|
||||
'expires': time.time() + self.cache_ttl,
|
||||
'size': len(data)
|
||||
}
|
||||
|
||||
with open(meta_path, 'w') as f:
|
||||
json.dump(meta, f)
|
||||
|
||||
except Exception as e:
|
||||
print(f"npm缓存写入失败: {e}")
|
||||
|
||||
def _get_cache_path(self, cache_key: str) -> str:
|
||||
"""获取缓存路径"""
|
||||
subdir = cache_key[:2]
|
||||
return os.path.join(self.storage_dir, subdir, cache_key)
|
||||
|
||||
def get_cache_stats(self) -> dict:
|
||||
"""获取缓存统计"""
|
||||
if not os.path.exists(self.storage_dir):
|
||||
return {'files': 0, 'size': 0}
|
||||
|
||||
total_size = 0
|
||||
file_count = 0
|
||||
|
||||
for root, dirs, files in os.walk(self.storage_dir):
|
||||
for f in files:
|
||||
if not f.endswith('.meta'):
|
||||
file_count += 1
|
||||
total_size += os.path.getsize(os.path.join(root, f))
|
||||
|
||||
return {
|
||||
'files': file_count,
|
||||
'size': total_size,
|
||||
'size_formatted': self._format_size(total_size)
|
||||
}
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
"""格式化文件大小"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
units = ["B", "KB", "MB", "GB"]
|
||||
i = 0
|
||||
while size_bytes >= 1024 and i < len(units) - 1:
|
||||
size_bytes /= 1024.0
|
||||
i += 1
|
||||
|
||||
return f"{size_bytes:.2f} {units[i]}"
|
||||
+788
@@ -0,0 +1,788 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
PyPI镜像代理处理器
|
||||
支持Python包索引
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
from typing import Dict, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class PyPIMirror:
|
||||
"""PyPI镜像代理"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
|
||||
# 配置 - storage_dir 基于 base_dir
|
||||
self.upstream_url = config.get('upstream_url', 'https://pypi.org')
|
||||
self.base_dir = config.get('base_dir', './downloads')
|
||||
storage_subdir = config.get('storage_dir', 'pypi')
|
||||
self.storage_dir = os.path.join(self.base_dir, storage_subdir)
|
||||
self.simple_dir = os.path.join(self.storage_dir, 'simple')
|
||||
self.web_dir = os.path.join(self.storage_dir, 'web')
|
||||
|
||||
# 确保存储目录存在
|
||||
os.makedirs(self.simple_dir, exist_ok=True)
|
||||
os.makedirs(self.web_dir, exist_ok=True)
|
||||
|
||||
def handle_request(self, handler, path: str) -> bool:
|
||||
"""
|
||||
处理PyPI请求
|
||||
路径格式: /simple/requests/ 或 /packages/xxx.tar.gz 或 /pypi/web/package 或 /pypi/packages/hash/file
|
||||
"""
|
||||
try:
|
||||
import sys
|
||||
# 如果路径以 pypi/ 开头，也需要去掉
|
||||
path = path.lstrip('/')
|
||||
if path.startswith('pypi/'):
|
||||
path = path[5:]
|
||||
parts = path.strip('/').split('/')
|
||||
|
||||
# 过滤空字符串
|
||||
parts = [p for p in parts if p]
|
||||
|
||||
if not parts:
|
||||
return self._handle_index(handler)
|
||||
|
||||
if parts[0] == 'simple':
|
||||
# Simple API
|
||||
import sys
|
||||
if len(parts) == 1:
|
||||
# /simple/ - 返回根索引
|
||||
return self._handle_index(handler)
|
||||
elif len(parts) == 2:
|
||||
# /simple/package/
|
||||
return self._handle_simple_index(handler, parts[1])
|
||||
elif len(parts) >= 3:
|
||||
# /simple/package/version/ 或 /simple/package/version#egg=...
|
||||
return self._handle_package_file(handler, parts[1], '/'.join(parts[2:]))
|
||||
else:
|
||||
handler.send_error(400, "Invalid simple API path")
|
||||
return False
|
||||
|
||||
elif parts[0] == 'web':
|
||||
# /web/package/ 或 /web/package/json
|
||||
# pip sends /pypi/web/<package>/json
|
||||
package = parts[1] if len(parts) >= 2 else ''
|
||||
return self._handle_web_api(handler, package)
|
||||
|
||||
elif parts[0] == 'packages':
|
||||
# 包下载
|
||||
filename = '/'.join(parts[1:])
|
||||
return self._handle_package_download(handler, filename)
|
||||
|
||||
elif parts[0] == 'legacy':
|
||||
# 旧版PyPI兼容
|
||||
return self._handle_legacy(handler, '/'.join(parts[1:]))
|
||||
|
||||
else:
|
||||
handler.send_error(404, "Unknown API")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, str(e))
|
||||
return False
|
||||
|
||||
def _handle_index(self, handler) -> bool:
|
||||
"""处理索引请求 - 返回所有可用包的列表"""
|
||||
# 从上游获取包列表
|
||||
url = self.upstream_url.rstrip('/')
|
||||
if url.endswith('/simple'):
|
||||
url = url # 保持 /simple
|
||||
else:
|
||||
url = url + '/simple'
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('Accept', 'text/html')
|
||||
req.add_header('User-Agent', 'PyPI-Mirror/1.0')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
data = response.read().decode('utf-8')
|
||||
|
||||
# 转换相对链接
|
||||
# 清华源返回的可能是完整的HTML，需要转换链接
|
||||
data = self._convert_simple_index_html(data)
|
||||
data_bytes = data.encode('utf-8')
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/html; charset=utf-8')
|
||||
handler.send_header('Content-Length', str(len(data_bytes)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data_bytes)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(502, f"Failed to fetch package index: {str(e)}")
|
||||
return False
|
||||
|
||||
def _convert_simple_index_html(self, html: str) -> str:
|
||||
"""转换根索引页面的HTML"""
|
||||
import re
|
||||
# 替换上游链接
|
||||
def convert_link(match):
|
||||
href = match.group(1)
|
||||
text = match.group(2)
|
||||
if href.startswith('/simple/'):
|
||||
return match.group(0) # 已经是相对路径
|
||||
elif href.startswith('https://pypi.tuna.tsinghua.edu.cn/simple/'):
|
||||
simple_part = href.split("/simple/")[-1]
|
||||
return f'<a href="/simple/{simple_part}">{text}</a>'
|
||||
elif href.startswith('https://'):
|
||||
# 其他上游链接，提取包名
|
||||
pkg_name = href.rstrip('/').split('/')[-1]
|
||||
return f'<a href="/simple/{pkg_name}/">{text}</a>'
|
||||
return match.group(0)
|
||||
|
||||
# 匹配 <a href="...">text</a>
|
||||
return re.sub(r'<a[^>]+href="([^"]+)"[^>]*>([^<]*)</a>', convert_link, html)
|
||||
|
||||
def _handle_simple_index(self, handler, package: str) -> bool:
|
||||
"""处理Simple API索引请求"""
|
||||
import json
|
||||
import time
|
||||
|
||||
package = package.lower()
|
||||
|
||||
# 调试 - 确保函数被调用
|
||||
debug_file = '/tmp/pypi_debug.log'
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[HANDLE_SIMPLE] START package={package}\n")
|
||||
|
||||
# 检查客户端Accept header
|
||||
accept = handler.headers.get('Accept', '')
|
||||
wants_json = 'application/vnd.pypi.simple.v1+json' in accept
|
||||
|
||||
# 调试
|
||||
debug_file = '/tmp/pypi_debug.log'
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[SIMPLE_INDEX] package={package}, wants_json={wants_json}, accept={accept[:50]}\n")
|
||||
|
||||
# 根据请求格式选择正确的缓存key，统一使用 simple/ 前缀
|
||||
cache_key = f"simple/{package}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
|
||||
# 调试缓存
|
||||
debug_file = '/tmp/pypi_debug.log'
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[CACHE_CHECK] cache_key={cache_key}, cached={'YES' if cached else 'NO'}\n")
|
||||
|
||||
if cached:
|
||||
# 返回缓存，使用正确的Content-Type
|
||||
if wants_json:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/vnd.pypi.simple.v1+json; charset=utf-8')
|
||||
else:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'text/html; charset=utf-8')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取HTML
|
||||
# upstream_url 已经是完整路径（如 https://pypi.tuna.tsinghua.edu.cn/simple）
|
||||
# 所以只需要添加 /package/
|
||||
url = f"{self.upstream_url}/{package}/"
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('Accept', 'text/html')
|
||||
req.add_header('User-Agent', 'PyPI-Mirror/1.0')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
data = response.read().decode('utf-8')
|
||||
|
||||
# 根据客户端请求返回不同格式，统一使用 simple/ 路径
|
||||
if wants_json:
|
||||
# 转换为JSON格式
|
||||
json_data = self._convert_to_json(package, data)
|
||||
proxy_data = json.dumps(json_data)
|
||||
content_type = 'application/vnd.pypi.simple.v1+json; charset=utf-8'
|
||||
cache_key = f"simple/{package}"
|
||||
else:
|
||||
# 转换为HTML格式
|
||||
debug_file = '/tmp/pypi_debug.log'
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[CONVERT_CALL] Before conversion\n")
|
||||
try:
|
||||
proxy_data = self._convert_simple_html(package, data)
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[CONVERT_CALL] After conversion\n")
|
||||
except Exception as e:
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[CONVERT_ERROR] {e}\n")
|
||||
proxy_data = data # fallback to raw data
|
||||
content_type = 'application/vnd.pypi.simple.v1+html; charset=utf-8'
|
||||
|
||||
if True:
|
||||
self._set_cache(cache_key, proxy_data.encode('utf-8'))
|
||||
|
||||
proxy_bytes = proxy_data.encode('utf-8')
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', content_type)
|
||||
handler.send_header('Content-Length', str(len(proxy_bytes)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(proxy_bytes)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
handler.send_error(404, f"Package not found: {package}")
|
||||
else:
|
||||
handler.send_error(502, f"Failed to fetch from upstream: {str(e)}")
|
||||
return False
|
||||
|
||||
def _convert_to_json(self, package: str, html: str) -> dict:
|
||||
"""将HTML转换为JSON格式"""
|
||||
import re
|
||||
|
||||
# 解析HTML中的链接
|
||||
links = []
|
||||
|
||||
# 匹配 <a href="...">text</a>
|
||||
pattern = r'<a[^>]+href="([^"]+)"[^>]*>([^<]*)</a>'
|
||||
matches = re.findall(pattern, html)
|
||||
|
||||
for href, text in matches:
|
||||
# 提取文件名（从链接文本）
|
||||
filename = text.strip() if text.strip() else ''
|
||||
|
||||
# 如果没有链接文本，从URL中提取
|
||||
if not filename:
|
||||
if '#' in href:
|
||||
filename = href.split('#')[0].split('/')[-1]
|
||||
else:
|
||||
filename = href.split('/')[-1]
|
||||
|
||||
# JSON格式中URL不应该包含fragment
|
||||
# pip从filename字段提取版本号（如 Flask-1.0.0.tar.gz -> 1.0.0）
|
||||
|
||||
# 解析URL并转换为代理路径
|
||||
url = href
|
||||
if href.startswith('../'):
|
||||
# 相对路径 - 需要转换为代理路径
|
||||
# 格式: ../../packages/hash1/hash2/fullhash/filename#sha256=...
|
||||
# 或: ../../packages/hash1/hash2/filename
|
||||
# parts = ['..', '..', 'packages', 'hash1', 'hash2', 'fullhash', 'filename', ...]
|
||||
parts = href.split('/')
|
||||
try:
|
||||
pkg_idx = parts.index('packages')
|
||||
# 提取从 packages 后面到文件名之前的所有部分作为 hash 路径
|
||||
# 文件名是最后一个非空部分（可能包含 #fragment）
|
||||
# 找到文件名的位置（最后一个部分）
|
||||
filename_idx = len(parts) - 1
|
||||
while filename_idx > pkg_idx and not parts[filename_idx]:
|
||||
filename_idx -= 1
|
||||
# hash_path 是 packages 后面到文件名之前的所有部分
|
||||
if filename_idx > pkg_idx + 1:
|
||||
hash_path = '/'.join(parts[pkg_idx+1:filename_idx])
|
||||
else:
|
||||
hash_path = parts[pkg_idx+1] if pkg_idx + 1 < len(parts) else ''
|
||||
# 文件名: 检查 pkg_idx+3 是否存在且不是哈希
|
||||
if pkg_idx + 3 < len(parts):
|
||||
fname_full = parts[pkg_idx+3]
|
||||
# 如果 fname_full 看起来像哈希（包含 sha256= 或长度>=32的十六进制），则使用原始 filename
|
||||
# 清华源格式: .../hash/filename#sha256=...
|
||||
# 其中 hash 是 28-30 位十六进制
|
||||
is_hash_like = ('sha256=' in fname_full or 'sha512=' in fname_full or
|
||||
(len(fname_full) >= 28 and all(c in '0123456789abcdef' for c in fname_full[:28].lower())))
|
||||
if is_hash_like:
|
||||
# 这是哈希，不是文件名
|
||||
fname = filename if filename else ''
|
||||
else:
|
||||
# 这是文件名
|
||||
fname = fname_full.split('#')[0]
|
||||
if not filename:
|
||||
filename = fname
|
||||
else:
|
||||
fname = filename if filename else ''
|
||||
# URL不包含fragment
|
||||
url = f"/pypi/packages/{hash_path}/{fname}"
|
||||
except ValueError:
|
||||
pass
|
||||
elif href.startswith('/pypi/'):
|
||||
# 绝对路径（如 /pypi/packages/hash/filename#egg=package-version）
|
||||
# 去掉fragment
|
||||
href_clean = href.split('#')[0]
|
||||
parts = href_clean.split('/')
|
||||
# parts = ['', 'pypi', 'packages', 'hash', 'filename']
|
||||
if len(parts) >= 5:
|
||||
hash_path = parts[3]
|
||||
fname = parts[4]
|
||||
if not filename:
|
||||
filename = fname
|
||||
url = f"/pypi/packages/{hash_path}/{fname}"
|
||||
elif href.startswith('http'):
|
||||
# 绝对URL - 转换为代理路径
|
||||
if 'files.pythonhosted.org' in href or 'files.pypi.org' in href:
|
||||
# 格式: https://files.pythonhosted.org/packages/hash1/hash2/完整哈希/filename
|
||||
# 例如: https://files.pythonhosted.org/packages/ec/f9/7f9263c5695f4bd0023734af91bedb2ff8209e8de6ead162f35d8dc762fd/flask-3.1.2-py3-none-any.whl
|
||||
parts = href.split('/packages/')
|
||||
if len(parts) >= 2:
|
||||
path_after_packages = parts[1]
|
||||
# 完整路径: hash1/hash2/完整哈希/filename
|
||||
url = f"/pypi/packages/{path_after_packages}"
|
||||
fname = path_after_packages.split('/')[-1]
|
||||
if not filename:
|
||||
filename = fname
|
||||
elif 'pypi.tuna.tsinghua.edu.cn' in href or 'mirrors.tuna.tsinghua.edu.cn' in href:
|
||||
parts = href.rsplit('/', 1)
|
||||
if len(parts) == 2:
|
||||
path_part = parts[0]
|
||||
fname = parts[1]
|
||||
hash_path = path_part.split('/')[-1]
|
||||
if not filename:
|
||||
filename = fname
|
||||
url = f"/pypi/packages/{hash_path}/{fname}"
|
||||
|
||||
link_entry = {
|
||||
"filename": filename,
|
||||
"url": url
|
||||
}
|
||||
|
||||
links.append(link_entry)
|
||||
|
||||
return {
|
||||
"meta": {
|
||||
"api-version": "1.0",
|
||||
"repository-version": "1.0"
|
||||
},
|
||||
"name": package,
|
||||
"files": links
|
||||
}
|
||||
|
||||
def _handle_package_file(self, handler, package: str, filename: str) -> bool:
|
||||
"""处理包文件请求"""
|
||||
import urllib.parse
|
||||
|
||||
# 解析文件名
|
||||
# 新格式: /pypi/packages/hash/filename#pip=package-version
|
||||
# 或旧格式: /pypi/packages/filename?url=...
|
||||
parsed = urllib.parse.urlparse(f"/{filename}")
|
||||
actual_filename = parsed.path.lstrip('/')
|
||||
query_params = urllib.parse.parse_qs(parsed.query)
|
||||
fragment = urllib.parse.parse_qs(parsed.fragment) if parsed.fragment else {}
|
||||
|
||||
# 从fragment中提取包信息（用于缓存键）
|
||||
actual_package = package
|
||||
if 'pip' in fragment:
|
||||
# 格式: #pip=flask-2.0.0
|
||||
pip_info = fragment['pip'][0]
|
||||
if '-' in pip_info:
|
||||
# 提取版本号
|
||||
parts = pip_info.split('-', 1)
|
||||
if len(parts) == 2:
|
||||
actual_package = parts[0]
|
||||
|
||||
cache_key = f"packages/{actual_filename}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 尝试从上游获取
|
||||
# 格式: /pypi/packages/hash/filename -> 构造上游URL
|
||||
possible_urls = []
|
||||
|
||||
# 获取基础URL（去掉 /simple 后缀）
|
||||
base_url = self.upstream_url.rstrip('/')
|
||||
if base_url.endswith('/simple'):
|
||||
base_url = base_url[:-7]
|
||||
|
||||
# 新格式: hash/filename -> 尝试清华源
|
||||
# 使用 actual_filename（已解析的纯文件路径）
|
||||
possible_urls.append(f"{base_url}/packages/{actual_filename}")
|
||||
|
||||
# 尝试官方源
|
||||
possible_urls.append(f"https://files.pythonhosted.org/packages/{actual_filename}")
|
||||
|
||||
# 如果有查询参数中的URL，也尝试
|
||||
if 'url' in query_params:
|
||||
possible_urls.insert(0, urllib.parse.unquote(query_params['url'][0]))
|
||||
|
||||
data = None
|
||||
last_error = None
|
||||
|
||||
for url in possible_urls:
|
||||
try:
|
||||
import sys
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('User-Agent', 'PyPI-Mirror/1.0')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=60) as response:
|
||||
data = response.read()
|
||||
break # 成功获取，退出循环
|
||||
except Exception as e:
|
||||
import sys
|
||||
last_error = e
|
||||
continue
|
||||
|
||||
if data is None:
|
||||
handler.send_error(502, f"Failed to fetch package: {last_error}")
|
||||
return False
|
||||
|
||||
if True:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
def _handle_web_api(self, handler, package: str) -> bool:
|
||||
"""处理Web API请求"""
|
||||
import sys
|
||||
package = package.lower()
|
||||
cache_key = f"web/{package}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/vnd.pypi.simple.v1+json; charset=utf-8')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取 - 需要去掉 /simple 后缀
|
||||
base_url = self.upstream_url.rstrip('/')
|
||||
if base_url.endswith('/simple'):
|
||||
base_url = base_url[:-7]
|
||||
url = f"{base_url}/pypi/{package}/json"
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('Accept', 'application/json')
|
||||
req.add_header('User-Agent', 'PyPI-Mirror/1.0')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
data = response.read().decode('utf-8')
|
||||
data_json = json.loads(data)
|
||||
|
||||
# 转换URL
|
||||
data_json = self._convert_package_json(package, data_json)
|
||||
|
||||
proxy_data = json.dumps(data_json)
|
||||
|
||||
if True:
|
||||
self._set_cache(cache_key, proxy_data.encode('utf-8'))
|
||||
|
||||
proxy_bytes = proxy_data.encode('utf-8')
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/vnd.pypi.simple.v1+json; charset=utf-8')
|
||||
handler.send_header('Content-Length', str(len(proxy_bytes)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(proxy_bytes)
|
||||
return True
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
handler.send_error(502, f"Failed to fetch package info: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_package_download(self, handler, filename: str) -> bool:
|
||||
"""处理包下载请求"""
|
||||
import sys
|
||||
# 使用 packages/ 前缀，保持标准 PyPI 目录结构
|
||||
cache_key = f"packages/{filename}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取 - 注意去掉 /simple 后缀
|
||||
base_url = self.upstream_url.rstrip('/')
|
||||
if base_url.endswith('/simple'):
|
||||
base_url = base_url[:-7] # 去掉 /simple
|
||||
url = f"{base_url}/packages/{filename}"
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('User-Agent', 'PyPI-Mirror/1.0')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=120) as response:
|
||||
data = response.read()
|
||||
|
||||
if True:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(502, f"Failed to download: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_legacy(self, handler, path: str) -> bool:
|
||||
"""处理旧版PyPI兼容"""
|
||||
handler.send_error(410, "Legacy PyPI API is deprecated")
|
||||
return False
|
||||
|
||||
def _convert_simple_html(self, package: str, html: str) -> str:
|
||||
"""转换Simple API HTML，替换URL为代理地址"""
|
||||
import urllib.parse
|
||||
import sys
|
||||
import os
|
||||
# 写入调试文件
|
||||
debug_file = '/tmp/pypi_debug.log'
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[PyPI] Converting HTML for package: {package}\n")
|
||||
# 打印前几个链接用于调试
|
||||
import re
|
||||
test_matches = re.findall(r'href="([^"]+)"', html)[:3]
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[PyPI] Sample links: {test_matches}\n")
|
||||
|
||||
def convert_absolute_url(match):
|
||||
"""转换绝对URL为代理链接"""
|
||||
original_url = match.group(1) if match.lastindex else match.group(0)
|
||||
# 提取文件名和完整的hash路径
|
||||
# 格式: https://pypi.tuna.tsinghua.edu.cn/packages/hash1/hash2/fullhash/filename
|
||||
# 我们将其转换为: /pypi/packages/hash1/hash2/fullhash/filename#pip=<package>
|
||||
parts = original_url.rsplit('/', 1)
|
||||
if len(parts) == 2:
|
||||
path_part = parts[0]
|
||||
filename = parts[1]
|
||||
# 提取从 packages/ 后面的完整路径（包含完整hash）
|
||||
try:
|
||||
pkg_idx = path_part.index('/packages/')
|
||||
hash_path = path_part[pkg_idx + 10:] # 去掉 /packages/
|
||||
except ValueError:
|
||||
hash_path = filename
|
||||
return f'href="/pypi/packages/{hash_path}/{filename}#pip={package}-{filename.split("-")[1] if "-" in filename else ""}"'
|
||||
return match.group(0)
|
||||
|
||||
# 替换绝对URL - pypi.tuna.tsinghua.edu.cn (清华源)
|
||||
html = re.sub(
|
||||
r'(https://pypi\.tuna\.tsinghua\.edu\.cn/packages/[^"\']+)',
|
||||
convert_absolute_url,
|
||||
html
|
||||
)
|
||||
# 替换绝对URL - mirrors.tuna.tsinghua.edu.cn
|
||||
html = re.sub(
|
||||
r'(https://mirrors\.tuna\.tsinghua\.edu\.cn/pypi/packages/[^"\']+)',
|
||||
convert_absolute_url,
|
||||
html
|
||||
)
|
||||
# 替换绝对URL - files.pypi.org
|
||||
html = re.sub(
|
||||
r'(https://files\.pypi\.org/packages/[^"\']+)',
|
||||
convert_absolute_url,
|
||||
html
|
||||
)
|
||||
# 替换绝对URL - files.pythonhosted.org
|
||||
html = re.sub(
|
||||
r'(https://files\.pythonhosted\.org/packages/[^"\']+)',
|
||||
convert_absolute_url,
|
||||
html
|
||||
)
|
||||
|
||||
# 替换相对路径链接 - ../../packages/hash1/hash2/fullhash/filename -> /pypi/packages/hash1/hash2/fullhash/filename#pip=...
|
||||
def convert_relative_match(match):
|
||||
"""转换相对路径链接"""
|
||||
href = match.group(1)
|
||||
debug_file = '/tmp/pypi_debug.log'
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[CONVERT] Input href: {href[:80]}...\n")
|
||||
# 提取文件名
|
||||
filename = href.split('/')[-1].split('#')[0]
|
||||
# 提取完整的hash路径（从 packages/ 后面的所有部分除了文件名）
|
||||
parts = href.split('/')
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[CONVERT] Parts: {parts}\n")
|
||||
try:
|
||||
pkg_idx = parts.index('packages')
|
||||
# packages 后面到倒数第二个是 hash 路径，最后一个是文件名
|
||||
hash_parts = parts[pkg_idx+1:-1] # 除了最后一个（文件名）
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[CONVERT] hash_parts: {hash_parts}\n")
|
||||
hash_path = '/'.join(hash_parts) if hash_parts else filename
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[CONVERT] hash_path: {hash_path}\n")
|
||||
except ValueError:
|
||||
hash_path = filename
|
||||
with open(debug_file, 'a') as f:
|
||||
f.write(f"[CONVERT] ValueError, hash_path: {hash_path}\n")
|
||||
|
||||
# 提取版本号 - 从文件名中提取，如 Flask-0.1.tar.gz -> 0.1
|
||||
base_name = filename
|
||||
# 去掉扩展名
|
||||
for ext in ['.tar.gz', '.whl', '.tar.bz2', '.tar.xz']:
|
||||
if base_name.endswith(ext):
|
||||
base_name = base_name[:-len(ext)]
|
||||
break
|
||||
|
||||
# 尝试多种大小写组合来去掉包名前缀
|
||||
version = base_name
|
||||
for pkg_name in [package, package.lower(), package.upper(), package.capitalize()]:
|
||||
if base_name.lower().startswith(pkg_name.lower() + '-'):
|
||||
version = base_name[len(pkg_name)+1:]
|
||||
break
|
||||
|
||||
return f'href="/pypi/packages/{hash_path}/{filename}#egg={package}-{version}"'
|
||||
|
||||
# 匹配相对路径的链接
|
||||
html = re.sub(
|
||||
r'href="(\.\./\.\./packages/[^"]+)"',
|
||||
convert_relative_match,
|
||||
html
|
||||
)
|
||||
html = re.sub(
|
||||
r"href='(\.\./\.\./packages/[^']+)'",
|
||||
convert_relative_match,
|
||||
html
|
||||
)
|
||||
|
||||
return html
|
||||
|
||||
def _convert_package_json(self, package: str, data: dict) -> dict:
|
||||
"""转换Package JSON，替换URL为代理地址"""
|
||||
# 转换URL函数
|
||||
def convert_url(url):
|
||||
# 处理 files.pythonhosted.org 和 files.pypi.org
|
||||
# 格式: https://files.pythonhosted.org/packages/<hash1>/<hash2>/<完整哈希>/<filename>
|
||||
# 例如: https://files.pythonhosted.org/packages/ec/f9/7f9263c5695f4bd0023734af91bedb2ff8209e8de6ead162f35d8dc762fd/flask-3.1.2-py3-none-any.whl
|
||||
if 'files.pythonhosted.org' in url or 'files.pypi.org' in url:
|
||||
path_parts = url.split('/packages/')
|
||||
if len(path_parts) >= 2:
|
||||
# 直接使用 /packages/ 后的完整路径
|
||||
path_after_packages = path_parts[1]
|
||||
return f'/pypi/packages/{path_after_packages}'
|
||||
return url
|
||||
|
||||
# 转换urls
|
||||
if 'urls' in data:
|
||||
for item in data['urls']:
|
||||
if 'url' in item:
|
||||
item['url'] = convert_url(item['url'])
|
||||
|
||||
return data
|
||||
|
||||
def _fetch(self, url: str) -> Optional[bytes]:
|
||||
"""从URL获取数据"""
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('User-Agent', 'PyPI-Mirror/1.0')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return response.read()
|
||||
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _get_cache(self, cache_key: str) -> Optional[bytes]:
|
||||
"""获取缓存"""
|
||||
if not True:
|
||||
return None
|
||||
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
if not os.path.exists(cache_path):
|
||||
return None
|
||||
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f)
|
||||
if time.time() > meta.get('expires', 0):
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
with open(cache_path, 'rb') as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _set_cache(self, cache_key: str, data: bytes):
|
||||
"""设置缓存"""
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
||||
|
||||
try:
|
||||
with open(cache_path, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
meta = {
|
||||
'cached_at': time.time(),
|
||||
'expires': time.time() + 86400,
|
||||
'size': len(data)
|
||||
}
|
||||
|
||||
with open(meta_path, 'w') as f:
|
||||
json.dump(meta, f)
|
||||
|
||||
except Exception as e:
|
||||
print(f"PyPI缓存写入失败: {e}")
|
||||
|
||||
def _get_cache_path(self, cache_key: str) -> str:
|
||||
"""获取缓存路径"""
|
||||
# cache_key 格式: packages/fe/df/88ccbee.../filename
|
||||
# 存储路径: downloads/pypi-cn/packages/fe/df/88ccbee.../filename
|
||||
# 确保使用正斜杠
|
||||
safe_key = cache_key.replace('\\', '/')
|
||||
return os.path.join(self.storage_dir, safe_key)
|
||||
|
||||
def get_cache_stats(self) -> dict:
|
||||
"""获取缓存统计"""
|
||||
if not os.path.exists(self.storage_dir):
|
||||
return {'files': 0, 'size': 0}
|
||||
|
||||
total_size = 0
|
||||
file_count = 0
|
||||
|
||||
for root, dirs, files in os.walk(self.storage_dir):
|
||||
for f in files:
|
||||
if not f.endswith('.meta'):
|
||||
file_count += 1
|
||||
total_size += os.path.getsize(os.path.join(root, f))
|
||||
|
||||
return {
|
||||
'files': file_count,
|
||||
'size': total_size,
|
||||
'size_formatted': self._format_size(total_size)
|
||||
}
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
"""格式化文件大小"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
units = ["B", "KB", "MB", "GB"]
|
||||
i = 0
|
||||
while size_bytes >= 1024 and i < len(units) - 1:
|
||||
size_bytes /= 1024.0
|
||||
i += 1
|
||||
|
||||
return f"{size_bytes:.2f} {units[i]}"
|
||||
+384
@@ -0,0 +1,384 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
YUM/DNF镜像代理处理器
|
||||
支持RHEL/CentOS/Rocky/AlmaLinux软件源
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import gzip
|
||||
import xml.etree.ElementTree as ET
|
||||
import urllib.request
|
||||
from typing import Dict, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class YUMMirror:
|
||||
"""YUM/DNF镜像代理"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
|
||||
# 配置 - 使用 storage_dir（基于 base_dir）
|
||||
self.base_url = config.get('base_url', 'http://mirror.centos.org/centos')
|
||||
self.storage_dir = config.get('storage_dir', './downloads/yum')
|
||||
self.base_dir = config.get('base_dir', './downloads')
|
||||
self.repo_id = config.get('repo_id', 'baseos')
|
||||
self.arch = config.get('arch', 'x86_64')
|
||||
|
||||
# 确保存储目录存在
|
||||
os.makedirs(self.storage_dir, exist_ok=True)
|
||||
|
||||
def handle_request(self, handler, path: str) -> bool:
|
||||
"""
|
||||
处理YUM请求
|
||||
路径格式: /centos/7/updates/x86_64/repodata/repomd.xml
|
||||
"""
|
||||
try:
|
||||
parts = path.strip('/').split('/')
|
||||
|
||||
if len(parts) < 3:
|
||||
return self._handle_index(handler)
|
||||
|
||||
distro = parts[0] # centos, rocky, alma
|
||||
version = parts[1] # 7, 8, 9
|
||||
repo = parts[2] # baseos, appstream, updates
|
||||
rest = '/'.join(parts[3:])
|
||||
|
||||
# 确定文件类型
|
||||
if 'repomd.xml' in rest:
|
||||
return self._handle_repomd(handler, distro, version, repo)
|
||||
elif 'primary.xml.gz' in rest:
|
||||
return self._handle_primary(handler, distro, version, repo, 'primary')
|
||||
elif 'filelists.xml.gz' in rest:
|
||||
return self._handle_filelists(handler, distro, version, repo, 'filelists')
|
||||
elif 'other.xml.gz' in rest:
|
||||
return self._handle_other(handler, distro, version, repo, 'other')
|
||||
else:
|
||||
return self._handle_repo_file(handler, distro, version, repo, rest)
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(500, str(e))
|
||||
return False
|
||||
|
||||
def _handle_index(self, handler) -> bool:
|
||||
"""处理索引请求"""
|
||||
handler.send_json_response({
|
||||
'base_url': self.base_url,
|
||||
'repo_id': self.repo_id,
|
||||
'arch': self.arch,
|
||||
'cache_stats': self.get_cache_stats()
|
||||
})
|
||||
return True
|
||||
|
||||
def _handle_repomd(self, handler, distro: str, version: str, repo: str) -> bool:
|
||||
"""处理repomd.xml请求"""
|
||||
cache_key = f"repomd:{distro}:{version}:{repo}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/xml')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 从上游获取
|
||||
url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/xml')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(502, f"Failed to fetch repomd: {str(e)}")
|
||||
return False
|
||||
|
||||
def _handle_primary(self, handler, distro: str, version: str, repo: str, db_type: str) -> bool:
|
||||
"""处理primary.xml.gz"""
|
||||
cache_key = f"primary:{distro}:{version}:{repo}:{self.arch}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/x-gzip')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 先获取repomd.xml找到对应的数据库文件
|
||||
repomd_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml"
|
||||
|
||||
try:
|
||||
repomd_data = self._fetch(repomd_url)
|
||||
|
||||
# 解析repomd.xml找到primary文件
|
||||
root = ET.fromstring(repomd_data)
|
||||
ns = {'repomd': 'http://linux.duke.edu/metadata/repo'}
|
||||
|
||||
data_location = None
|
||||
for elem in root.findall('.//repomd:data', ns):
|
||||
if elem.get('type') == 'primary':
|
||||
data_location = elem.find('repomd:location', ns).get('href')
|
||||
break
|
||||
|
||||
if data_location:
|
||||
db_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/{data_location}"
|
||||
data = self._fetch(db_url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/x-gzip')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
handler.send_error(502, "Failed to fetch primary database")
|
||||
return False
|
||||
|
||||
def _handle_filelists(self, handler, distro: str, version: str, repo: str, db_type: str) -> bool:
|
||||
"""处理filelists.xml.gz"""
|
||||
cache_key = f"filelists:{distro}:{version}:{repo}:{self.arch}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/x-gzip')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 先获取repomd.xml找到对应的数据库文件
|
||||
repomd_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml"
|
||||
|
||||
try:
|
||||
repomd_data = self._fetch(repomd_url)
|
||||
|
||||
# 解析repomd.xml找到filelists文件
|
||||
root = ET.fromstring(repomd_data)
|
||||
ns = {'repomd': 'http://linux.duke.edu/metadata/repo'}
|
||||
|
||||
data_location = None
|
||||
for elem in root.findall('.//repomd:data', ns):
|
||||
if elem.get('type') == 'filelists':
|
||||
data_location = elem.find('repomd:location', ns).get('href')
|
||||
break
|
||||
|
||||
if data_location:
|
||||
db_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/{data_location}"
|
||||
data = self._fetch(db_url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/x-gzip')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
handler.send_error(502, "Failed to fetch filelists database")
|
||||
return False
|
||||
|
||||
def _handle_other(self, handler, distro: str, version: str, repo: str, db_type: str) -> bool:
|
||||
"""处理other.xml.gz"""
|
||||
cache_key = f"other:{distro}:{version}:{repo}:{self.arch}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/x-gzip')
|
||||
handler.send_header('Content-Length', str(len(cached)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
# 先获取repomd.xml找到对应的数据库文件
|
||||
repomd_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml"
|
||||
|
||||
try:
|
||||
repomd_data = self._fetch(repomd_url)
|
||||
|
||||
# 解析repomd.xml找到other文件
|
||||
root = ET.fromstring(repomd_data)
|
||||
ns = {'repomd': 'http://linux.duke.edu/metadata/repo'}
|
||||
|
||||
data_location = None
|
||||
for elem in root.findall('.//repomd:data', ns):
|
||||
if elem.get('type') == 'other':
|
||||
data_location = elem.find('repomd:location', ns).get('href')
|
||||
break
|
||||
|
||||
if data_location:
|
||||
db_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/{data_location}"
|
||||
data = self._fetch(db_url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/x-gzip')
|
||||
handler.send_header('Content-Length', str(len(data)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
handler.send_error(502, "Failed to fetch other database")
|
||||
return False
|
||||
|
||||
def _handle_repo_file(self, handler, distro: str, version: str, repo: str, path: str) -> bool:
|
||||
"""处理仓库中的其他文件"""
|
||||
cache_key = f"file:{distro}:{version}:{repo}:{path.replace('/', ':')}"
|
||||
|
||||
cached = self._get_cache(cache_key)
|
||||
if cached:
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(cached)
|
||||
return True
|
||||
|
||||
url = f"{self.base_url}/{version}/{repo}/{self.arch}/{path}"
|
||||
|
||||
try:
|
||||
data = self._fetch(url)
|
||||
|
||||
if self.cache_enabled:
|
||||
self._set_cache(cache_key, data)
|
||||
|
||||
handler.send_response(200)
|
||||
handler.send_header('Content-Type', 'application/octet-stream')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
handler.send_error(404, f"File not found: {str(e)}")
|
||||
return False
|
||||
|
||||
def _fetch(self, url: str) -> Optional[bytes]:
|
||||
"""从URL获取数据"""
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header('User-Agent', 'YUM-Mirror/1.0')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return response.read()
|
||||
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _get_cache(self, cache_key: str) -> Optional[bytes]:
|
||||
"""获取缓存"""
|
||||
if not self.cache_enabled:
|
||||
return None
|
||||
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
if not os.path.exists(cache_path):
|
||||
return None
|
||||
|
||||
if os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path, 'r') as f:
|
||||
meta = json.load(f)
|
||||
if time.time() > meta.get('expires', 0):
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
with open(cache_path, 'rb') as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _set_cache(self, cache_key: str, data: bytes):
|
||||
"""设置缓存"""
|
||||
cache_path = self._get_cache_path(cache_key)
|
||||
meta_path = cache_path + '.meta'
|
||||
|
||||
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
||||
|
||||
try:
|
||||
with open(cache_path, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
meta = {
|
||||
'cached_at': time.time(),
|
||||
'expires': time.time() + self.cache_ttl,
|
||||
'size': len(data)
|
||||
}
|
||||
|
||||
with open(meta_path, 'w') as f:
|
||||
json.dump(meta, f)
|
||||
|
||||
except Exception as e:
|
||||
print(f"YUM缓存写入失败: {e}")
|
||||
|
||||
def _get_cache_path(self, cache_key: str) -> str:
|
||||
"""获取缓存路径"""
|
||||
subdir = cache_key[:2]
|
||||
return os.path.join(self.storage_dir, subdir, cache_key)
|
||||
|
||||
def get_cache_stats(self) -> dict:
|
||||
"""获取缓存统计"""
|
||||
if not os.path.exists(self.storage_dir):
|
||||
return {'files': 0, 'size': 0}
|
||||
|
||||
total_size = 0
|
||||
file_count = 0
|
||||
|
||||
for root, dirs, files in os.walk(self.storage_dir):
|
||||
for f in files:
|
||||
if not f.endswith('.meta'):
|
||||
file_count += 1
|
||||
total_size += os.path.getsize(os.path.join(root, f))
|
||||
|
||||
return {
|
||||
'files': file_count,
|
||||
'size': total_size,
|
||||
'size_formatted': self._format_size(total_size)
|
||||
}
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
"""格式化文件大小"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
units = ["B", "KB", "MB", "GB"]
|
||||
i = 0
|
||||
while size_bytes >= 1024 and i < len(units) - 1:
|
||||
size_bytes /= 1024.0
|
||||
i += 1
|
||||
|
||||
return f"{size_bytes:.2f} {units[i]}"
|
||||
@@ -0,0 +1,310 @@
|
||||
# -*- mode: python ; coding: utf-8 -*-
|
||||
"""
|
||||
PyInstaller 打包配置
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 获取项目根目录
|
||||
def get_abs_path(path):
|
||||
"""获取绝对路径"""
|
||||
if os.path.isabs(path):
|
||||
return path
|
||||
return os.path.join(os.getcwd(), path)
|
||||
|
||||
a = Analysis(
|
||||
['main.py'],
|
||||
pathex=[os.getcwd()],
|
||||
binaries=[],
|
||||
datas=[
|
||||
(get_abs_path('api'), 'api'),
|
||||
(get_abs_path('core'), 'core'),
|
||||
(get_abs_path('handlers'), 'handlers'),
|
||||
(get_abs_path('mirrors'), 'mirrors'),
|
||||
(get_abs_path('settings.json'), '.'),
|
||||
],
|
||||
hiddenimports=[
|
||||
# aiohttp
|
||||
'aiohttp',
|
||||
'aiohttp.abc',
|
||||
'aiohttp.connector',
|
||||
'aiohttp.client',
|
||||
'aiohttp.client_proto',
|
||||
'aiohttp.client_reqrep',
|
||||
'aiohttp.cookiejar',
|
||||
'aiohttp.fileresponse',
|
||||
'aiohttp.hdrs',
|
||||
'aiohttp.http_exceptions',
|
||||
'aiohttp.http_parser',
|
||||
'aiohttp.http_protocol',
|
||||
'aiohttp.http_server',
|
||||
'aiohttp.http_websocket',
|
||||
'aiohttp.multipart',
|
||||
'aiohttp.payload',
|
||||
'aiohttp.proxy',
|
||||
'aiohttp.streams',
|
||||
'aiohttp.tcp_helpers',
|
||||
'aiohttp.web',
|
||||
'aiohttp.web_app',
|
||||
'aiohttp.web_exceptions',
|
||||
'aiohttp.web_fileresponse',
|
||||
'aiohttp.web_log',
|
||||
'aiohttp.web_middlewares',
|
||||
'aiohttp.web_protocol',
|
||||
'aiohttp.web_request',
|
||||
'aiohttp.web_response',
|
||||
'aiohttp.web_routedef',
|
||||
'aiohttp.router',
|
||||
'aiohttp.web_server',
|
||||
'aiohttp.web_urldispatcher',
|
||||
'aiohttp.web_ws',
|
||||
'aiohttp.websocket',
|
||||
'aiohttp.websocket_client',
|
||||
'aiohttp.websocket_handshake',
|
||||
'aiohttp.websocket_protocol',
|
||||
# aiohttp.multipart
|
||||
'multipart',
|
||||
'email.generator',
|
||||
'email.policy',
|
||||
'email.utils',
|
||||
# sqlalchemy
|
||||
'sqlalchemy',
|
||||
'sqlalchemy.engine',
|
||||
'sqlalchemy.engine.base',
|
||||
'sqlalchemy.engine.cursor',
|
||||
'sqlalchemy.engine.default',
|
||||
'sqlalchemy.engine.interfaces',
|
||||
'sqlalchemy.engine.row',
|
||||
'sqlalchemy.engine.util',
|
||||
'sqlalchemy.orm',
|
||||
'sqlalchemy.orm.base',
|
||||
'sqlalchemy.orm.clear_mappers',
|
||||
'sqlalchemy.orm.decl_api',
|
||||
'sqlalchemy.orm.declarative',
|
||||
'sqlalchemy.orm.descriptor_props',
|
||||
'sqlalchemy.orm.interfaces',
|
||||
'sqlalchemy.orm.mapper',
|
||||
'sqlalchemy.orm.properties',
|
||||
'sqlalchemy.orm.query',
|
||||
'sqlalchemy.orm.relationships',
|
||||
'sqlalchemy.orm.session',
|
||||
'sqlalchemy.orm.strategies',
|
||||
'sqlalchemy.orm.strategy_options',
|
||||
'sqlalchemy.pool',
|
||||
'sqlalchemy.pool.base',
|
||||
'sqlalchemy.pool.impl',
|
||||
'sqlalchemy.dialects',
|
||||
'sqlalchemy.dialects.sqlite',
|
||||
'sqlalchemy.dialects.postgresql',
|
||||
'sqlalchemy.dialects.mysql',
|
||||
# apscheduler
|
||||
'apscheduler',
|
||||
'apscheduler.schedulers',
|
||||
'apscheduler.schedulers.background',
|
||||
'apscheduler.schedulers.base',
|
||||
'apscheduler.schedulers.blocking',
|
||||
'apscheduler.triggers',
|
||||
'apscheduler.triggers.cron',
|
||||
'apscheduler.triggers.date',
|
||||
'apscheduler.triggers.interval',
|
||||
'apscheduler.executors',
|
||||
'apscheduler.executors.pool',
|
||||
'apscheduler.job',
|
||||
'apscheduler.events',
|
||||
# paramiko
|
||||
'paramiko',
|
||||
'paramiko.transport',
|
||||
'paramiko.client',
|
||||
'paramiko.config',
|
||||
'paramiko.hostkeys',
|
||||
'paramiko.message',
|
||||
'paramiko.packet',
|
||||
'paramiko.primitives',
|
||||
'paramiko.util',
|
||||
'paramiko.ecdh',
|
||||
'paramiko.ed25519key',
|
||||
'paramiko.rsakey',
|
||||
'paramiko.dsskey',
|
||||
'paramiko.pkey',
|
||||
'paramiko.agent',
|
||||
'paramiko.sftp',
|
||||
'paramiko.sftp_attr',
|
||||
'paramiko.sftp_handle',
|
||||
'paramiko.sftp_server',
|
||||
'paramiko.sftp_si',
|
||||
'paramiko.ssh_exception',
|
||||
'paramiko.auth_handler',
|
||||
'paramiko.channel',
|
||||
'paramiko.shell',
|
||||
'paramiko.exec',
|
||||
# requests
|
||||
'requests',
|
||||
'requests.api',
|
||||
'requests.auth',
|
||||
'requests.certs',
|
||||
'requests.compat',
|
||||
'requests.cookies',
|
||||
'requests.exceptions',
|
||||
'requests.hooks',
|
||||
'requests.models',
|
||||
'requests.sessions',
|
||||
'requests.status_codes',
|
||||
'requests.structures',
|
||||
'requests.utils',
|
||||
# pymysql
|
||||
'pymysql',
|
||||
'pymysql.connections',
|
||||
'pymysql.cursors',
|
||||
'pymysql.converters',
|
||||
'pymysql.err',
|
||||
'pymysql.optionfile',
|
||||
'pymysql.protocol',
|
||||
'pymysql.times',
|
||||
'pymysql.constants',
|
||||
'pymysql.charset',
|
||||
# psycopg2
|
||||
'psycopg2',
|
||||
'psycopg2._psycopg2',
|
||||
'psycopg2.sql',
|
||||
'psycopg2.extensions',
|
||||
'psycopg2.extras',
|
||||
# cryptography
|
||||
'cryptography',
|
||||
'cryptography.hazmat',
|
||||
'cryptography.hazmat.backends',
|
||||
'cryptography.hazmat.primitives',
|
||||
'cryptography.hazmat.primitives.asymmetric',
|
||||
'cryptography.hazmat.primitives.ciphers',
|
||||
'cryptography.hazmat.primitives.hashes',
|
||||
'cryptography.hazmat.primitives.serialization',
|
||||
'cryptography.hazmat.primitives.kdf.pbkdf2',
|
||||
'cryptography.x509',
|
||||
'cryptography.x509.name',
|
||||
'cryptography.x509.extensions',
|
||||
'cryptography.fernet',
|
||||
# 其他常用
|
||||
'urllib3',
|
||||
'urllib3.util',
|
||||
'urllib3.exceptions',
|
||||
'urllib3.request',
|
||||
'urllib3.response',
|
||||
'urllib3.poolmanager',
|
||||
'charset_normalizer',
|
||||
'charset_normalizer.utils',
|
||||
'charset_normalizer.models',
|
||||
'idna',
|
||||
'idna.core',
|
||||
'idna.idnadata',
|
||||
'cachetools',
|
||||
'croniter',
|
||||
'psutil',
|
||||
'psutil._psutil_linux',
|
||||
'psutil._psutil_osx',
|
||||
'psutil._psutil_windows',
|
||||
'python_multipart',
|
||||
'python_multipart.multipart',
|
||||
'dataclasses',
|
||||
'inspect',
|
||||
'concurrent.futures',
|
||||
'subprocess',
|
||||
'threading',
|
||||
'socket',
|
||||
'ssl',
|
||||
'hmac',
|
||||
'hashlib',
|
||||
'base64',
|
||||
'struct',
|
||||
'tempfile',
|
||||
'shutil',
|
||||
'pickle',
|
||||
'json',
|
||||
'os',
|
||||
'sys',
|
||||
'signal',
|
||||
'time',
|
||||
'datetime',
|
||||
'math',
|
||||
'logging',
|
||||
'warnings',
|
||||
'ctypes',
|
||||
'ctypes.util',
|
||||
'ctypes.wintypes',
|
||||
'_ctypes',
|
||||
'_socket',
|
||||
'_ssl',
|
||||
'select',
|
||||
'errno',
|
||||
'fcntl',
|
||||
'grp',
|
||||
'pwd',
|
||||
'termios',
|
||||
'tty',
|
||||
'fcntl',
|
||||
'resource',
|
||||
'msvcrt',
|
||||
'win32api',
|
||||
'win32con',
|
||||
'win32file',
|
||||
'pywintypes',
|
||||
'pythoncom',
|
||||
# pyasn1 (paramiko 依赖)
|
||||
'pyasn1',
|
||||
'pyasn1.type',
|
||||
'pyasn1.codec',
|
||||
'pyasn1.codec.der',
|
||||
'pyasn1.codec.der.encoder',
|
||||
'pyasn1.codec.der.decoder',
|
||||
'pyasn1.util',
|
||||
'pyasn1.error',
|
||||
# bcrypt (cryptography 依赖)
|
||||
'bcrypt',
|
||||
'bcrypt.__about__',
|
||||
# cffi (cryptography 依赖)
|
||||
'cffi',
|
||||
'cffi.api',
|
||||
'cffi.ffiplatform',
|
||||
'cffi.backend_ctypes',
|
||||
'cffi.recompiler',
|
||||
'cffi.verifier',
|
||||
# packaging (cryptography 依赖)
|
||||
'packaging',
|
||||
'packaging.version',
|
||||
'packaging.specifiers',
|
||||
'packaging.requirements',
|
||||
# six (paramiko 依赖)
|
||||
'six',
|
||||
# ipaddress (paramiko 依赖)
|
||||
'ipaddress',
|
||||
# json (标准库)
|
||||
'json.decoder',
|
||||
'json.encoder',
|
||||
# typing
|
||||
'typing_extensions',
|
||||
],
|
||||
hookspath=[],
|
||||
hooksconfig={},
|
||||
runtime_hooks=[],
|
||||
excludes=[],
|
||||
win_no_prefer_redirects=False,
|
||||
win_private_assemblies=False,
|
||||
cipher=None,
|
||||
noarchive=False,
|
||||
)
|
||||
|
||||
pyz = PYZ(a.pure, a.zipped_data, cipher=None)
|
||||
|
||||
# 使用 EXE() 创建 --onefile 单文件模式
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
name='hyc-download',
|
||||
debug=False,
|
||||
bootloader_ignore_signals=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
upx_exclude=[],
|
||||
console=True,
|
||||
)
|
||||
@@ -0,0 +1,19 @@
|
||||
psutil>=5.9.0
|
||||
pyinstaller>=6.0.0
|
||||
python-multipart>=0.0.6
|
||||
paramiko>=2.11.0
|
||||
aiohttp>=3.9.0
|
||||
cachetools>=5.3.0
|
||||
requests>=2.31.0
|
||||
croniter>=2.0.0
|
||||
|
||||
# Database (SQLite 内置, 以下为可选)
|
||||
sqlalchemy>=2.0.0
|
||||
apscheduler>=3.10.0
|
||||
|
||||
# PostgreSQL 驱动
|
||||
psycopg2-binary>=2.9.0
|
||||
|
||||
# MySQL 驱动
|
||||
pymysql>=1.1.0
|
||||
cryptography>=41.0.0
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/bin/bash
|
||||
# ============================================
|
||||
# HYC下载站 v2.3 - 多架构 Docker 构建脚本
|
||||
# 支持: amd64, arm64, arm/v7, i386
|
||||
# ============================================
|
||||
|
||||
set -e
|
||||
|
||||
# 颜色
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
VERSION=${1:-v2.3}
|
||||
REGISTRY=${2:-hyc-download-station}
|
||||
TAG=${3:-$VERSION}
|
||||
|
||||
# 支持的平台
|
||||
PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7,linux/386"
|
||||
|
||||
echo "========================================"
|
||||
echo " HYC下载站 v2.3 - 多架构构建"
|
||||
echo "========================================"
|
||||
echo " 版本: $TAG"
|
||||
echo " 平台: $PLATFORMS"
|
||||
echo
|
||||
|
||||
# 检查 Docker
|
||||
if ! command -v docker &> /dev/null; then
|
||||
echo -e "${RED}✗ Docker 未安装${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查 BuildX
|
||||
if ! docker buildx version &> /dev/null; then
|
||||
echo -e "${YELLOW}⚠ Docker BuildX 未安装，尝试安装...${NC}"
|
||||
docker buildx install || true
|
||||
fi
|
||||
|
||||
# 创建构建器
|
||||
echo "[1/4] 创建构建器..."
|
||||
docker buildx rm hyc-builder 2>/dev/null || true
|
||||
docker buildx create \
|
||||
--name hyc-builder \
|
||||
--platform $PLATFORMS \
|
||||
--use
|
||||
|
||||
# 引导构建器
|
||||
echo "[2/4] 引导构建器..."
|
||||
docker buildx inspect --bootstrap
|
||||
|
||||
# 构建镜像
|
||||
echo "[3/4] 构建镜像..."
|
||||
echo " 构建中，请稍候..."
|
||||
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
docker buildx build \
|
||||
--platform $PLATFORMS \
|
||||
--tag "$REGISTRY:$TAG" \
|
||||
--tag "$REGISTRY:latest" \
|
||||
--push \
|
||||
--file docker/Dockerfile.multiarch \
|
||||
.
|
||||
|
||||
END_TIME=$(date +%s)
|
||||
BUILD_TIME=$((END_TIME - START_TIME))
|
||||
|
||||
echo -e "${GREEN}✓ 构建完成 (${BUILD_TIME}秒)${NC}"
|
||||
|
||||
# 推送清单
|
||||
echo "[4/4] 创建清单..."
|
||||
docker manifest create "$REGISTRY:$TAG" \
|
||||
"$REGISTRY:$TAG-amd64" \
|
||||
"$REGISTRY:$TAG-arm64" \
|
||||
"$REGISTRY:$TAG-arm-v7" \
|
||||
"$REGISTRY:$TAG-386" 2>/dev/null || true
|
||||
|
||||
docker manifest push "$REGISTRY:$TAG" || true
|
||||
|
||||
echo
|
||||
echo "========================================"
|
||||
echo -e "${GREEN}✓ 多架构构建完成${NC}"
|
||||
echo "========================================"
|
||||
echo
|
||||
echo "镜像标签:"
|
||||
echo " - $REGISTRY:$TAG"
|
||||
echo " - $REGISTRY:latest"
|
||||
echo
|
||||
echo "推送到仓库:"
|
||||
echo " docker pull $REGISTRY:$TAG"
|
||||
echo
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/bin/bash
|
||||
# ============================================
|
||||
# HYC下载站 v2.3 - 系统兼容性检查脚本
|
||||
# 支持: X86_32, X86_64, ARMv7, ARM64, MIPS 等
|
||||
# ============================================
|
||||
|
||||
set -e
|
||||
|
||||
# 颜色
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo "========================================"
|
||||
echo " HYC下载站 v2.3 - 兼容性检查"
|
||||
echo "========================================"
|
||||
echo
|
||||
|
||||
# 检测架构
|
||||
ARCH=$(uname -m)
|
||||
OS=$(uname -s)
|
||||
echo "系统信息:"
|
||||
echo " - 架构: $ARCH"
|
||||
echo " - 系统: $OS"
|
||||
echo
|
||||
|
||||
# 架构特定配置
|
||||
case "$ARCH" in
|
||||
x86_64)
|
||||
echo "✓ x86_64 (64位) - 完全支持"
|
||||
RECOMMENDED_WORKERS=4
|
||||
;;
|
||||
i386|i686)
|
||||
echo "⚠ i386/i686 (32位) - 有限支持"
|
||||
RECOMMENDED_WORKERS=2
|
||||
echo " 建议: --workers 2"
|
||||
;;
|
||||
armv7l|armv7hl)
|
||||
echo "✓ ARMv7 (32位) - 完全支持"
|
||||
RECOMMENDED_WORKERS=2
|
||||
;;
|
||||
armv8l|aarch64)
|
||||
echo "✓ ARM64 (64位) - 完全支持"
|
||||
RECOMMENDED_WORKERS=4
|
||||
;;
|
||||
armv6l)
|
||||
echo "⚠ ARMv6 - 有限支持 (可能需要重新编译)"
|
||||
RECOMMENDED_WORKERS=1
|
||||
;;
|
||||
mips*)
|
||||
echo "⚠ MIPS - 实验性支持"
|
||||
RECOMMENDED_WORKERS=1
|
||||
;;
|
||||
*)
|
||||
echo "⚠ 未知架构: $ARCH - 需要测试"
|
||||
RECOMMENDED_WORKERS=1
|
||||
;;
|
||||
esac
|
||||
echo
|
||||
|
||||
# Python 版本检查
|
||||
PYTHON_VERSION=$(python3 --version 2>&1 || echo "未安装")
|
||||
echo "Python 版本: $PYTHON_VERSION"
|
||||
|
||||
PYTHON_MAJOR=$(python3 -c 'import sys; print(sys.version_info.major)' 2>/dev/null || echo "0")
|
||||
PYTHON_MINOR=$(python3 -c 'import sys; print(sys.version_info.minor)' 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$PYTHON_MAJOR" -ge 3 ] && [ "$PYTHON_MINOR" -ge 8 ]; then
|
||||
echo "✓ Python 3.8+ - 支持"
|
||||
else
|
||||
echo "✗ 需要 Python 3.8 或更高版本"
|
||||
exit 1
|
||||
fi
|
||||
echo
|
||||
|
||||
# 内存检查
|
||||
echo "内存检查:"
|
||||
if command -v free &> /dev/null; then
|
||||
TOTAL_MEM=$(free -m | awk '/^Mem:/{print $2}')
|
||||
echo " - 总内存: ${TOTAL_MEM}MB"
|
||||
|
||||
if [ "$TOTAL_MEM" -lt 256 ]; then
|
||||
echo "⚠ 内存低于 256MB - 使用 ultra_low 预设"
|
||||
echo " python main.py --preset ultra_low"
|
||||
elif [ "$TOTAL_MEM" -lt 512 ]; then
|
||||
echo "⚠ 内存 256-512MB - 使用 low 预设"
|
||||
echo " python main.py --preset low"
|
||||
elif [ "$TOTAL_MEM" -lt 1024 ]; then
|
||||
echo "✓ 内存 512MB-1GB - 使用 medium 预设 (默认)"
|
||||
else
|
||||
echo "✓ 内存 1GB+ - 使用 high 预设 (默认)"
|
||||
fi
|
||||
else
|
||||
echo " (无法检测，使用默认配置)"
|
||||
fi
|
||||
echo
|
||||
|
||||
# 磁盘空间检查
|
||||
echo "磁盘空间检查:"
|
||||
DISK_FREE=$(df -BG . | awk '/^\//{print $4}' | sed 's/G//')
|
||||
echo " - 可用: ${DISK_FREE}GB"
|
||||
|
||||
if [ "$DISK_FREE" -lt 1 ]; then
|
||||
echo "⚠ 可用空间不足 1GB"
|
||||
fi
|
||||
echo
|
||||
|
||||
# 必需依赖检查
|
||||
echo "必需依赖检查:"
|
||||
MISSING_DEPS=""
|
||||
|
||||
check_dep() {
|
||||
if command -v "$1" &> /dev/null; then
|
||||
echo " ✓ $1"
|
||||
else
|
||||
echo " ✗ $1 (必需)"
|
||||
MISSING_DEPS="$MISSING_DEPS $1"
|
||||
fi
|
||||
}
|
||||
|
||||
check_dep "python3"
|
||||
echo
|
||||
|
||||
# 可选依赖检查
|
||||
echo "可选依赖检查:"
|
||||
check_opt() {
|
||||
if command -v "$1" &> /dev/null; then
|
||||
echo " ✓ $1"
|
||||
else
|
||||
echo " ⚠ $1 (可选)"
|
||||
fi
|
||||
}
|
||||
|
||||
check_opt "psutil" || pip3 install psutil
|
||||
check_opt "sqlalchemy" || pip3 install sqlalchemy
|
||||
echo
|
||||
|
||||
# Docker 构建检查 (如果使用 Docker)
|
||||
if [ -f "../Dockerfile" ]; then
|
||||
echo "Docker 构建检查:"
|
||||
if command -v docker &> /dev/null; then
|
||||
echo " ✓ Docker 可用"
|
||||
|
||||
# 检查 buildx 支持
|
||||
if docker buildx version &> /dev/null; then
|
||||
echo " ✓ Docker BuildX 支持多架构构建"
|
||||
echo
|
||||
echo "多架构构建命令:"
|
||||
echo " docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t hyc:v2.3 ."
|
||||
fi
|
||||
else
|
||||
echo " ⚠ Docker 不可用 (需要安装 Docker)"
|
||||
fi
|
||||
echo
|
||||
fi
|
||||
|
||||
# 推荐启动命令
|
||||
echo "========================================"
|
||||
echo " 推荐启动命令"
|
||||
echo "========================================"
|
||||
|
||||
# 根据检测结果生成推荐命令
|
||||
CMD="python main.py --preset "
|
||||
|
||||
if [ "$TOTAL_MEM" -lt 256 ]; then
|
||||
CMD="${CMD}ultra_low"
|
||||
elif [ "$TOTAL_MEM" -lt 512 ]; then
|
||||
CMD="${CMD}low"
|
||||
elif [ "$TOTAL_MEM" -lt 1024 ]; then
|
||||
CMD="${CMD}medium"
|
||||
else
|
||||
CMD="${CMD}high"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "$CMD"
|
||||
echo
|
||||
|
||||
# 完整命令示例
|
||||
echo "完整示例:"
|
||||
echo " $CMD --host 0.0.0.0 --port 8080 --base-dir ./downloads"
|
||||
echo
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
平台检测工具
|
||||
自动检测系统架构并推荐配置
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# 添加项目根目录
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from core.optimization import (
|
||||
ArchitectureDetector, LowMemoryConfig, check_compatibility
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print(" HYC下载站 v2.3 - 平台检测")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# 架构检测
|
||||
print("[1/3] 架构信息")
|
||||
arch = ArchitectureDetector.get_architecture()
|
||||
for key, value in arch.items():
|
||||
print(f" {key}: {value}")
|
||||
print()
|
||||
|
||||
# 兼容性检查
|
||||
print("[2/3] 兼容性检查")
|
||||
compat = check_compatibility()
|
||||
print(f" 兼容状态: {'✓ 通过' if compat['compatible'] else '✗ 有问题'}")
|
||||
if compat['warnings']:
|
||||
print(" 警告:")
|
||||
for w in compat['warnings']:
|
||||
print(f" ⚠ {w}")
|
||||
print()
|
||||
|
||||
# 推荐配置
|
||||
print("[3/3] 推荐配置")
|
||||
recommended = ArchitectureDetector.get_recommended_config()
|
||||
for key, value in recommended.items():
|
||||
print(f" {key}: {value}")
|
||||
|
||||
# 低内存配置
|
||||
low_mem = LowMemoryConfig('auto')
|
||||
status = low_mem.get_status()
|
||||
print()
|
||||
print(" 自动检测预设:")
|
||||
print(f" - 预设: {status['preset']}")
|
||||
print(f" - 描述: {status['description']}")
|
||||
print()
|
||||
print(" 建议启动命令:")
|
||||
cmd = f"python main.py --preset {status['preset']}"
|
||||
if 'ultra_low' in str(status):
|
||||
cmd += " --disable-ws --disable-sse"
|
||||
print(f" {cmd}")
|
||||
print()
|
||||
|
||||
# 返回状态码
|
||||
return 0 if compat['compatible'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
MC核心目录结构迁移脚本
|
||||
将旧结构迁移到新结构：
|
||||
旧: MCServerCore/{core}/{major}/{file.jar}
|
||||
新: MCServerCore/{core}/{major}/{version}/{file.jar}
|
||||
|
||||
版本号提取规则：
|
||||
- paper-1.20.1-6550.jar -> 1.20.1
|
||||
- spongeneo-1.21.2-21.2.0-beta-13.0.0-RC1868-universal.jar -> 1.21.2
|
||||
- spongevanilla-1.20.6-11.0.1-RC1916-universal.jar -> 1.20.6
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def extract_version(filename, core_name):
|
||||
"""
|
||||
从文件名中提取版本号
|
||||
例如: spongevanilla-1.20.6-11.0.1-RC1916-universal.jar -> 1.20.6
|
||||
例如: craftbukkit-1.9.jar -> 1.9
|
||||
例如: spongevanilla-1.17-9.0.0-RC668-universal.jar -> 1.17
|
||||
"""
|
||||
# 去掉 .jar 后缀
|
||||
name_without_jar = filename[:-4] if filename.endswith('.jar') else filename
|
||||
|
||||
# 模式：核心名-版本号-其他
|
||||
# 版本号格式：数字.数字 或 数字.数字.数字 (如 1.9, 1.20.6, 1.21.2)
|
||||
pattern = rf'^{re.escape(core_name)}-(\d+\.\d+(?:\.\d+)?)'
|
||||
match = re.match(pattern, name_without_jar)
|
||||
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def migrate_mc_structure(base_dir='./downloads/MCServerCore'):
|
||||
"""迁移MC核心目录结构"""
|
||||
mc_dir = Path(base_dir)
|
||||
|
||||
if not mc_dir.exists():
|
||||
print(f"目录不存在: {mc_dir}")
|
||||
return
|
||||
|
||||
migrated_count = 0
|
||||
skipped_count = 0
|
||||
errors = []
|
||||
|
||||
for jar_file in mc_dir.rglob('*.jar'):
|
||||
try:
|
||||
relative = jar_file.relative_to(mc_dir)
|
||||
parts = relative.parts
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# 格式: core/major/file.jar 或 core/major/version/file.jar
|
||||
if len(parts) >= 3:
|
||||
core_name = parts[0]
|
||||
major = parts[1]
|
||||
|
||||
# 如果已经是新结构（len >= 4），跳过
|
||||
if len(parts) >= 4:
|
||||
print(f" 已在正确位置: {relative}")
|
||||
continue
|
||||
|
||||
# 旧结构：core/major/file.jar
|
||||
# 提取版本号
|
||||
version = extract_version(jar_file.name, core_name)
|
||||
|
||||
if not version:
|
||||
print(f" 无法提取版本号: {relative}")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
# 构建新路径: core/major/version/file.jar
|
||||
new_version_dir = jar_file.parent / version
|
||||
new_path = new_version_dir / jar_file.name
|
||||
|
||||
# 如果目标目录已存在同名文件
|
||||
if new_path.exists():
|
||||
print(f" 目标已存在，跳过: {relative}")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
new_version_dir.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(jar_file), str(new_path))
|
||||
print(f" 移动: {relative} -> {new_path.relative_to(mc_dir)}")
|
||||
migrated_count += 1
|
||||
except Exception as e:
|
||||
errors.append(f"移动失败 {relative}: {e}")
|
||||
else:
|
||||
print(f" 跳过（未知结构）: {relative}")
|
||||
skipped_count += 1
|
||||
|
||||
# 尝试清理空的目录
|
||||
cleaned_dirs = 0
|
||||
for dir_path in sorted(mc_dir.rglob('*'), reverse=True):
|
||||
if dir_path.is_dir():
|
||||
try:
|
||||
if not any(dir_path.iterdir()):
|
||||
dir_path.rmdir()
|
||||
cleaned_dirs += 1
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
print(f"\n迁移完成！")
|
||||
print(f" 移动文件: {migrated_count}")
|
||||
print(f" 跳过文件: {skipped_count}")
|
||||
print(f" 清理空目录: {cleaned_dirs}")
|
||||
|
||||
if errors:
|
||||
print(f"\n错误:")
|
||||
for e in errors[:10]:
|
||||
print(f" {e}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
|
||||
base_dir = sys.argv[1] if len(sys.argv) > 1 else './downloads/MCServerCore'
|
||||
|
||||
print("=" * 60)
|
||||
print("MC核心目录结构迁移工具")
|
||||
print("旧结构: MCServerCore/{core}/{major}/{file.jar}")
|
||||
print("新结构: MCServerCore/{core}/{major}/{version}/{file.jar}")
|
||||
print("=" * 60)
|
||||
print(f"基础目录: {base_dir}")
|
||||
print()
|
||||
|
||||
migrate_mc_structure(base_dir)
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
# ============================================
|
||||
# HYC下载站 v2.3 - QEMU 仿真支持设置
|
||||
# 用于在 x86_64 上构建 ARM 镜像
|
||||
# ============================================
|
||||
|
||||
set -e
|
||||
|
||||
echo "========================================"
|
||||
echo " QEMU 仿真支持设置"
|
||||
echo "========================================"
|
||||
|
||||
# 检查是否需要设置
|
||||
if [ "$(uname -m)" = "x86_64" ]; then
|
||||
echo "检测到 x86_64 平台"
|
||||
echo "将启用 ARM/i386 仿真支持"
|
||||
fi
|
||||
|
||||
# 启用 QEMU 仿真
|
||||
echo
|
||||
echo "[1/3] 检查 QEMU ..."
|
||||
|
||||
# 注册 ARM 二进制格式
|
||||
if [ -f /usr/bin/qemu-arm-static ]; then
|
||||
echo "✓ qemu-arm-static 已安装"
|
||||
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes || true
|
||||
elif [ -f /usr/bin/qemu-arm ]; then
|
||||
echo "✓ qemu-arm 已安装"
|
||||
else
|
||||
echo "⚠ QEMU 未安装，正在安装..."
|
||||
apt-get update && apt-get install -y qemu-user-static || \
|
||||
yum install -y qemu-user-static || \
|
||||
apk add --no-cache qemu-arm qemu-i386
|
||||
fi
|
||||
|
||||
# 设置 binfmt-misc
|
||||
echo
|
||||
echo "[2/3] 注册二进制格式..."
|
||||
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes 2>/dev/null || \
|
||||
echo " (可能需要手动配置)"
|
||||
|
||||
echo
|
||||
echo "[3/3] 验证设置..."
|
||||
echo "可用的仿真架构:"
|
||||
|
||||
for arch in arm arm64 i386; do
|
||||
if [ -f "/usr/bin/qemu-$arch-static" ] || [ -f "/usr/bin/qemu-$arch" ]; then
|
||||
echo " ✓ $arch"
|
||||
else
|
||||
echo " ✗ $arch"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "========================================"
|
||||
echo "✓ QEMU 设置完成"
|
||||
echo "========================================"
|
||||
echo
|
||||
echo "现在可以构建多架构镜像:"
|
||||
echo " ./scripts/build-multiarch.sh"
|
||||
echo
|
||||
+601
@@ -0,0 +1,601 @@
|
||||
{
|
||||
"server_name": "HYC下载站",
|
||||
"host": "0.0.0.0",
|
||||
"port": 8080,
|
||||
"base_dir": "./downloads",
|
||||
"api_version": "v2",
|
||||
"directory_listing": true,
|
||||
"enable_stats": true,
|
||||
"show_hash": true,
|
||||
"ignore_hidden": true,
|
||||
"max_upload_size": 1073741824,
|
||||
"timeout": 30,
|
||||
"enable_range": true,
|
||||
"max_workers": 10,
|
||||
"auth_type": "basic",
|
||||
"auth_user": "admin",
|
||||
"auth_pass": "admin123",
|
||||
"auth_token": "admin_token_123456",
|
||||
"auth_session_timeout": 3600,
|
||||
"auth_cookie_max_age": 86400,
|
||||
"ip_whitelist_enabled": false,
|
||||
"ip_whitelist": [],
|
||||
"ip_blacklist": [],
|
||||
"enable_ws": true,
|
||||
"enable_sse": true,
|
||||
"enable_monitor": true,
|
||||
"monitor_interval": 3,
|
||||
"enable_sync": true,
|
||||
"sync_config": null,
|
||||
"sync_sources": {},
|
||||
"enable_mirrors": true,
|
||||
"database": {
|
||||
"enabled": true,
|
||||
"type": "sqlite",
|
||||
"sync_interval": 60,
|
||||
"sqlite": {
|
||||
"path": "./data/hyc.db"
|
||||
},
|
||||
"postgresql": {
|
||||
"enabled": false,
|
||||
"host": "localhost",
|
||||
"port": 5432,
|
||||
"database": "hyc",
|
||||
"table_prefix": "hyc_",
|
||||
"user": "postgres",
|
||||
"password": "",
|
||||
"pool_size": 5,
|
||||
"max_overflow": 10,
|
||||
"ssl_mode": "prefer",
|
||||
"connection_timeout": 30
|
||||
},
|
||||
"mysql": {
|
||||
"enabled": false,
|
||||
"host": "localhost",
|
||||
"port": 3306,
|
||||
"database": "hyc",
|
||||
"table_prefix": "hyc_",
|
||||
"user": "root",
|
||||
"password": "",
|
||||
"pool_size": 5,
|
||||
"max_overflow": 10,
|
||||
"charset": "utf8mb4",
|
||||
"connection_timeout": 30
|
||||
},
|
||||
"external": {
|
||||
"enabled": false,
|
||||
"connection_string": "",
|
||||
"table_prefix": "hyc_",
|
||||
"pool_size": 5,
|
||||
"max_overflow": 10
|
||||
}
|
||||
},
|
||||
"mirrors": {
|
||||
"docker-cn": {
|
||||
"type": "docker",
|
||||
"url": "https://docker.m.daocloud.io",
|
||||
"target": "docker",
|
||||
"enabled": false,
|
||||
"storage_dir": "docker-cn",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Docker 镜像 - DaoCloud",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "interval",
|
||||
"interval": {
|
||||
"hours": 6
|
||||
}
|
||||
}
|
||||
},
|
||||
"docker": {
|
||||
"type": "docker",
|
||||
"url": "https://registry-1.docker.io",
|
||||
"target": "docker",
|
||||
"enabled": false,
|
||||
"storage_dir": "docker",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Docker 官方镜像",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "interval",
|
||||
"interval": {
|
||||
"hours": 6
|
||||
}
|
||||
}
|
||||
},
|
||||
"ubuntu": {
|
||||
"type": "apt",
|
||||
"url": "http://archive.ubuntu.com/ubuntu",
|
||||
"target": "apt/ubuntu",
|
||||
"enabled": false,
|
||||
"storage_dir": "ubuntu",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Ubuntu 软件源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"ubuntu-cn": {
|
||||
"type": "apt",
|
||||
"url": "https://mirrors.tuna.tsinghua.edu.cn/ubuntu",
|
||||
"target": "apt/ubuntu-cn",
|
||||
"enabled": false,
|
||||
"storage_dir": "ubuntu-cn",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Ubuntu 镜像 - 清华源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"debian": {
|
||||
"type": "apt",
|
||||
"url": "http://deb.debian.org/debian",
|
||||
"target": "apt/debian",
|
||||
"enabled": false,
|
||||
"storage_dir": "debian",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Debian 软件源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"debian-cn": {
|
||||
"type": "apt",
|
||||
"url": "https://mirrors.tuna.tsinghua.edu.cn/debian",
|
||||
"target": "apt/debian-cn",
|
||||
"enabled": false,
|
||||
"storage_dir": "debian-cn",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Debian 镜像 - 清华源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"centos": {
|
||||
"type": "yum",
|
||||
"url": "http://mirror.centos.org/centos",
|
||||
"target": "yum/centos",
|
||||
"enabled": false,
|
||||
"storage_dir": "centos",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "CentOS 软件源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"almalinux": {
|
||||
"type": "yum",
|
||||
"url": "https://repo.almalinux.org/vault",
|
||||
"target": "yum/almalinux",
|
||||
"enabled": false,
|
||||
"storage_dir": "almalinux",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "AlmaLinux 软件源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"archlinux": {
|
||||
"type": "http",
|
||||
"url": "https://geo.mirror.pkgbuild.com",
|
||||
"target": "pacman",
|
||||
"enabled": false,
|
||||
"storage_dir": "archlinux",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Arch Linux 镜像",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"pypi": {
|
||||
"type": "pypi",
|
||||
"url": "https://pypi.org",
|
||||
"target": "pypi",
|
||||
"enabled": false,
|
||||
"storage_dir": "pypi",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "PyPI 官方源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"pypi-cn": {
|
||||
"type": "pypi",
|
||||
"url": "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"target": "pypi-cn",
|
||||
"enabled": false,
|
||||
"storage_dir": "pypi-cn",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "PyPI 镜像 - 清华源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
},
|
||||
"updated_at": "2026-02-18T15:37:32.588006"
|
||||
},
|
||||
"npm": {
|
||||
"type": "npm",
|
||||
"url": "https://registry.npmjs.org",
|
||||
"target": "npm",
|
||||
"enabled": false,
|
||||
"storage_dir": "npm",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "NPM 官方源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"npm-cn": {
|
||||
"type": "npm",
|
||||
"url": "https://registry.npmmirror.com",
|
||||
"target": "npm-cn",
|
||||
"enabled": false,
|
||||
"storage_dir": "npm-cn",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "NPM 镜像 - 阿里源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"go": {
|
||||
"type": "go",
|
||||
"url": "https://proxy.golang.org",
|
||||
"target": "go",
|
||||
"enabled": false,
|
||||
"storage_dir": "go",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Go 模块代理",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"go-cn": {
|
||||
"type": "go",
|
||||
"url": "https://goproxy.cn",
|
||||
"target": "go-cn",
|
||||
"enabled": false,
|
||||
"storage_dir": "go-cn",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Go 模块代理 - 七牛云",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"maven": {
|
||||
"type": "http",
|
||||
"url": "https://repo.maven.apache.org/maven2",
|
||||
"target": "maven",
|
||||
"enabled": false,
|
||||
"storage_dir": "maven",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Maven 中央仓库",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"maven-cn": {
|
||||
"type": "http",
|
||||
"url": "https://maven.aliyun.com/repository/public",
|
||||
"target": "maven-cn",
|
||||
"enabled": false,
|
||||
"storage_dir": "maven-cn",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Maven 镜像 - 阿里云",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"gradle": {
|
||||
"type": "http",
|
||||
"url": "https://services.gradle.org/distributions",
|
||||
"target": "gradle",
|
||||
"enabled": false,
|
||||
"storage_dir": "gradle",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Gradle 分发包",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"rubygems": {
|
||||
"type": "http",
|
||||
"url": "https://rubygems.org",
|
||||
"target": "gem",
|
||||
"enabled": false,
|
||||
"storage_dir": "rubygems",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "RubyGems 官方源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"rubygems-cn": {
|
||||
"type": "http",
|
||||
"url": "https://gems.ruby-china.com",
|
||||
"target": "gem-cn",
|
||||
"enabled": false,
|
||||
"storage_dir": "rubygems-cn",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "RubyGems 镜像 - Ruby China",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"cargo": {
|
||||
"type": "http",
|
||||
"url": "https://crates.io",
|
||||
"target": "cargo",
|
||||
"enabled": false,
|
||||
"storage_dir": "cargo",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Crates.io Rust 包",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"cargo-cn": {
|
||||
"type": "http",
|
||||
"url": "https://mirrors.tuna.tsinghua.edu.cn/crates.io-index",
|
||||
"target": "cargo-cn",
|
||||
"enabled": false,
|
||||
"storage_dir": "cargo-cn",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "Cargo 镜像 - 清华源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"nuget": {
|
||||
"type": "http",
|
||||
"url": "https://api.nuget.org/v3/index.json",
|
||||
"target": "nuget",
|
||||
"enabled": false,
|
||||
"storage_dir": "nuget",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "NuGet 官方源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"cocoapods": {
|
||||
"type": "http",
|
||||
"url": "https://cdn.cocoapods.org",
|
||||
"target": "cocoapods",
|
||||
"enabled": false,
|
||||
"storage_dir": "cocoapods",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "CocoaPods 官方源",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"cran": {
|
||||
"type": "http",
|
||||
"url": "https://cloud.r-project.org",
|
||||
"target": "cran",
|
||||
"enabled": false,
|
||||
"storage_dir": "cran",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "CRAN R 包镜像",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"ctan": {
|
||||
"type": "http",
|
||||
"url": "https://ctan.math.illinois.edu",
|
||||
"target": "ctan",
|
||||
"enabled": false,
|
||||
"storage_dir": "ctan",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "CTAN LaTeX 包",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
},
|
||||
"nvidia-cuda": {
|
||||
"type": "http",
|
||||
"url": "https://developer.download.nvidia.com/compute/cuda/repos",
|
||||
"target": "cuda",
|
||||
"enabled": false,
|
||||
"storage_dir": "cuda",
|
||||
"auto_sync": false,
|
||||
"custom": true,
|
||||
"description": "NVIDIA CUDA 仓库",
|
||||
"schedule": {
|
||||
"enabled": false,
|
||||
"type": "cron",
|
||||
"cron": "0 4 * * *"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scheduled_tasks": {
|
||||
"cleanup_db": {
|
||||
"type": "cron",
|
||||
"config": {
|
||||
"cron": "0 2 * * *",
|
||||
"enabled": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"health_check": {
|
||||
"enabled": true,
|
||||
"interval": 60,
|
||||
"timeout": 10,
|
||||
"max_retries": 3,
|
||||
"failure_threshold": 3
|
||||
},
|
||||
"failover": {
|
||||
"enabled": false,
|
||||
"threshold": 3
|
||||
},
|
||||
"alerts": {
|
||||
"enabled": false,
|
||||
"email": {
|
||||
"enabled": false,
|
||||
"smtp_host": "smtp.example.com",
|
||||
"smtp_port": 587,
|
||||
"smtp_user": "",
|
||||
"smtp_password": "",
|
||||
"from_address": "[email protected]",
|
||||
"to_addresses": [],
|
||||
"use_tls": true
|
||||
},
|
||||
"webhook": {
|
||||
"enabled": false,
|
||||
"url": ""
|
||||
},
|
||||
"rules": {
|
||||
"disk_high": {
|
||||
"enabled": true,
|
||||
"threshold": 80,
|
||||
"severity": "warning"
|
||||
},
|
||||
"disk_critical": {
|
||||
"enabled": true,
|
||||
"threshold": 90,
|
||||
"severity": "critical"
|
||||
},
|
||||
"sync_failed": {
|
||||
"enabled": true,
|
||||
"severity": "error"
|
||||
},
|
||||
"source_unhealthy": {
|
||||
"enabled": true,
|
||||
"severity": "warning"
|
||||
},
|
||||
"service_down": {
|
||||
"enabled": true,
|
||||
"severity": "critical"
|
||||
}
|
||||
}
|
||||
},
|
||||
"webhooks": {
|
||||
"enabled": false,
|
||||
"storage": "webhooks.json",
|
||||
"url": ""
|
||||
},
|
||||
"auth_sessions_file": "auth_sessions.json",
|
||||
"ssl": {
|
||||
"enabled": false,
|
||||
"cert": null,
|
||||
"key": null
|
||||
},
|
||||
"rate_limit": 100,
|
||||
"debug": false,
|
||||
"debug_log_file": null,
|
||||
"debug_http": false,
|
||||
"debug_auth": false,
|
||||
"debug_api": false,
|
||||
"debug_v2": false,
|
||||
"debug_error": false,
|
||||
"debug_download": false,
|
||||
"restart": {
|
||||
"enabled": true,
|
||||
"graceful_timeout": 30,
|
||||
"shutdown_timeout": 10,
|
||||
"strategy": "graceful"
|
||||
},
|
||||
"version": "2.2",
|
||||
"sort_by": "name",
|
||||
"sort_reverse": false,
|
||||
"calculate_hash": false,
|
||||
"max_search_results": 100,
|
||||
"session_timeout": 60,
|
||||
"sync_interval": 60,
|
||||
"auto_sync": false,
|
||||
"auto_clean": false,
|
||||
"clean_interval": 7,
|
||||
"enable_browse": true,
|
||||
"show_hidden": false,
|
||||
"enable_websocket": true,
|
||||
"max_threads": 4,
|
||||
"health_check_interval": 30,
|
||||
"request_timeout": 300,
|
||||
"enable_alerts": true,
|
||||
"disk_warning_threshold": 80,
|
||||
"disk_critical_threshold": 90,
|
||||
"smtp_server": "",
|
||||
"smtp_port": 587,
|
||||
"smtp_from": "",
|
||||
"alert_email": "",
|
||||
"log_file": "./logs/hyc.log",
|
||||
"log_level": "INFO",
|
||||
"cache_size": 1073741824,
|
||||
"cache_ttl": 3600,
|
||||
"pool_size": 5,
|
||||
"max_overflow": 10
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd /root/vs1
|
||||
python3 main.py --host 0.0.0.0 --port 8080
|
||||
Reference in New Issue
Block a user