HYCEXITADMIN/mirror_server
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P1: 功能修复

Browse Source 
- v2 三处 self.format_file_size -> format_file_size(3 个端点不再必崩)
- requirements 加 bcrypt、移除未使用的 aiohttp
- verify_user 改用 bcrypt.checkpw(DB basic 登录恢复);SHA256 回退改自描述 PBKDF2,去掉明文接受
- config.py token 模式只在缺失时生成(不再每次 update 失效)
- docker/npm/go/apt/yum 镜像补 cache_enabled/cache_ttl 初始化(不再必 500)
- docker 镜像兼容 upstream_url 配置键
- 定时同步链路: 回调委托 sync_starter 并注入 _sync_manager;cron 逗号列表修复;星期偏移修复
- v2 同步方法与 v1 签名兼容;MirrorSyncManager 补 get_source_status/stop_all_tasks_for_source
This commit is contained in:
HYC Fixer
2026-08-30 12:22:27 +08:00
parent 2a899c411e
commit 084164296d
13 changed files with 2089 additions and 2018 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/v2.py
+16 -10
View File
@@ -1020,7 +1020,7 @@ class APIv2(APIv1):
"path": rel_path, "path": rel_path,
"type": mime_type, "type": mime_type,
"size": size, "size": size,
"size_formatted": self.format_file_size(size), "size_formatted": format_file_size(size),
"modified": datetime.fromtimestamp(mtime).isoformat(), "modified": datetime.fromtimestamp(mtime).isoformat(),
"match_score": self._calculate_match_score(file_name, search_term, search_mode) "match_score": self._calculate_match_score(file_name, search_term, search_mode)
} }
@@ -1222,7 +1222,7 @@ class APIv2(APIv1):
"total_files": total_files, "total_files": total_files,
"total_dirs": total_dirs, "total_dirs": total_dirs,
"total_size": total_size, "total_size": total_size,
"total_size_formatted": self.format_file_size(total_size) "total_size_formatted": format_file_size(total_size)
}, },
"file_types": dict(sorted(file_types.items(), key=lambda x: x[1], reverse=True)), "file_types": dict(sorted(file_types.items(), key=lambda x: x[1], reverse=True)),
"size_distribution": size_distribution, "size_distribution": size_distribution,
@@ -1238,7 +1238,7 @@ class APIv2(APIv1):
"largest_file": { "largest_file": {
"name": largest_file[0] if largest_file else None, "name": largest_file[0] if largest_file else None,
"size": largest_file[1] if largest_file else None, "size": largest_file[1] if largest_file else None,
"size_formatted": self.format_file_size(largest_file[1]) if largest_file else None "size_formatted": format_file_size(largest_file[1]) if largest_file else None
} }
}, },
"updated": datetime.now().isoformat() "updated": datetime.now().isoformat()
@@ -1263,7 +1263,7 @@ class APIv2(APIv1):
"path": filepath, "path": filepath,
"name": os.path.basename(filepath), "name": os.path.basename(filepath),
"size": info.st_size, "size": info.st_size,
"size_formatted": self.format_file_size(info.st_size), "size_formatted": format_file_size(info.st_size),
"modified": datetime.fromtimestamp(info.st_mtime).isoformat(), "modified": datetime.fromtimestamp(info.st_mtime).isoformat(),
"downloads": count "downloads": count
}) })
@@ -2249,8 +2249,10 @@ class APIv2(APIv1):
except Exception as e: except Exception as e:
handler.send_json_response({"error": str(e)}, 500) handler.send_json_response({"error": str(e)}, 500)
def api_start_sync(self, handler, source_name): def api_start_sync(self, handler, source_name=None):
"""启动同步""" """启动同步（兼容 v1 调用：无 source_name 时回退到 v1 行为）"""
if source_name is None:
return super().api_start_sync(handler)
if hasattr(handler, 'sync_manager') and handler.sync_manager: if hasattr(handler, 'sync_manager') and handler.sync_manager:
task_id = handler.sync_manager.start_sync(source_name) task_id = handler.sync_manager.start_sync(source_name)
if task_id: if task_id:
@@ -2268,8 +2270,10 @@ class APIv2(APIv1):
else: else:
handler.send_json_response({"error": "Sync manager not available"}, 500) handler.send_json_response({"error": "Sync manager not available"}, 500)
def api_stop_sync(self, handler, source_name): def api_stop_sync(self, handler, source_name=None):
"""停止同步""" """停止同步（兼容 v1 调用：无 source_name 时回退到 v1 行为）"""
if source_name is None:
return super().api_stop_sync(handler)
if hasattr(handler, 'sync_manager') and handler.sync_manager: if hasattr(handler, 'sync_manager') and handler.sync_manager:
handler.sync_manager.stop_all_tasks_for_source(source_name) handler.sync_manager.stop_all_tasks_for_source(source_name)
handler.send_json_response({ handler.send_json_response({
@@ -2279,8 +2283,10 @@ class APIv2(APIv1):
else: else:
handler.send_json_response({"error": "Sync manager not available"}, 500) handler.send_json_response({"error": "Sync manager not available"}, 500)
def api_get_sync_status(self, handler, source_name): def api_get_sync_status(self, handler, source_name=None):
"""获取同步状态（真实数据）""" """获取同步状态（真实数据；兼容 v1 调用）"""
if source_name is None:
return super().api_get_sync_status(handler)
# source_name 格式: source_name/status，需要提取 # source_name 格式: source_name/status，需要提取
if source_name.endswith('/status'): if source_name.endswith('/status'):
source_name = source_name[:-7] source_name = source_name[:-7]
core/config.py
+4 -3
View File
@@ -198,9 +198,10 @@ class ConfigManager:
if 'auth_pass' not in config: if 'auth_pass' not in config:
config['auth_pass'] = 'admin123' config['auth_pass'] = 'admin123'
elif auth_type == 'token': elif auth_type == 'token':
# 每次运行都重新生成标准的 token # 只在缺失时生成 token，避免每次配置验证/更新都使已签发 token 失效
import secrets if not config.get('auth_token'):
config['auth_token'] = secrets.token_hex(32) import secrets
config['auth_token'] = secrets.token_hex(32)
# 验证上传大小配置 # 验证上传大小配置
if 'max_upload_size' in config: if 'max_upload_size' in config:
core/database.py
+28 -20
View File
@@ -1220,8 +1220,8 @@ class DatabaseManager:
return True return True
return False return False
def verify_user(self, username: str, password_hash: str) -> dict: def verify_user(self, username: str, password: str) -> dict:
"""验证用户登录""" """验证用户登录（password 为明文密码，用 bcrypt 校验）"""
with self.session() as session: with self.session() as session:
user = session.query(UserRecord).filter_by(username=username).first() user = session.query(UserRecord).filter_by(username=username).first()
@@ -1236,8 +1236,8 @@ class DatabaseManager:
if not user.enabled: if not user.enabled:
return {'valid': False, 'reason': '账号已被禁用'} return {'valid': False, 'reason': '账号已被禁用'}
# 验证密码 # 验证密码（bcrypt.checkpw，恒定时间比较）
if user.password_hash == password_hash: if self.verify_password(password, user.password_hash):
# 登录成功 # 登录成功
user.last_login = time.time() user.last_login = time.time()
user.login_count = (user.login_count or 0) + 1 user.login_count = (user.login_count or 0) + 1
@@ -1291,16 +1291,22 @@ class DatabaseManager:
print(f"[数据库] 已创建默认用户: {username}") print(f"[数据库] 已创建默认用户: {username}")
def hash_password(self, password: str) -> str: def hash_password(self, password: str) -> str:
"""使用 bcrypt 加密密码""" """使用 bcrypt 加密密码（回退: 自描述格式 PBKDF2-HMAC-SHA256）"""
try: try:
import bcrypt import bcrypt
salt = bcrypt.gensalt(rounds=12) salt = bcrypt.gensalt(rounds=12)
return bcrypt.hashpw(password.encode('utf-8'), salt).decode('utf-8') return bcrypt.hashpw(password.encode('utf-8'), salt).decode('utf-8')
except ImportError: except ImportError:
# 如果 bcrypt 不可用，回退到 SHA256 # 如果 bcrypt 不可用，回退到 PBKDF2-HMAC-SHA256
# 格式: $pbkdf2-sha256$<iterations>$<salt_b64>$<hash_b64>
import hashlib import hashlib
salt = hashlib.sha256(str(time.time()).encode()).hexdigest()[:16] import os as _os
return hashlib.sha256((password + salt).encode()).hexdigest() import base64
iterations = 100_000
salt = _os.urandom(16)
dk = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, iterations)
return (f"$pbkdf2-sha256${iterations}${base64.b64encode(salt).decode()}"
f"${base64.b64encode(dk).decode()}")
def verify_password(self, password: str, password_hash: str) -> bool: def verify_password(self, password: str, password_hash: str) -> bool:
"""验证密码是否正确""" """验证密码是否正确"""
@@ -1308,19 +1314,21 @@ class DatabaseManager:
import bcrypt import bcrypt
return bcrypt.checkpw(password.encode('utf-8'), password_hash.encode('utf-8')) return bcrypt.checkpw(password.encode('utf-8'), password_hash.encode('utf-8'))
except ImportError: except ImportError:
# 如果 bcrypt 不可用，回退到 SHA256 验证 # 回退: 只接受自描述 PBKDF2 格式，绝不接受明文
import hashlib import hashlib
# 尝试直接比较（可能是旧格式） import base64
if password_hash == password: import hmac
return True if not password_hash.startswith('$pbkdf2-sha256$'):
# 尝试带 salt 验证 return False
for salt_len in range(1, 17): try:
salt = password_hash[:salt_len] if len(password_hash) > salt_len else "" _, algo, iters_s, salt_b64, hash_b64 = password_hash.split('$')
if len(salt) >= 4: iterations = int(iters_s)
test_hash = hashlib.sha256((password + salt).encode()).hexdigest() salt = base64.b64decode(salt_b64)
if test_hash == password_hash: expected = base64.b64decode(hash_b64)
return True dk = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, iterations)
return False return hmac.compare_digest(dk, expected)
except Exception:
return False
def get_user_by_token(self, token: str) -> dict: def get_user_by_token(self, token: str) -> dict:
"""通过 token 获取用户信息""" """通过 token 获取用户信息"""
core/mirror_sync.py
+9
View File
@@ -1278,6 +1278,15 @@ class MirrorSyncManager:
"""获取同步状态""" """获取同步状态"""
with self.sync_lock: with self.sync_lock:
return self.sync_status.copy() return self.sync_status.copy()
def get_source_status(self, name):
"""获取指定同步源状态"""
with self.sync_lock:
return dict(self.sync_status.get(name, {}))
def stop_all_tasks_for_source(self, name):
"""停止指定同步源的所有任务（v2 API 兼容）"""
return self.stop_sync(name)
def start(self): def start(self):
"""启动同步管理器""" """启动同步管理器"""
core/scheduler.py
+8 -2
View File
@@ -98,7 +98,10 @@ class ScheduledTask:
# 处理列表: "1,2,3" # 处理列表: "1,2,3"
if ',' in part: if ',' in part:
return self._parse_cron_part(part.replace(',', ' '), min_val, max_val) result = []
for sub in part.split(','):
result.extend(self._parse_cron_part(sub.strip(), min_val, max_val))
return result
# 处理范围: "1-5" # 处理范围: "1-5"
if '-' in part: if '-' in part:
@@ -144,12 +147,15 @@ class ScheduledTask:
if not self.cron_parts: if not self.cron_parts:
return False return False
# cron 约定: 0=周日...6=周六; datetime.weekday(): 0=周一...6=周日
cron_weekday = (dt.weekday() + 1) % 7
return ( return (
dt.minute in self.cron_parts['minute'] and dt.minute in self.cron_parts['minute'] and
dt.hour in self.cron_parts['hour'] and dt.hour in self.cron_parts['hour'] and
dt.day in self.cron_parts['day'] and dt.day in self.cron_parts['day'] and
dt.month in self.cron_parts['month'] and dt.month in self.cron_parts['month'] and
dt.weekday() in self.cron_parts['weekday'] cron_weekday in self.cron_parts['weekday']
) )
def get_next_run_time(self) -> Optional[datetime]: def get_next_run_time(self) -> Optional[datetime]:
core/server.py
+2
View File
@@ -92,6 +92,8 @@ class MirrorServer:
if self.config.get('enable_sync', True): if self.config.get('enable_sync', True):
self.sync_manager = MirrorSyncManager(self.config) self.sync_manager = MirrorSyncManager(self.config)
self.sync_manager.start() self.sync_manager.start()
# 注入到配置，供 SyncScheduler 定时回调使用
self.config['_sync_manager'] = self.sync_manager
# 创建系统监控器（仅当启用时） # 创建系统监控器（仅当启用时）
self.monitor = None self.monitor = None
core/sync_scheduler.py
+20 -2
View File
@@ -52,6 +52,9 @@ class SyncScheduler:
self.on_file_updated: Optional[Callable] = None self.on_file_updated: Optional[Callable] = None
self.on_sync_complete: Optional[Callable] = None self.on_sync_complete: Optional[Callable] = None
# 定时同步执行器（注入 MirrorSyncManager.start_sync 或等效 callable）
self.sync_starter: Optional[Callable] = None
# 待同步队列 # 待同步队列
self._pending_add = [] # 待添加的文件 self._pending_add = [] # 待添加的文件
self._pending_update = [] # 待更新的文件 self._pending_update = [] # 待更新的文件
@@ -97,10 +100,25 @@ class SyncScheduler:
"""创建同步回调函数""" """创建同步回调函数"""
def sync_callback(task_name: str, config: dict): def sync_callback(task_name: str, config: dict):
logger.info(f"定时同步任务触发: {source_name}") logger.info(f"定时同步任务触发: {source_name}")
self.start_sync(source_name) # 委托给注入的同步执行器（MirrorSyncManager.start_sync）
return True starter = self.sync_starter or self._default_sync_starter()
if starter:
try:
return bool(starter(source_name))
except Exception as e:
logger.error(f"触发同步 {source_name} 失败: {e}")
return False
logger.error(f"无法触发同步 {source_name}: 未配置同步执行器(sync_starter)")
return False
return sync_callback return sync_callback
def _default_sync_starter(self):
"""尝试从配置获取 MirrorSyncManager 实例（server.py 启动时注入）"""
mgr = self.config.get('_sync_manager')
if mgr and hasattr(mgr, 'start_sync'):
return mgr.start_sync
return None
def start(self): def start(self):
"""启动同步调度器""" """启动同步调度器"""
if self._running: if self._running:
mirrors/apt.py
+436 -432
View File
@@ -1,432 +1,436 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
APT镜像代理处理器 APT镜像代理处理器
支持Debian/Ubuntu软件源 支持Debian/Ubuntu软件源
""" """
import os import os
import json import json
import time import time
import gzip import gzip
import re import re
import urllib.request import urllib.request
from typing import Dict, List, Optional, Tuple from typing import Dict, List, Optional, Tuple
from datetime import datetime from datetime import datetime
class APTMirror: class APTMirror:
"""APT镜像代理""" """APT镜像代理"""
def __init__(self, config: dict): def __init__(self, config: dict):
self.config = config self.config = config
# 配置 - 使用 storage_dir（基于 base_dir） # 配置 - 使用 storage_dir（基于 base_dir）
self.mirrors = config.get('mirrors', [ self.mirrors = config.get('mirrors', [
'http://archive.ubuntu.com/ubuntu', 'http://archive.ubuntu.com/ubuntu',
'http://security.ubuntu.com/ubuntu' 'http://security.ubuntu.com/ubuntu'
]) ])
self.storage_dir = config.get('storage_dir', './downloads/apt') self.storage_dir = config.get('storage_dir', './downloads/apt')
self.base_dir = config.get('base_dir', './downloads') self.base_dir = config.get('base_dir', './downloads')
self.default_suite = config.get('suite', 'jammy') self.default_suite = config.get('suite', 'jammy')
self.default_components = config.get('components', ['main', 'restricted', 'universe', 'multiverse']) self.default_components = config.get('components', ['main', 'restricted', 'universe', 'multiverse'])
self.default_arch = config.get('arch', 'amd64') self.default_arch = config.get('arch', 'amd64')
# 确保存储目录存在 # 缓存配置（缺失时默认启用，避免 AttributeError）
os.makedirs(self.storage_dir, exist_ok=True) self.cache_enabled = config.get('cache_enabled', True)
self.cache_ttl = config.get('cache_ttl', 3600)
def handle_request(self, handler, path: str) -> bool:
""" # 确保存储目录存在
处理APT请求 os.makedirs(self.storage_dir, exist_ok=True)
路径格式: /ubuntu/dists/jammy/main/binary-amd64/Packages.gz
""" def handle_request(self, handler, path: str) -> bool:
try: """
# 解析路径 处理APT请求
parts = path.strip('/').split('/') 路径格式: /ubuntu/dists/jammy/main/binary-amd64/Packages.gz
"""
if len(parts) < 5: try:
# 返回镜像列表或帮助信息 # 解析路径
return self._handle_index(handler) parts = path.strip('/').split('/')
# 提取组件 if len(parts) < 5:
distro = parts[0] # ubuntu, debian 等 # 返回镜像列表或帮助信息
dist_type = parts[1] # dists return self._handle_index(handler)
suite = parts[2] # jammy, focal 等
component = parts[3] # main, updates 等 # 提取组件
rest = '/'.join(parts[4:]) distro = parts[0] # ubuntu, debian 等
dist_type = parts[1] # dists
# 确定请求类型 suite = parts[2] # jammy, focal 等
if rest.endswith('Packages.gz'): component = parts[3] # main, updates 等
return self._handle_packages(handler, distro, suite, component, rest) rest = '/'.join(parts[4:])
elif rest.endswith('Packages'):
return self._handle_packages_uncompressed(handler, distro, suite, component, rest) # 确定请求类型
elif rest.endswith('Release'): if rest.endswith('Packages.gz'):
return self._handle_release(handler, distro, suite, component, rest) return self._handle_packages(handler, distro, suite, component, rest)
elif rest.endswith('Release.gpg'): elif rest.endswith('Packages'):
return self._handle_release_gpg(handler, distro, suite, component, rest) return self._handle_packages_uncompressed(handler, distro, suite, component, rest)
elif rest.endswith('InRelease'): elif rest.endswith('Release'):
return self._handle_inrelease(handler, distro, suite, component, rest) return self._handle_release(handler, distro, suite, component, rest)
else: elif rest.endswith('Release.gpg'):
# 其他文件（源码包等） return self._handle_release_gpg(handler, distro, suite, component, rest)
return self._handle_file(handler, distro, suite, rest) elif rest.endswith('InRelease'):
return self._handle_inrelease(handler, distro, suite, component, rest)
except Exception as e: else:
handler.send_error(500, str(e)) # 其他文件（源码包等）
return False return self._handle_file(handler, distro, suite, rest)
def _handle_index(self, handler) -> bool: except Exception as e:
"""处理索引请求""" handler.send_error(500, str(e))
handler.send_json_response({ return False
'mirrors': self.mirrors,
'default_suite': self.default_suite, def _handle_index(self, handler) -> bool:
'default_components': self.default_components, """处理索引请求"""
'cache_stats': self.get_cache_stats() handler.send_json_response({
}) 'mirrors': self.mirrors,
return True 'default_suite': self.default_suite,
'default_components': self.default_components,
def _handle_packages(self, handler, distro: str, suite: str, component: str, path: str) -> bool: 'cache_stats': self.get_cache_stats()
"""处理Packages.gz请求""" })
cache_key = f"packages:{distro}:{suite}:{component}:{self.default_arch}" return True
# 检查架构 def _handle_packages(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
if 'binary-' in path: """处理Packages.gz请求"""
arch = path.split('binary-')[1].split('/')[0] cache_key = f"packages:{distro}:{suite}:{component}:{self.default_arch}"
else:
arch = self.default_arch # 检查架构
if 'binary-' in path:
cache_key = f"packages:{distro}:{suite}:{component}:{arch}" arch = path.split('binary-')[1].split('/')[0]
else:
cached = self._get_cache(cache_key) arch = self.default_arch
if cached:
handler.send_response(200) cache_key = f"packages:{distro}:{suite}:{component}:{arch}"
handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(cached))) cached = self._get_cache(cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(cached)))
# 从上游获取 handler.end_headers()
for mirror in self.mirrors: handler.wfile.write(cached)
url = f"{mirror}/{path}" return True
try: # 从上游获取
data = self._fetch(url) for mirror in self.mirrors:
if data: url = f"{mirror}/{path}"
# 缓存
if self.cache_enabled: try:
self._set_cache(cache_key, data) data = self._fetch(url)
if data:
handler.send_response(200) # 缓存
handler.send_header('Content-Type', 'application/x-gzip') if self.cache_enabled:
handler.send_header('Content-Length', str(len(data))) self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(data)))
except Exception as e: handler.end_headers()
continue handler.wfile.write(data)
return True
handler.send_error(502, "Failed to fetch from all mirrors")
return False except Exception as e:
continue
def _handle_packages_uncompressed(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
"""处理未压缩的Packages文件""" handler.send_error(502, "Failed to fetch from all mirrors")
# 先获取gz版本 return False
gz_path = path + '.gz'
def _handle_packages_uncompressed(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
for mirror in self.mirrors: """处理未压缩的Packages文件"""
url = f"{mirror}/{gz_path}" # 先获取gz版本
try: gz_path = path + '.gz'
data = self._fetch(url)
if data: for mirror in self.mirrors:
# 解压 url = f"{mirror}/{gz_path}"
packages_data = gzip.decompress(data) try:
data = self._fetch(url)
handler.send_response(200) if data:
handler.send_header('Content-Type', 'text/plain') # 解压
handler.send_header('Content-Length', str(len(packages_data))) packages_data = gzip.decompress(data)
handler.end_headers()
handler.wfile.write(packages_data) handler.send_response(200)
return True handler.send_header('Content-Type', 'text/plain')
handler.send_header('Content-Length', str(len(packages_data)))
except Exception: handler.end_headers()
continue handler.wfile.write(packages_data)
return True
handler.send_error(502, "Failed to fetch packages")
return False except Exception:
continue
def _handle_release(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
"""处理Release文件""" handler.send_error(502, "Failed to fetch packages")
cache_key = f"release:{distro}:{suite}" return False
cached = self._get_cache(cache_key) def _handle_release(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
if cached: """处理Release文件"""
handler.send_response(200) cache_key = f"release:{distro}:{suite}"
handler.send_header('Content-Type', 'text/plain')
handler.send_header('Content-Length', str(len(cached))) cached = self._get_cache(cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'text/plain')
handler.send_header('Content-Length', str(len(cached)))
# 获取Release文件 handler.end_headers()
release_path = f"/{distro}/dists/{suite}/Release" handler.wfile.write(cached)
return True
for mirror in self.mirrors:
url = mirror + release_path # 获取Release文件
try: release_path = f"/{distro}/dists/{suite}/Release"
data = self._fetch(url)
if data: for mirror in self.mirrors:
if self.cache_enabled: url = mirror + release_path
self._set_cache(cache_key, data) try:
data = self._fetch(url)
handler.send_response(200) if data:
handler.send_header('Content-Type', 'text/plain') if self.cache_enabled:
handler.send_header('Content-Length', str(len(data))) self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'text/plain')
handler.send_header('Content-Length', str(len(data)))
except Exception: handler.end_headers()
continue handler.wfile.write(data)
return True
handler.send_error(502, "Failed to fetch Release")
return False except Exception:
continue
def _handle_release_gpg(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
"""处理Release.gpg文件""" handler.send_error(502, "Failed to fetch Release")
cache_key = f"release_gpg:{distro}:{suite}" return False
cached = self._get_cache(cache_key) def _handle_release_gpg(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
if cached: """处理Release.gpg文件"""
handler.send_response(200) cache_key = f"release_gpg:{distro}:{suite}"
handler.send_header('Content-Type', 'application/pgp-signature')
handler.end_headers() cached = self._get_cache(cache_key)
handler.wfile.write(cached) if cached:
return True handler.send_response(200)
handler.send_header('Content-Type', 'application/pgp-signature')
# 尝试获取 handler.end_headers()
gpg_path = f"/{distro}/dists/{suite}/Release.gpg" handler.wfile.write(cached)
return True
for mirror in self.mirrors:
url = mirror + gpg_path # 尝试获取
try: gpg_path = f"/{distro}/dists/{suite}/Release.gpg"
data = self._fetch(url)
if data: for mirror in self.mirrors:
if self.cache_enabled: url = mirror + gpg_path
self._set_cache(cache_key, data) try:
data = self._fetch(url)
handler.send_response(200) if data:
handler.send_header('Content-Type', 'application/pgp-signature') if self.cache_enabled:
handler.end_headers() self._set_cache(cache_key, data)
handler.wfile.write(data)
return True handler.send_response(200)
handler.send_header('Content-Type', 'application/pgp-signature')
except Exception: handler.end_headers()
continue handler.wfile.write(data)
return True
handler.send_error(404, "Release.gpg not found")
return False except Exception:
continue
def _handle_inrelease(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
"""处理InRelease文件 - 获取或生成签名后的Release信息""" handler.send_error(404, "Release.gpg not found")
cache_key = f"inrelease:{distro}:{suite}" return False
cached = self._get_cache(cache_key) def _handle_inrelease(self, handler, distro: str, suite: str, component: str, path: str) -> bool:
if cached: """处理InRelease文件 - 获取或生成签名后的Release信息"""
handler.send_response(200) cache_key = f"inrelease:{distro}:{suite}"
handler.send_header('Content-Type', 'text/plain')
handler.send_header('Content-Length', str(len(cached))) cached = self._get_cache(cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'text/plain')
handler.send_header('Content-Length', str(len(cached)))
# 尝试从上游获取 InRelease handler.end_headers()
inrelease_path = f"/{distro}/dists/{suite}/InRelease" handler.wfile.write(cached)
return True
for mirror in self.mirrors:
url = mirror + inrelease_path # 尝试从上游获取 InRelease
try: inrelease_path = f"/{distro}/dists/{suite}/InRelease"
data = self._fetch(url)
if data: for mirror in self.mirrors:
if self.cache_enabled: url = mirror + inrelease_path
self._set_cache(cache_key, data) try:
data = self._fetch(url)
handler.send_response(200) if data:
handler.send_header('Content-Type', 'text/plain') if self.cache_enabled:
handler.send_header('Content-Length', str(len(data))) self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'text/plain')
except Exception: handler.send_header('Content-Length', str(len(data)))
continue handler.end_headers()
handler.wfile.write(data)
# 如果没有 InRelease，尝试生成一个（基于 Release + 方括号注释） return True
# 注意：这不是有效的签名，但可以用于不验证签名的客户端 except Exception:
release_cache_key = f"release:{distro}:{suite}" continue
release_data = self._get_cache(release_cache_key)
# 如果没有 InRelease，尝试生成一个（基于 Release + 方括号注释）
if not release_data: # 注意：这不是有效的签名，但可以用于不验证签名的客户端
# 尝试获取 Release release_cache_key = f"release:{distro}:{suite}"
release_path = f"/{distro}/dists/{suite}/Release" release_data = self._get_cache(release_cache_key)
for mirror in self.mirrors:
url = mirror + release_path if not release_data:
try: # 尝试获取 Release
release_data = self._fetch(url) release_path = f"/{distro}/dists/{suite}/Release"
if release_data: for mirror in self.mirrors:
break url = mirror + release_path
except Exception: try:
continue release_data = self._fetch(url)
if release_data:
if release_data: break
# 添加注释说明这是未签名的 Release except Exception:
comment = f"# Note: This is a synthesized InRelease (original InRelease not available)\n" continue
inrelease_data = comment + release_data.decode('utf-8', errors='replace')
if release_data:
if self.cache_enabled: # 添加注释说明这是未签名的 Release
self._set_cache(cache_key, inrelease_data.encode()) comment = f"# Note: This is a synthesized InRelease (original InRelease not available)\n"
inrelease_data = comment + release_data.decode('utf-8', errors='replace')
handler.send_response(200)
handler.send_header('Content-Type', 'text/plain') if self.cache_enabled:
handler.send_header('Content-Length', str(len(inrelease_data))) self._set_cache(cache_key, inrelease_data.encode())
handler.end_headers()
handler.wfile.write(inrelease_data.encode()) handler.send_response(200)
return True handler.send_header('Content-Type', 'text/plain')
handler.send_header('Content-Length', str(len(inrelease_data)))
handler.send_error(502, "Failed to fetch InRelease") handler.end_headers()
return False handler.wfile.write(inrelease_data.encode())
return True
def _handle_file(self, handler, distro: str, suite: str, path: str) -> bool:
"""处理普通文件请求（如源码包）""" handler.send_error(502, "Failed to fetch InRelease")
cache_key = f"file:{distro}:{path.replace('/', ':')}" return False
cached = self._get_cache(cache_key) def _handle_file(self, handler, distro: str, suite: str, path: str) -> bool:
if cached: """处理普通文件请求（如源码包）"""
handler.send_response(200) cache_key = f"file:{distro}:{path.replace('/', ':')}"
handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Length', str(len(cached))) cached = self._get_cache(cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Length', str(len(cached)))
# 从上游获取 handler.end_headers()
for mirror in self.mirrors: handler.wfile.write(cached)
url = f"{mirror}/{path}" return True
try:
data = self._fetch(url) # 从上游获取
if data: for mirror in self.mirrors:
if self.cache_enabled: url = f"{mirror}/{path}"
self._set_cache(cache_key, data) try:
data = self._fetch(url)
handler.send_response(200) if data:
handler.send_header('Content-Type', 'application/octet-stream') if self.cache_enabled:
handler.send_header('Content-Length', str(len(data))) self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Length', str(len(data)))
except Exception: handler.end_headers()
continue handler.wfile.write(data)
return True
handler.send_error(404, "File not found")
return False except Exception:
continue
def _fetch(self, url: str) -> Optional[bytes]:
"""从URL获取数据""" handler.send_error(404, "File not found")
try: return False
req = urllib.request.Request(url)
req.add_header('User-Agent', 'APT-Mirror/1.0') def _fetch(self, url: str) -> Optional[bytes]:
"""从URL获取数据"""
with urllib.request.urlopen(req, timeout=30) as response: try:
return response.read() req = urllib.request.Request(url)
req.add_header('User-Agent', 'APT-Mirror/1.0')
except Exception:
return None with urllib.request.urlopen(req, timeout=30) as response:
return response.read()
def _get_cache(self, cache_key: str) -> Optional[bytes]:
"""获取缓存""" except Exception:
if not self.cache_enabled: return None
return None
def _get_cache(self, cache_key: str) -> Optional[bytes]:
cache_path = self._get_cache_path(cache_key) """获取缓存"""
meta_path = cache_path + '.meta' if not self.cache_enabled:
return None
if not os.path.exists(cache_path):
return None cache_path = self._get_cache_path(cache_key)
meta_path = cache_path + '.meta'
# 检查过期
if os.path.exists(meta_path): if not os.path.exists(cache_path):
try: return None
with open(meta_path, 'r') as f:
meta = json.load(f) # 检查过期
if time.time() > meta.get('expires', 0): if os.path.exists(meta_path):
return None try:
except Exception: with open(meta_path, 'r') as f:
pass meta = json.load(f)
if time.time() > meta.get('expires', 0):
try: return None
with open(cache_path, 'rb') as f: except Exception:
return f.read() pass
except Exception:
return None try:
with open(cache_path, 'rb') as f:
def _set_cache(self, cache_key: str, data: bytes): return f.read()
"""设置缓存""" except Exception:
cache_path = self._get_cache_path(cache_key) return None
meta_path = cache_path + '.meta'
def _set_cache(self, cache_key: str, data: bytes):
os.makedirs(os.path.dirname(cache_path), exist_ok=True) """设置缓存"""
cache_path = self._get_cache_path(cache_key)
try: meta_path = cache_path + '.meta'
with open(cache_path, 'wb') as f:
f.write(data) os.makedirs(os.path.dirname(cache_path), exist_ok=True)
meta = { try:
'cached_at': time.time(), with open(cache_path, 'wb') as f:
'expires': time.time() + self.cache_ttl, f.write(data)
'size': len(data)
} meta = {
'cached_at': time.time(),
with open(meta_path, 'w') as f: 'expires': time.time() + self.cache_ttl,
json.dump(meta, f) 'size': len(data)
}
except Exception as e:
print(f"APT缓存写入失败: {e}") with open(meta_path, 'w') as f:
json.dump(meta, f)
def _get_cache_path(self, cache_key: str) -> str:
"""获取缓存路径""" except Exception as e:
subdir = cache_key[:2] print(f"APT缓存写入失败: {e}")
return os.path.join(self.storage_dir, subdir, cache_key)
def _get_cache_path(self, cache_key: str) -> str:
def get_cache_stats(self) -> dict: """获取缓存路径"""
"""获取缓存统计""" subdir = cache_key[:2]
if not os.path.exists(self.storage_dir): return os.path.join(self.storage_dir, subdir, cache_key)
return {'files': 0, 'size': 0}
def get_cache_stats(self) -> dict:
total_size = 0 """获取缓存统计"""
file_count = 0 if not os.path.exists(self.storage_dir):
return {'files': 0, 'size': 0}
for root, dirs, files in os.walk(self.storage_dir):
for f in files: total_size = 0
if not f.endswith('.meta'): file_count = 0
file_count += 1
total_size += os.path.getsize(os.path.join(root, f)) for root, dirs, files in os.walk(self.storage_dir):
for f in files:
return { if not f.endswith('.meta'):
'files': file_count, file_count += 1
'size': total_size, total_size += os.path.getsize(os.path.join(root, f))
'size_formatted': self._format_size(total_size)
} return {
'files': file_count,
def _format_size(self, size_bytes: int) -> str: 'size': total_size,
"""格式化文件大小""" 'size_formatted': self._format_size(total_size)
if size_bytes == 0: }
return "0 B"
def _format_size(self, size_bytes: int) -> str:
units = ["B", "KB", "MB", "GB"] """格式化文件大小"""
i = 0 if size_bytes == 0:
while size_bytes >= 1024 and i < len(units) - 1: return "0 B"
size_bytes /= 1024.0
i += 1 units = ["B", "KB", "MB", "GB"]
i = 0
return f"{size_bytes:.2f} {units[i]}" while size_bytes >= 1024 and i < len(units) - 1:
size_bytes /= 1024.0
i += 1
return f"{size_bytes:.2f} {units[i]}"
mirrors/docker.py
+364 -359
View File
@@ -1,359 +1,364 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
Docker镜像代理处理器 Docker镜像代理处理器
支持Docker Registry API v2 支持Docker Registry API v2
""" """
import os import os
import json import json
import time import time
import uuid import uuid
import urllib.request import urllib.request
import base64 import base64
import hashlib import hashlib
import hmac import hmac
from typing import Dict, List, Optional, Tuple from typing import Dict, List, Optional, Tuple
from datetime import datetime from datetime import datetime
class DockerMirror: class DockerMirror:
"""Docker镜像代理""" """Docker镜像代理"""
def __init__(self, config: dict): def __init__(self, config: dict):
self.config = config self.config = config
# 配置 - 使用 storage_dir（基于 base_dir） # 配置 - 使用 storage_dir（基于 base_dir）；兼容 v2 分支传入的 upstream_url 键
self.registry_url = config.get('registry_url', 'https://registry-1.docker.io') self.registry_url = (config.get('registry_url') or config.get('upstream_url')
self.mirror_url = config.get('mirror_url', '') or 'https://registry-1.docker.io')
self.storage_dir = config.get('storage_dir', './downloads/docker') self.mirror_url = config.get('mirror_url', '')
self.base_dir = config.get('base_dir', './downloads') self.storage_dir = config.get('storage_dir', './downloads/docker')
self.base_dir = config.get('base_dir', './downloads')
# 认证（可选）
self.username = config.get('username') # 认证（可选）
self.password = config.get('password') self.username = config.get('username')
self.password = config.get('password')
# 确保存储目录存在
os.makedirs(self.storage_dir, exist_ok=True) # 缓存配置（缺失时默认启用，避免 AttributeError）
self.cache_enabled = config.get('cache_enabled', True)
def handle_request(self, handler, path: str) -> bool: self.cache_ttl = config.get('cache_ttl', 3600)
"""
处理Docker镜像请求 # 确保存储目录存在
路径格式: /v2/library/ubuntu/tags/list 或 /v2/library/ubuntu/manifests/latest os.makedirs(self.storage_dir, exist_ok=True)
"""
try: def handle_request(self, handler, path: str) -> bool:
# 解析路径 """
parts = path.strip('/').split('/') 处理Docker镜像请求
路径格式: /v2/library/ubuntu/tags/list 或 /v2/library/ubuntu/manifests/latest
if len(parts) < 2 or parts[0] != 'v2': """
handler.send_error(400, "Invalid Docker API path") try:
return False # 解析路径
parts = path.strip('/').split('/')
# 提取组件
if parts[1] == 'library': if len(parts) < 2 or parts[0] != 'v2':
# 官方镜像 handler.send_error(400, "Invalid Docker API path")
image = 'library/' + '/'.join(parts[2:-2]) if len(parts) > 4 else 'library/' + parts[2] return False
action = parts[-2] # tags 或 manifests
reference = parts[-1] # 提取组件
else: if parts[1] == 'library':
# 非官方镜像 # 官方镜像
image = '/'.join(parts[1:-2]) image = 'library/' + '/'.join(parts[2:-2]) if len(parts) > 4 else 'library/' + parts[2]
action = parts[-2] action = parts[-2] # tags 或 manifests
reference = parts[-1] reference = parts[-1]
else:
# 根据操作类型处理 # 非官方镜像
if action == 'tags' and reference == 'list': image = '/'.join(parts[1:-2])
return self._handle_tag_list(handler, image.rstrip('/tags')) action = parts[-2]
elif action == 'manifests': reference = parts[-1]
return self._handle_manifest(handler, image, reference)
elif action == 'blobs': # 根据操作类型处理
return self._handle_blob(handler, image, reference) if action == 'tags' and reference == 'list':
elif action == 'token': return self._handle_tag_list(handler, image.rstrip('/tags'))
return self._handle_token(handler) elif action == 'manifests':
else: return self._handle_manifest(handler, image, reference)
handler.send_error(404, "Unknown action") elif action == 'blobs':
return False return self._handle_blob(handler, image, reference)
elif action == 'token':
except Exception as e: return self._handle_token(handler)
handler.send_error(500, str(e)) else:
return False handler.send_error(404, "Unknown action")
return False
def _handle_tag_list(self, handler, image: str) -> bool:
"""处理标签列表请求""" except Exception as e:
cache_key = f"tags:{image}" handler.send_error(500, str(e))
cached = self._get_cache(handler, cache_key) return False
if cached:
handler.send_response(200) def _handle_tag_list(self, handler, image: str) -> bool:
handler.send_header('Content-Type', 'application/json') """处理标签列表请求"""
handler.send_header('Content-Length', str(len(cached))) cache_key = f"tags:{image}"
handler.end_headers() cached = self._get_cache(handler, cache_key)
handler.wfile.write(cached) if cached:
return True handler.send_response(200)
handler.send_header('Content-Type', 'application/json')
# 从上游获取 handler.send_header('Content-Length', str(len(cached)))
url = f"{self.registry_url}/v2/{image}/tags/list" handler.end_headers()
handler.wfile.write(cached)
try: return True
data = self._fetch_from_upstream(url)
# 从上游获取
# 缓存 url = f"{self.registry_url}/v2/{image}/tags/list"
if self.cache_enabled:
self._set_cache(cache_key, data) try:
data = self._fetch_from_upstream(url)
handler.send_response(200)
handler.send_header('Content-Type', 'application/json') # 缓存
handler.send_header('Content-Length', str(len(data))) if self.cache_enabled:
handler.end_headers() self._set_cache(cache_key, data)
handler.wfile.write(data)
return True handler.send_response(200)
handler.send_header('Content-Type', 'application/json')
except Exception as e: handler.send_header('Content-Length', str(len(data)))
handler.send_error(500, f"Failed to fetch tags: {str(e)}") handler.end_headers()
return False handler.wfile.write(data)
return True
def _handle_manifest(self, handler, image: str, reference: str) -> bool:
"""处理清单请求""" except Exception as e:
cache_key = f"manifest:{image}:{reference}" handler.send_error(500, f"Failed to fetch tags: {str(e)}")
cached = self._get_cache(handler, cache_key) return False
if cached:
handler.send_response(200) def _handle_manifest(self, handler, image: str, reference: str) -> bool:
handler.send_header('Content-Type', 'application/vnd.docker.distribution.manifest.v2+json') """处理清单请求"""
handler.send_header('Content-Length', str(len(cached))) cache_key = f"manifest:{image}:{reference}"
handler.send_header('Docker-Content-Digest', f"sha256:{hashlib.sha256(cached).hexdigest()}") cached = self._get_cache(handler, cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/vnd.docker.distribution.manifest.v2+json')
handler.send_header('Content-Length', str(len(cached)))
# 从上游获取 handler.send_header('Docker-Content-Digest', f"sha256:{hashlib.sha256(cached).hexdigest()}")
url = f"{self.registry_url}/v2/{image}/manifests/{reference}" handler.end_headers()
handler.wfile.write(cached)
try: return True
req = urllib.request.Request(url)
req.add_header('Accept', 'application/vnd.docker.distribution.manifest.v2+json') # 从上游获取
url = f"{self.registry_url}/v2/{image}/manifests/{reference}"
if self.username and self.password:
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode() try:
req.add_header('Authorization', f"Basic {auth}") req = urllib.request.Request(url)
req.add_header('Accept', 'application/vnd.docker.distribution.manifest.v2+json')
with urllib.request.urlopen(req) as response:
data = response.read() if self.username and self.password:
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
# 缓存 req.add_header('Authorization', f"Basic {auth}")
if self.cache_enabled:
self._set_cache(cache_key, data) with urllib.request.urlopen(req) as response:
data = response.read()
digest = f"sha256:{hashlib.sha256(data).hexdigest()}"
# 缓存
handler.send_response(200) if self.cache_enabled:
handler.send_header('Content-Type', 'application/vnd.docker.distribution.manifest.v2+json') self._set_cache(cache_key, data)
handler.send_header('Content-Length', str(len(data)))
handler.send_header('Docker-Content-Digest', digest) digest = f"sha256:{hashlib.sha256(data).hexdigest()}"
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/vnd.docker.distribution.manifest.v2+json')
handler.send_header('Content-Length', str(len(data)))
except Exception as e: handler.send_header('Docker-Content-Digest', digest)
handler.send_error(500, f"Failed to fetch manifest: {str(e)}") handler.end_headers()
return False handler.wfile.write(data)
return True
def _handle_blob(self, handler, image: str, digest: str) -> bool:
"""处理Blob层下载""" except Exception as e:
# 移除 sha256: 前缀 handler.send_error(500, f"Failed to fetch manifest: {str(e)}")
if digest.startswith('sha256:'): return False
digest = digest[7:]
def _handle_blob(self, handler, image: str, digest: str) -> bool:
cache_key = f"blob:{digest}" """处理Blob层下载"""
cached = self._get_cache(handler, cache_key) # 移除 sha256: 前缀
if cached: if digest.startswith('sha256:'):
handler.send_response(200) digest = digest[7:]
handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Length', str(len(cached))) cache_key = f"blob:{digest}"
handler.send_header('Docker-Content-Digest', f"sha256:{digest}") cached = self._get_cache(handler, cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Length', str(len(cached)))
# 从上游获取 handler.send_header('Docker-Content-Digest', f"sha256:{digest}")
url = f"{self.registry_url}/v2/{image}/blobs/sha256:{digest}" handler.end_headers()
handler.wfile.write(cached)
try: return True
req = urllib.request.Request(url)
# 从上游获取
if self.username and self.password: url = f"{self.registry_url}/v2/{image}/blobs/sha256:{digest}"
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
req.add_header('Authorization', f"Basic {auth}") try:
req = urllib.request.Request(url)
with urllib.request.urlopen(req) as response:
data = response.read() if self.username and self.password:
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
# 缓存 req.add_header('Authorization', f"Basic {auth}")
if self.cache_enabled:
self._set_cache(cache_key, data) with urllib.request.urlopen(req) as response:
data = response.read()
handler.send_response(200)
handler.send_header('Content-Type', 'application/octet-stream') # 缓存
handler.send_header('Content-Length', str(len(data))) if self.cache_enabled:
handler.send_header('Docker-Content-Digest', f"sha256:{digest}") self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Length', str(len(data)))
except Exception as e: handler.send_header('Docker-Content-Digest', f"sha256:{digest}")
handler.send_error(500, f"Failed to fetch blob: {str(e)}") handler.end_headers()
return False handler.wfile.write(data)
return True
def _handle_token(self, handler) -> bool:
"""处理Token请求 - 生成真实的访问令牌""" except Exception as e:
# 解析认证信息 handler.send_error(500, f"Failed to fetch blob: {str(e)}")
auth_header = handler.headers.get('Authorization', '') return False
username = None
password = None def _handle_token(self, handler) -> bool:
"""处理Token请求 - 生成真实的访问令牌"""
if auth_header.startswith('Basic '): # 解析认证信息
try: auth_header = handler.headers.get('Authorization', '')
decoded = base64.b64decode(auth_header[6:]).decode('utf-8') username = None
username, password = decoded.split(':', 1) password = None
except Exception:
pass if auth_header.startswith('Basic '):
try:
# 验证凭据（如果有） decoded = base64.b64decode(auth_header[6:]).decode('utf-8')
if self.username and self.password: username, password = decoded.split(':', 1)
if username != self.username or password != self.password: except Exception:
handler.send_error(401, "Invalid credentials") pass
return False
# 验证凭据（如果有）
# 生成唯一的访问令牌 if self.username and self.password:
token_id = str(uuid.uuid4()) if username != self.username or password != self.password:
issued_at = int(time.time()) handler.send_error(401, "Invalid credentials")
expires_in = 300 # 5分钟 return False
expires_at = issued_at + expires_in
# 生成唯一的访问令牌
# 创建令牌信息（简化版 JWT 结构） token_id = str(uuid.uuid4())
token_data = { issued_at = int(time.time())
"iss": "hyc-mirror", expires_in = 300 # 5分钟
"sub": username or "anonymous", expires_at = issued_at + expires_in
"aud": self.registry_url,
"iat": issued_at, # 创建令牌信息（简化版 JWT 结构）
"exp": expires_at, token_data = {
"access": [ "iss": "hyc-mirror",
{"type": "repository", "actions": ["pull"]}, "sub": username or "anonymous",
{"type": "registry", "actions": ["catalog"]} "aud": self.registry_url,
] "iat": issued_at,
} "exp": expires_at,
"access": [
# 使用 HMAC-SHA256 对令牌进行简单签名 {"type": "repository", "actions": ["pull"]},
secret_key = f"hyc-mirror-{self.registry_url}".encode() {"type": "registry", "actions": ["catalog"]}
signature = hmac.new( ]
secret_key, }
f"{token_id}:{issued_at}".encode(),
hashlib.sha256 # 使用 HMAC-SHA256 对令牌进行简单签名
).hexdigest()[:32] secret_key = f"hyc-mirror-{self.registry_url}".encode()
signature = hmac.new(
full_token = f"{token_id}-{signature}" secret_key,
f"{token_id}:{issued_at}".encode(),
handler.send_json_response({ hashlib.sha256
"token": full_token, ).hexdigest()[:32]
"expires_in": expires_in,
"issued_at": issued_at full_token = f"{token_id}-{signature}"
})
return True handler.send_json_response({
"token": full_token,
def _fetch_from_upstream(self, url: str) -> bytes: "expires_in": expires_in,
"""从上游获取数据""" "issued_at": issued_at
req = urllib.request.Request(url) })
return True
if self.username and self.password:
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode() def _fetch_from_upstream(self, url: str) -> bytes:
req.add_header('Authorization', f"Basic {auth}") """从上游获取数据"""
req = urllib.request.Request(url)
with urllib.request.urlopen(req, timeout=30) as response:
return response.read() if self.username and self.password:
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
def _get_cache(self, handler, cache_key: str) -> Optional[bytes]: req.add_header('Authorization', f"Basic {auth}")
"""获取缓存"""
if not self.cache_enabled: with urllib.request.urlopen(req, timeout=30) as response:
return None return response.read()
cache_path = self._get_cache_path(cache_key) def _get_cache(self, handler, cache_key: str) -> Optional[bytes]:
meta_path = cache_path + '.meta' """获取缓存"""
if not self.cache_enabled:
if not os.path.exists(cache_path): return None
return None
cache_path = self._get_cache_path(cache_key)
# 检查是否过期 meta_path = cache_path + '.meta'
if os.path.exists(meta_path):
try: if not os.path.exists(cache_path):
with open(meta_path, 'r') as f: return None
meta = json.load(f)
if time.time() > meta.get('expires', 0): # 检查是否过期
return None if os.path.exists(meta_path):
except Exception: try:
pass with open(meta_path, 'r') as f:
meta = json.load(f)
try: if time.time() > meta.get('expires', 0):
with open(cache_path, 'rb') as f: return None
return f.read() except Exception:
except Exception: pass
return None
try:
def _set_cache(self, cache_key: str, data: bytes): with open(cache_path, 'rb') as f:
"""设置缓存""" return f.read()
cache_path = self._get_cache_path(cache_key) except Exception:
meta_path = cache_path + '.meta' return None
os.makedirs(os.path.dirname(cache_path), exist_ok=True) def _set_cache(self, cache_key: str, data: bytes):
"""设置缓存"""
try: cache_path = self._get_cache_path(cache_key)
with open(cache_path, 'wb') as f: meta_path = cache_path + '.meta'
f.write(data)
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
meta = {
'cached_at': time.time(), try:
'expires': time.time() + self.cache_ttl, with open(cache_path, 'wb') as f:
'size': len(data) f.write(data)
}
meta = {
with open(meta_path, 'w') as f: 'cached_at': time.time(),
json.dump(meta, f) 'expires': time.time() + self.cache_ttl,
'size': len(data)
except Exception as e: }
print(f"Docker缓存写入失败: {e}")
with open(meta_path, 'w') as f:
def _get_cache_path(self, cache_key: str) -> str: json.dump(meta, f)
"""获取缓存路径"""
subdir = cache_key[:2] except Exception as e:
return os.path.join(self.storage_dir, subdir, cache_key) print(f"Docker缓存写入失败: {e}")
def get_cache_stats(self) -> dict: def _get_cache_path(self, cache_key: str) -> str:
"""获取缓存统计""" """获取缓存路径"""
if not os.path.exists(self.storage_dir): subdir = cache_key[:2]
return {'files': 0, 'size': 0} return os.path.join(self.storage_dir, subdir, cache_key)
total_size = 0 def get_cache_stats(self) -> dict:
file_count = 0 """获取缓存统计"""
if not os.path.exists(self.storage_dir):
for root, dirs, files in os.walk(self.storage_dir): return {'files': 0, 'size': 0}
for f in files:
if not f.endswith('.meta'): total_size = 0
file_count += 1 file_count = 0
total_size += os.path.getsize(os.path.join(root, f))
for root, dirs, files in os.walk(self.storage_dir):
return { for f in files:
'files': file_count, if not f.endswith('.meta'):
'size': total_size, file_count += 1
'size_formatted': self._format_size(total_size) total_size += os.path.getsize(os.path.join(root, f))
}
return {
def _format_size(self, size_bytes: int) -> str: 'files': file_count,
"""格式化文件大小""" 'size': total_size,
if size_bytes == 0: 'size_formatted': self._format_size(total_size)
return "0 B" }
units = ["B", "KB", "MB", "GB"] def _format_size(self, size_bytes: int) -> str:
i = 0 """格式化文件大小"""
while size_bytes >= 1024 and i < len(units) - 1: if size_bytes == 0:
size_bytes /= 1024.0 return "0 B"
i += 1
units = ["B", "KB", "MB", "GB"]
return f"{size_bytes:.2f} {units[i]}" i = 0
while size_bytes >= 1024 and i < len(units) - 1:
size_bytes /= 1024.0
i += 1
return f"{size_bytes:.2f} {units[i]}"
mirrors/go.py
+529 -525
View File
File diff suppressed because it is too large Load Diff
mirrors/npm.py
+284 -280
View File
@@ -1,280 +1,284 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
npm镜像代理处理器 npm镜像代理处理器
支持Node.js包管理器 支持Node.js包管理器
""" """
import os import os
import json import json
import time import time
import urllib.request import urllib.request
import urllib.parse import urllib.parse
from typing import Dict, List, Optional from typing import Dict, List, Optional
from datetime import datetime from datetime import datetime
class NpmMirror: class NpmMirror:
"""npm镜像代理""" """npm镜像代理"""
def __init__(self, config: dict): def __init__(self, config: dict):
self.config = config self.config = config
# 配置 - 使用 storage_dir（基于 base_dir） # 配置 - 使用 storage_dir（基于 base_dir）
self.upstream_url = config.get('upstream_url', 'https://registry.npmjs.org') self.upstream_url = config.get('upstream_url', 'https://registry.npmjs.org')
self.storage_dir = config.get('storage_dir', './downloads/npm') self.storage_dir = config.get('storage_dir', './downloads/npm')
self.base_dir = config.get('base_dir', './downloads') self.base_dir = config.get('base_dir', './downloads')
# 确保存储目录存在 # 缓存配置（缺失时默认启用，避免 AttributeError）
os.makedirs(self.storage_dir, exist_ok=True) self.cache_enabled = config.get('cache_enabled', True)
self.cache_ttl = config.get('cache_ttl', 3600)
def handle_request(self, handler, path: str) -> bool:
""" # 确保存储目录存在
处理npm请求 os.makedirs(self.storage_dir, exist_ok=True)
路径格式: /lodash 或 /-/package/lodash/dist
""" def handle_request(self, handler, path: str) -> bool:
try: """
parts = path.strip('/').split('/') 处理npm请求
路径格式: /lodash 或 /-/package/lodash/dist
if not parts: """
return self._handle_index(handler) try:
parts = path.strip('/').split('/')
if parts[0] == '-':
# Scoped package 或其他特殊请求 if not parts:
if len(parts) >= 4 and parts[1] == 'package': return self._handle_index(handler)
return self._handleScopedPackage(handler, parts[2], parts[3] if len(parts) > 3 else None)
elif len(parts) >= 3 and parts[1] == 'package': if parts[0] == '-':
return self._handle_package(handler, parts[2], None) # Scoped package 或其他特殊请求
else: if len(parts) >= 4 and parts[1] == 'package':
handler.send_error(400, "Invalid npm API path") return self._handleScopedPackage(handler, parts[2], parts[3] if len(parts) > 3 else None)
return False elif len(parts) >= 3 and parts[1] == 'package':
return self._handle_package(handler, parts[2], None)
elif parts[0] == '@': else:
# Scoped package handler.send_error(400, "Invalid npm API path")
if len(parts) >= 2: return False
scope = parts[0]
package = '/'.join(parts[1:]) elif parts[0] == '@':
return self._handle_scoped_package(handler, scope, package) # Scoped package
else: if len(parts) >= 2:
handler.send_error(400, "Invalid scoped package") scope = parts[0]
return False package = '/'.join(parts[1:])
return self._handle_scoped_package(handler, scope, package)
elif parts[0] == '-/': else:
# npm特殊路径 handler.send_error(400, "Invalid scoped package")
return self._handle_special(handler, '/'.join(parts)) return False
elif len(parts) == 1: elif parts[0] == '-/':
# 单个包名 # npm特殊路径
return self._handle_package(handler, parts[0], None) return self._handle_special(handler, '/'.join(parts))
else: elif len(parts) == 1:
# 其他请求 # 单个包名
return self._handle_package(handler, parts[0], parts[1] if len(parts) > 1 else None) return self._handle_package(handler, parts[0], None)
except Exception as e: else:
handler.send_error(500, str(e)) # 其他请求
return False return self._handle_package(handler, parts[0], parts[1] if len(parts) > 1 else None)
def _handle_index(self, handler) -> bool: except Exception as e:
"""处理索引请求""" handler.send_error(500, str(e))
handler.send_json_response({ return False
'registry_url': self.upstream_url,
'cache_stats': self.get_cache_stats() def _handle_index(self, handler) -> bool:
}) """处理索引请求"""
return True handler.send_json_response({
'registry_url': self.upstream_url,
def _handle_package(self, handler, package: str, version: str = None) -> bool: 'cache_stats': self.get_cache_stats()
"""处理包元数据请求""" })
cache_key = f"package:{package}:{version or 'latest'}" return True
cached = self._get_cache(cache_key) def _handle_package(self, handler, package: str, version: str = None) -> bool:
if cached: """处理包元数据请求"""
handler.send_response(200) cache_key = f"package:{package}:{version or 'latest'}"
handler.send_header('Content-Type', 'application/json')
handler.end_headers() cached = self._get_cache(cache_key)
handler.wfile.write(cached) if cached:
return True handler.send_response(200)
handler.send_header('Content-Type', 'application/json')
# 从上游获取 handler.end_headers()
if version: handler.wfile.write(cached)
url = f"{self.upstream_url}/{package}/{version}" return True
else:
url = f"{self.upstream_url}/{package}/latest" # 从上游获取
if version:
try: url = f"{self.upstream_url}/{package}/{version}"
data = self._fetch(url) else:
url = f"{self.upstream_url}/{package}/latest"
if self.cache_enabled:
self._set_cache(cache_key, data) try:
data = self._fetch(url)
handler.send_response(200)
handler.send_header('Content-Type', 'application/json') if self.cache_enabled:
handler.end_headers() self._set_cache(cache_key, data)
handler.wfile.write(data)
return True handler.send_response(200)
handler.send_header('Content-Type', 'application/json')
except urllib.error.HTTPError as e: handler.end_headers()
handler.send_error(404, f"Package not found: {package}") handler.wfile.write(data)
return False return True
def _handle_scoped_package(self, handler, scope: str, package: str) -> bool: except urllib.error.HTTPError as e:
"""处理scoped包""" handler.send_error(404, f"Package not found: {package}")
full_name = f"{scope}/{package}" return False
return self._handle_package(handler, full_name, None)
def _handle_scoped_package(self, handler, scope: str, package: str) -> bool:
def _handleScopedPackage(self, handler, scope: str, package: str) -> bool: """处理scoped包"""
"""处理特殊路径的scoped包""" full_name = f"{scope}/{package}"
full_name = f"{scope}/{package}" return self._handle_package(handler, full_name, None)
return self._handle_package(handler, full_name, None)
def _handleScopedPackage(self, handler, scope: str, package: str) -> bool:
def _handle_special(self, handler, path: str) -> bool: """处理特殊路径的scoped包"""
"""处理特殊npm路径""" full_name = f"{scope}/{package}"
# 简化实现：转发到上游 return self._handle_package(handler, full_name, None)
url = f"{self.upstream_url}/{path}"
def _handle_special(self, handler, path: str) -> bool:
try: """处理特殊npm路径"""
data = self._fetch(url) # 简化实现：转发到上游
url = f"{self.upstream_url}/{path}"
handler.send_response(200)
handler.send_header('Content-Type', 'application/json') try:
handler.end_headers() data = self._fetch(url)
handler.wfile.write(data)
return True handler.send_response(200)
handler.send_header('Content-Type', 'application/json')
except Exception as e: handler.end_headers()
handler.send_error(502, f"Failed to fetch: {str(e)}") handler.wfile.write(data)
return False return True
def _handle_tarball(self, handler, package: str, filename: str) -> bool: except Exception as e:
"""处理tarball下载""" handler.send_error(502, f"Failed to fetch: {str(e)}")
cache_key = f"tarball:{package}:{filename}" return False
cached = self._get_cache(cache_key) def _handle_tarball(self, handler, package: str, filename: str) -> bool:
if cached: """处理tarball下载"""
handler.send_response(200) cache_key = f"tarball:{package}:{filename}"
handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Length', str(len(cached))) cached = self._get_cache(cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Length', str(len(cached)))
# 从上游获取 handler.end_headers()
url = f"{self.upstream_url}/{package}/-/{filename}" handler.wfile.write(cached)
return True
try:
data = self._fetch(url) # 从上游获取
url = f"{self.upstream_url}/{package}/-/{filename}"
if self.cache_enabled:
self._set_cache(cache_key, data) try:
data = self._fetch(url)
handler.send_response(200)
handler.send_header('Content-Type', 'application/octet-stream') if self.cache_enabled:
handler.send_header('Content-Length', str(len(data))) self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Length', str(len(data)))
except Exception as e: handler.end_headers()
handler.send_error(502, f"Failed to fetch tarball: {str(e)}") handler.wfile.write(data)
return False return True
def _fetch(self, url: str) -> Optional[bytes]: except Exception as e:
"""从URL获取数据""" handler.send_error(502, f"Failed to fetch tarball: {str(e)}")
try: return False
req = urllib.request.Request(url)
req.add_header('User-Agent', 'npm-Mirror/1.0') def _fetch(self, url: str) -> Optional[bytes]:
req.add_header('Accept', 'application/json') """从URL获取数据"""
try:
with urllib.request.urlopen(req, timeout=30) as response: req = urllib.request.Request(url)
return response.read() req.add_header('User-Agent', 'npm-Mirror/1.0')
req.add_header('Accept', 'application/json')
except Exception:
return None with urllib.request.urlopen(req, timeout=30) as response:
return response.read()
def _get_cache(self, cache_key: str) -> Optional[bytes]:
"""获取缓存""" except Exception:
if not self.cache_enabled: return None
return None
def _get_cache(self, cache_key: str) -> Optional[bytes]:
cache_path = self._get_cache_path(cache_key) """获取缓存"""
meta_path = cache_path + '.meta' if not self.cache_enabled:
return None
if not os.path.exists(cache_path):
return None cache_path = self._get_cache_path(cache_key)
meta_path = cache_path + '.meta'
if os.path.exists(meta_path):
try: if not os.path.exists(cache_path):
with open(meta_path, 'r') as f: return None
meta = json.load(f)
if time.time() > meta.get('expires', 0): if os.path.exists(meta_path):
return None try:
except Exception: with open(meta_path, 'r') as f:
pass meta = json.load(f)
if time.time() > meta.get('expires', 0):
try: return None
with open(cache_path, 'rb') as f: except Exception:
return f.read() pass
except Exception:
return None try:
with open(cache_path, 'rb') as f:
def _set_cache(self, cache_key: str, data: bytes): return f.read()
"""设置缓存""" except Exception:
cache_path = self._get_cache_path(cache_key) return None
meta_path = cache_path + '.meta'
def _set_cache(self, cache_key: str, data: bytes):
os.makedirs(os.path.dirname(cache_path), exist_ok=True) """设置缓存"""
cache_path = self._get_cache_path(cache_key)
try: meta_path = cache_path + '.meta'
with open(cache_path, 'wb') as f:
f.write(data) os.makedirs(os.path.dirname(cache_path), exist_ok=True)
meta = { try:
'cached_at': time.time(), with open(cache_path, 'wb') as f:
'expires': time.time() + self.cache_ttl, f.write(data)
'size': len(data)
} meta = {
'cached_at': time.time(),
with open(meta_path, 'w') as f: 'expires': time.time() + self.cache_ttl,
json.dump(meta, f) 'size': len(data)
}
except Exception as e:
print(f"npm缓存写入失败: {e}") with open(meta_path, 'w') as f:
json.dump(meta, f)
def _get_cache_path(self, cache_key: str) -> str:
"""获取缓存路径""" except Exception as e:
subdir = cache_key[:2] print(f"npm缓存写入失败: {e}")
return os.path.join(self.storage_dir, subdir, cache_key)
def _get_cache_path(self, cache_key: str) -> str:
def get_cache_stats(self) -> dict: """获取缓存路径"""
"""获取缓存统计""" subdir = cache_key[:2]
if not os.path.exists(self.storage_dir): return os.path.join(self.storage_dir, subdir, cache_key)
return {'files': 0, 'size': 0}
def get_cache_stats(self) -> dict:
total_size = 0 """获取缓存统计"""
file_count = 0 if not os.path.exists(self.storage_dir):
return {'files': 0, 'size': 0}
for root, dirs, files in os.walk(self.storage_dir):
for f in files: total_size = 0
if not f.endswith('.meta'): file_count = 0
file_count += 1
total_size += os.path.getsize(os.path.join(root, f)) for root, dirs, files in os.walk(self.storage_dir):
for f in files:
return { if not f.endswith('.meta'):
'files': file_count, file_count += 1
'size': total_size, total_size += os.path.getsize(os.path.join(root, f))
'size_formatted': self._format_size(total_size)
} return {
'files': file_count,
def _format_size(self, size_bytes: int) -> str: 'size': total_size,
"""格式化文件大小""" 'size_formatted': self._format_size(total_size)
if size_bytes == 0: }
return "0 B"
def _format_size(self, size_bytes: int) -> str:
units = ["B", "KB", "MB", "GB"] """格式化文件大小"""
i = 0 if size_bytes == 0:
while size_bytes >= 1024 and i < len(units) - 1: return "0 B"
size_bytes /= 1024.0
i += 1 units = ["B", "KB", "MB", "GB"]
i = 0
return f"{size_bytes:.2f} {units[i]}" while size_bytes >= 1024 and i < len(units) - 1:
size_bytes /= 1024.0
i += 1
return f"{size_bytes:.2f} {units[i]}"
mirrors/yum.py
+388 -384
View File
@@ -1,384 +1,388 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
YUM/DNF镜像代理处理器 YUM/DNF镜像代理处理器
支持RHEL/CentOS/Rocky/AlmaLinux软件源 支持RHEL/CentOS/Rocky/AlmaLinux软件源
""" """
import os import os
import json import json
import time import time
import gzip import gzip
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
import urllib.request import urllib.request
from typing import Dict, List, Optional from typing import Dict, List, Optional
from datetime import datetime from datetime import datetime
class YUMMirror: class YUMMirror:
"""YUM/DNF镜像代理""" """YUM/DNF镜像代理"""
def __init__(self, config: dict): def __init__(self, config: dict):
self.config = config self.config = config
# 配置 - 使用 storage_dir（基于 base_dir） # 配置 - 使用 storage_dir（基于 base_dir）
self.base_url = config.get('base_url', 'http://mirror.centos.org/centos') self.base_url = config.get('base_url', 'http://mirror.centos.org/centos')
self.storage_dir = config.get('storage_dir', './downloads/yum') self.storage_dir = config.get('storage_dir', './downloads/yum')
self.base_dir = config.get('base_dir', './downloads') self.base_dir = config.get('base_dir', './downloads')
self.repo_id = config.get('repo_id', 'baseos') self.repo_id = config.get('repo_id', 'baseos')
self.arch = config.get('arch', 'x86_64') self.arch = config.get('arch', 'x86_64')
# 确保存储目录存在 # 缓存配置（缺失时默认启用，避免 AttributeError）
os.makedirs(self.storage_dir, exist_ok=True) self.cache_enabled = config.get('cache_enabled', True)
self.cache_ttl = config.get('cache_ttl', 3600)
def handle_request(self, handler, path: str) -> bool:
""" # 确保存储目录存在
处理YUM请求 os.makedirs(self.storage_dir, exist_ok=True)
路径格式: /centos/7/updates/x86_64/repodata/repomd.xml
""" def handle_request(self, handler, path: str) -> bool:
try: """
parts = path.strip('/').split('/') 处理YUM请求
路径格式: /centos/7/updates/x86_64/repodata/repomd.xml
if len(parts) < 3: """
return self._handle_index(handler) try:
parts = path.strip('/').split('/')
distro = parts[0] # centos, rocky, alma
version = parts[1] # 7, 8, 9 if len(parts) < 3:
repo = parts[2] # baseos, appstream, updates return self._handle_index(handler)
rest = '/'.join(parts[3:])
distro = parts[0] # centos, rocky, alma
# 确定文件类型 version = parts[1] # 7, 8, 9
if 'repomd.xml' in rest: repo = parts[2] # baseos, appstream, updates
return self._handle_repomd(handler, distro, version, repo) rest = '/'.join(parts[3:])
elif 'primary.xml.gz' in rest:
return self._handle_primary(handler, distro, version, repo, 'primary') # 确定文件类型
elif 'filelists.xml.gz' in rest: if 'repomd.xml' in rest:
return self._handle_filelists(handler, distro, version, repo, 'filelists') return self._handle_repomd(handler, distro, version, repo)
elif 'other.xml.gz' in rest: elif 'primary.xml.gz' in rest:
return self._handle_other(handler, distro, version, repo, 'other') return self._handle_primary(handler, distro, version, repo, 'primary')
else: elif 'filelists.xml.gz' in rest:
return self._handle_repo_file(handler, distro, version, repo, rest) return self._handle_filelists(handler, distro, version, repo, 'filelists')
elif 'other.xml.gz' in rest:
except Exception as e: return self._handle_other(handler, distro, version, repo, 'other')
handler.send_error(500, str(e)) else:
return False return self._handle_repo_file(handler, distro, version, repo, rest)
def _handle_index(self, handler) -> bool: except Exception as e:
"""处理索引请求""" handler.send_error(500, str(e))
handler.send_json_response({ return False
'base_url': self.base_url,
'repo_id': self.repo_id, def _handle_index(self, handler) -> bool:
'arch': self.arch, """处理索引请求"""
'cache_stats': self.get_cache_stats() handler.send_json_response({
}) 'base_url': self.base_url,
return True 'repo_id': self.repo_id,
'arch': self.arch,
def _handle_repomd(self, handler, distro: str, version: str, repo: str) -> bool: 'cache_stats': self.get_cache_stats()
"""处理repomd.xml请求""" })
cache_key = f"repomd:{distro}:{version}:{repo}" return True
cached = self._get_cache(cache_key) def _handle_repomd(self, handler, distro: str, version: str, repo: str) -> bool:
if cached: """处理repomd.xml请求"""
handler.send_response(200) cache_key = f"repomd:{distro}:{version}:{repo}"
handler.send_header('Content-Type', 'application/xml')
handler.send_header('Content-Length', str(len(cached))) cached = self._get_cache(cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/xml')
handler.send_header('Content-Length', str(len(cached)))
# 从上游获取 handler.end_headers()
url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml" handler.wfile.write(cached)
return True
try:
data = self._fetch(url) # 从上游获取
url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml"
if self.cache_enabled:
self._set_cache(cache_key, data) try:
data = self._fetch(url)
handler.send_response(200)
handler.send_header('Content-Type', 'application/xml') if self.cache_enabled:
handler.send_header('Content-Length', str(len(data))) self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/xml')
handler.send_header('Content-Length', str(len(data)))
except Exception as e: handler.end_headers()
handler.send_error(502, f"Failed to fetch repomd: {str(e)}") handler.wfile.write(data)
return False return True
def _handle_primary(self, handler, distro: str, version: str, repo: str, db_type: str) -> bool: except Exception as e:
"""处理primary.xml.gz""" handler.send_error(502, f"Failed to fetch repomd: {str(e)}")
cache_key = f"primary:{distro}:{version}:{repo}:{self.arch}" return False
cached = self._get_cache(cache_key) def _handle_primary(self, handler, distro: str, version: str, repo: str, db_type: str) -> bool:
if cached: """处理primary.xml.gz"""
handler.send_response(200) cache_key = f"primary:{distro}:{version}:{repo}:{self.arch}"
handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(cached))) cached = self._get_cache(cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(cached)))
# 先获取repomd.xml找到对应的数据库文件 handler.end_headers()
repomd_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml" handler.wfile.write(cached)
return True
try:
repomd_data = self._fetch(repomd_url) # 先获取repomd.xml找到对应的数据库文件
repomd_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml"
# 解析repomd.xml找到primary文件
root = ET.fromstring(repomd_data) try:
ns = {'repomd': 'http://linux.duke.edu/metadata/repo'} repomd_data = self._fetch(repomd_url)
data_location = None # 解析repomd.xml找到primary文件
for elem in root.findall('.//repomd:data', ns): root = ET.fromstring(repomd_data)
if elem.get('type') == 'primary': ns = {'repomd': 'http://linux.duke.edu/metadata/repo'}
data_location = elem.find('repomd:location', ns).get('href')
break data_location = None
for elem in root.findall('.//repomd:data', ns):
if data_location: if elem.get('type') == 'primary':
db_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/{data_location}" data_location = elem.find('repomd:location', ns).get('href')
data = self._fetch(db_url) break
if self.cache_enabled: if data_location:
self._set_cache(cache_key, data) db_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/{data_location}"
data = self._fetch(db_url)
handler.send_response(200)
handler.send_header('Content-Type', 'application/x-gzip') if self.cache_enabled:
handler.send_header('Content-Length', str(len(data))) self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(data)))
except Exception as e: handler.end_headers()
pass handler.wfile.write(data)
return True
handler.send_error(502, "Failed to fetch primary database")
return False except Exception as e:
pass
def _handle_filelists(self, handler, distro: str, version: str, repo: str, db_type: str) -> bool:
"""处理filelists.xml.gz""" handler.send_error(502, "Failed to fetch primary database")
cache_key = f"filelists:{distro}:{version}:{repo}:{self.arch}" return False
cached = self._get_cache(cache_key) def _handle_filelists(self, handler, distro: str, version: str, repo: str, db_type: str) -> bool:
if cached: """处理filelists.xml.gz"""
handler.send_response(200) cache_key = f"filelists:{distro}:{version}:{repo}:{self.arch}"
handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(cached))) cached = self._get_cache(cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(cached)))
# 先获取repomd.xml找到对应的数据库文件 handler.end_headers()
repomd_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml" handler.wfile.write(cached)
return True
try:
repomd_data = self._fetch(repomd_url) # 先获取repomd.xml找到对应的数据库文件
repomd_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml"
# 解析repomd.xml找到filelists文件
root = ET.fromstring(repomd_data) try:
ns = {'repomd': 'http://linux.duke.edu/metadata/repo'} repomd_data = self._fetch(repomd_url)
data_location = None # 解析repomd.xml找到filelists文件
for elem in root.findall('.//repomd:data', ns): root = ET.fromstring(repomd_data)
if elem.get('type') == 'filelists': ns = {'repomd': 'http://linux.duke.edu/metadata/repo'}
data_location = elem.find('repomd:location', ns).get('href')
break data_location = None
for elem in root.findall('.//repomd:data', ns):
if data_location: if elem.get('type') == 'filelists':
db_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/{data_location}" data_location = elem.find('repomd:location', ns).get('href')
data = self._fetch(db_url) break
if self.cache_enabled: if data_location:
self._set_cache(cache_key, data) db_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/{data_location}"
data = self._fetch(db_url)
handler.send_response(200)
handler.send_header('Content-Type', 'application/x-gzip') if self.cache_enabled:
handler.send_header('Content-Length', str(len(data))) self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(data)))
except Exception as e: handler.end_headers()
pass handler.wfile.write(data)
return True
handler.send_error(502, "Failed to fetch filelists database")
return False except Exception as e:
pass
def _handle_other(self, handler, distro: str, version: str, repo: str, db_type: str) -> bool:
"""处理other.xml.gz""" handler.send_error(502, "Failed to fetch filelists database")
cache_key = f"other:{distro}:{version}:{repo}:{self.arch}" return False
cached = self._get_cache(cache_key) def _handle_other(self, handler, distro: str, version: str, repo: str, db_type: str) -> bool:
if cached: """处理other.xml.gz"""
handler.send_response(200) cache_key = f"other:{distro}:{version}:{repo}:{self.arch}"
handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(cached))) cached = self._get_cache(cache_key)
handler.end_headers() if cached:
handler.wfile.write(cached) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(cached)))
# 先获取repomd.xml找到对应的数据库文件 handler.end_headers()
repomd_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml" handler.wfile.write(cached)
return True
try:
repomd_data = self._fetch(repomd_url) # 先获取repomd.xml找到对应的数据库文件
repomd_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/repomd.xml"
# 解析repomd.xml找到other文件
root = ET.fromstring(repomd_data) try:
ns = {'repomd': 'http://linux.duke.edu/metadata/repo'} repomd_data = self._fetch(repomd_url)
data_location = None # 解析repomd.xml找到other文件
for elem in root.findall('.//repomd:data', ns): root = ET.fromstring(repomd_data)
if elem.get('type') == 'other': ns = {'repomd': 'http://linux.duke.edu/metadata/repo'}
data_location = elem.find('repomd:location', ns).get('href')
break data_location = None
for elem in root.findall('.//repomd:data', ns):
if data_location: if elem.get('type') == 'other':
db_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/{data_location}" data_location = elem.find('repomd:location', ns).get('href')
data = self._fetch(db_url) break
if self.cache_enabled: if data_location:
self._set_cache(cache_key, data) db_url = f"{self.base_url}/{version}/{repo}/{self.arch}/repodata/{data_location}"
data = self._fetch(db_url)
handler.send_response(200)
handler.send_header('Content-Type', 'application/x-gzip') if self.cache_enabled:
handler.send_header('Content-Length', str(len(data))) self._set_cache(cache_key, data)
handler.end_headers()
handler.wfile.write(data) handler.send_response(200)
return True handler.send_header('Content-Type', 'application/x-gzip')
handler.send_header('Content-Length', str(len(data)))
except Exception as e: handler.end_headers()
pass handler.wfile.write(data)
return True
handler.send_error(502, "Failed to fetch other database")
return False except Exception as e:
pass
def _handle_repo_file(self, handler, distro: str, version: str, repo: str, path: str) -> bool:
"""处理仓库中的其他文件""" handler.send_error(502, "Failed to fetch other database")
cache_key = f"file:{distro}:{version}:{repo}:{path.replace('/', ':')}" return False
cached = self._get_cache(cache_key) def _handle_repo_file(self, handler, distro: str, version: str, repo: str, path: str) -> bool:
if cached: """处理仓库中的其他文件"""
handler.send_response(200) cache_key = f"file:{distro}:{version}:{repo}:{path.replace('/', ':')}"
handler.send_header('Content-Type', 'application/octet-stream')
handler.end_headers() cached = self._get_cache(cache_key)
handler.wfile.write(cached) if cached:
return True handler.send_response(200)
handler.send_header('Content-Type', 'application/octet-stream')
url = f"{self.base_url}/{version}/{repo}/{self.arch}/{path}" handler.end_headers()
handler.wfile.write(cached)
try: return True
data = self._fetch(url)
url = f"{self.base_url}/{version}/{repo}/{self.arch}/{path}"
if self.cache_enabled:
self._set_cache(cache_key, data) try:
data = self._fetch(url)
handler.send_response(200)
handler.send_header('Content-Type', 'application/octet-stream') if self.cache_enabled:
handler.end_headers() self._set_cache(cache_key, data)
handler.wfile.write(data)
return True handler.send_response(200)
handler.send_header('Content-Type', 'application/octet-stream')
except Exception as e: handler.end_headers()
handler.send_error(404, f"File not found: {str(e)}") handler.wfile.write(data)
return False return True
def _fetch(self, url: str) -> Optional[bytes]: except Exception as e:
"""从URL获取数据""" handler.send_error(404, f"File not found: {str(e)}")
try: return False
req = urllib.request.Request(url)
req.add_header('User-Agent', 'YUM-Mirror/1.0') def _fetch(self, url: str) -> Optional[bytes]:
"""从URL获取数据"""
with urllib.request.urlopen(req, timeout=30) as response: try:
return response.read() req = urllib.request.Request(url)
req.add_header('User-Agent', 'YUM-Mirror/1.0')
except Exception:
return None with urllib.request.urlopen(req, timeout=30) as response:
return response.read()
def _get_cache(self, cache_key: str) -> Optional[bytes]:
"""获取缓存""" except Exception:
if not self.cache_enabled: return None
return None
def _get_cache(self, cache_key: str) -> Optional[bytes]:
cache_path = self._get_cache_path(cache_key) """获取缓存"""
meta_path = cache_path + '.meta' if not self.cache_enabled:
return None
if not os.path.exists(cache_path):
return None cache_path = self._get_cache_path(cache_key)
meta_path = cache_path + '.meta'
if os.path.exists(meta_path):
try: if not os.path.exists(cache_path):
with open(meta_path, 'r') as f: return None
meta = json.load(f)
if time.time() > meta.get('expires', 0): if os.path.exists(meta_path):
return None try:
except Exception: with open(meta_path, 'r') as f:
pass meta = json.load(f)
if time.time() > meta.get('expires', 0):
try: return None
with open(cache_path, 'rb') as f: except Exception:
return f.read() pass
except Exception:
return None try:
with open(cache_path, 'rb') as f:
def _set_cache(self, cache_key: str, data: bytes): return f.read()
"""设置缓存""" except Exception:
cache_path = self._get_cache_path(cache_key) return None
meta_path = cache_path + '.meta'
def _set_cache(self, cache_key: str, data: bytes):
os.makedirs(os.path.dirname(cache_path), exist_ok=True) """设置缓存"""
cache_path = self._get_cache_path(cache_key)
try: meta_path = cache_path + '.meta'
with open(cache_path, 'wb') as f:
f.write(data) os.makedirs(os.path.dirname(cache_path), exist_ok=True)
meta = { try:
'cached_at': time.time(), with open(cache_path, 'wb') as f:
'expires': time.time() + self.cache_ttl, f.write(data)
'size': len(data)
} meta = {
'cached_at': time.time(),
with open(meta_path, 'w') as f: 'expires': time.time() + self.cache_ttl,
json.dump(meta, f) 'size': len(data)
}
except Exception as e:
print(f"YUM缓存写入失败: {e}") with open(meta_path, 'w') as f:
json.dump(meta, f)
def _get_cache_path(self, cache_key: str) -> str:
"""获取缓存路径""" except Exception as e:
subdir = cache_key[:2] print(f"YUM缓存写入失败: {e}")
return os.path.join(self.storage_dir, subdir, cache_key)
def _get_cache_path(self, cache_key: str) -> str:
def get_cache_stats(self) -> dict: """获取缓存路径"""
"""获取缓存统计""" subdir = cache_key[:2]
if not os.path.exists(self.storage_dir): return os.path.join(self.storage_dir, subdir, cache_key)
return {'files': 0, 'size': 0}
def get_cache_stats(self) -> dict:
total_size = 0 """获取缓存统计"""
file_count = 0 if not os.path.exists(self.storage_dir):
return {'files': 0, 'size': 0}
for root, dirs, files in os.walk(self.storage_dir):
for f in files: total_size = 0
if not f.endswith('.meta'): file_count = 0
file_count += 1
total_size += os.path.getsize(os.path.join(root, f)) for root, dirs, files in os.walk(self.storage_dir):
for f in files:
return { if not f.endswith('.meta'):
'files': file_count, file_count += 1
'size': total_size, total_size += os.path.getsize(os.path.join(root, f))
'size_formatted': self._format_size(total_size)
} return {
'files': file_count,
def _format_size(self, size_bytes: int) -> str: 'size': total_size,
"""格式化文件大小""" 'size_formatted': self._format_size(total_size)
if size_bytes == 0: }
return "0 B"
def _format_size(self, size_bytes: int) -> str:
units = ["B", "KB", "MB", "GB"] """格式化文件大小"""
i = 0 if size_bytes == 0:
while size_bytes >= 1024 and i < len(units) - 1: return "0 B"
size_bytes /= 1024.0
i += 1 units = ["B", "KB", "MB", "GB"]
i = 0
return f"{size_bytes:.2f} {units[i]}" while size_bytes >= 1024 and i < len(units) - 1:
size_bytes /= 1024.0
i += 1
return f"{size_bytes:.2f} {units[i]}"
requirements.txt
+1 -1
View File
@@ -2,7 +2,7 @@ psutil>=5.9.0
pyinstaller>=6.0.0 pyinstaller>=6.0.0
python-multipart>=0.0.6 python-multipart>=0.0.6
paramiko>=2.11.0 paramiko>=2.11.0
aiohttp>=3.9.0 bcrypt>=4.0.0
cachetools>=5.3.0 cachetools>=5.3.0
requests>=2.31.0 requests>=2.31.0
croniter>=2.0.0 croniter>=2.0.0