Files

723 lines
23 KiB
Python

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
内存优化模块
专为低端设备优化 (2CPU/1G 内存等)
"""
import os
import sys
import gc
import time
import threading
from typing import Dict, List, Optional, Callable
from dataclasses import dataclass
from contextlib import contextmanager
from functools import wraps
# 跨平台兼容处理:resource 模块仅在 Unix/Linux 上可用
try:
import resource
_HAS_RESOURCE = True
except ImportError:
_HAS_RESOURCE = False
resource = None
# 内存限制配置
DEFAULT_MEMORY_LIMIT = 512 * 1024 * 1024 # 512MB
LOW_MEMORY_LIMIT = 256 * 1024 * 1024 # 256MB
VERY_LOW_MEMORY_LIMIT = 128 * 1024 * 1024 # 128MB
class MemoryManager:
"""内存管理器 - 支持定时垃圾回收"""
def __init__(self, config: dict = None):
self.config = config or {}
self.enabled = self.config.get('enabled', True)
self.memory_limit = self.config.get('memory_limit', DEFAULT_MEMORY_LIMIT)
self.soft_limit = self.memory_limit * 0.8 # 80% 时触发警告
self.check_interval = self.config.get('check_interval', 10) # 秒
# 定时垃圾回收配置
self.gc_interval = self.config.get('gc_interval', 300) # 默认 5 分钟
self.enable_scheduled_gc = self.config.get('enable_scheduled_gc', True)
# 缓存清理回调
self.cache_cleaners: List[Callable] = []
# 回调函数
self.on_memory_warning: Optional[Callable] = None
self.on_memory_critical: Optional[Callable] = None
self._running = False
self._monitor_thread = None
self._gc_thread = None
def start(self):
"""启动内存监控"""
if not self.enabled:
return
self._running = True
# 启动内存监控线程
self._monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
self._monitor_thread.start()
# 启动定时垃圾回收线程
if self.enable_scheduled_gc:
self._gc_thread = threading.Thread(target=self._gc_loop, daemon=True)
self._gc_thread.start()
print(f"[内存管理] 定时GC: {self.gc_interval}秒")
# 设置内存限制
self.set_memory_limit(self.memory_limit)
print(f"[内存管理] 已启动, 限制: {self.memory_limit // 1024 // 1024}MB")
def stop(self):
"""停止内存监控"""
self._running = False
if self._monitor_thread:
self._monitor_thread.join(timeout=2)
if self._gc_thread:
self._gc_thread.join(timeout=2)
def register_cache_cleaner(self, cleaner: Callable):
"""注册缓存清理回调函数"""
self.cache_cleaners.append(cleaner)
def set_memory_limit(self, limit: int):
"""设置内存限制 (Linux/Unix)"""
if not _HAS_RESOURCE or resource is None:
# Windows 平台不支持内存限制,跳过
print(f"[内存管理] 跳过内存限制设置 (Windows平台不支持)")
return
try:
# 软限制
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
print(f"[内存管理] 已设置内存限制: {limit // 1024 // 1024}MB")
except Exception as e:
print(f"[内存管理] 设置内存限制失败: {e}")
def get_memory_usage(self) -> dict:
"""获取内存使用情况"""
try:
# 进程内存
import psutil
process = psutil.Process(os.getpid())
mem_info = process.memory_info()
# 系统内存
sys_mem = psutil.virtual_memory()
return {
'process_rss': mem_info.rss,
'process_vms': mem_info.vms,
'process_percent': process.memory_percent(),
'system_total': sys_mem.total,
'system_available': sys_mem.available,
'system_percent': sys_mem.percent,
'process_rss_mb': mem_info.rss / 1024 / 1024,
'system_available_mb': sys_mem.available / 1024 / 1024
}
except Exception as e:
# 备用方法:使用 resource (Unix) 或返回估计值 (Windows)
if _HAS_RESOURCE and resource is not None:
try:
return {
'process_rss': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024,
'process_rss_mb': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
}
except Exception:
pass
# Windows 无 psutil 的极端情况
return {
'process_rss': 0,
'process_rss_mb': 0,
'error': str(e)
}
def _monitor_loop(self):
"""监控循环"""
while self._running:
try:
usage = self.get_memory_usage()
# 检查是否达到软限制
if usage['process_rss'] >= self.soft_limit:
if self.on_memory_warning:
self.on_memory_warning(usage)
self._aggressive_cleanup()
# 检查是否达到硬限制
if usage['process_rss'] >= self.memory_limit:
if self.on_memory_critical:
self.on_memory_critical(usage)
self._emergency_cleanup()
time.sleep(self.check_interval)
except Exception:
pass
def _gc_loop(self):
"""定时垃圾回收循环"""
while self._running:
try:
# 执行垃圾回收
self._scheduled_gc()
# 清理注册过的缓存
for cleaner in self.cache_cleaners:
try:
cleaner()
except Exception:
pass
except Exception:
pass
time.sleep(self.gc_interval)
def _scheduled_gc(self):
"""定时 GC 执行"""
# 标准 GC
collected = gc.collect()
# 清理 Python 内部缓存
if hasattr(sys, 'exc_clear'):
sys.exc_clear()
def _aggressive_cleanup(self):
"""激进清理"""
# 强制垃圾回收
gc.collect()
# 清理 Python 缓存
if hasattr(gc, 'set_threshold'):
gc.set_threshold(500, 10, 5)
# 尝试释放内存
try:
import psutil
process = psutil.Process(os.getpid())
process.memory_info().rss # 刷新
except Exception:
pass
def _emergency_cleanup(self):
"""紧急清理"""
print("[内存管理] ⚠️ 达到内存限制,尝试紧急清理...")
# 完全垃圾回收
gc.collect()
gc.collect()
gc.collect()
# 清理所有缓存
if hasattr(gc, 'garbage'):
del gc.garbage[:]
# 触发警告
print("[内存管理] ⚠️ 内存仍过高,考虑重启服务")
def get_status(self) -> dict:
"""获取状态"""
usage = self.get_memory_usage()
return {
'enabled': self.enabled,
'memory_limit_mb': self.memory_limit // 1024 // 1024,
'current_mb': usage['process_rss_mb'],
'available_mb': usage.get('system_available_mb', 0),
'percent': (usage['process_rss'] / self.memory_limit * 100) if self.memory_limit else 0
}
# ==================== 低内存配置 ====================
class LowMemoryConfig:
"""低端设备配置"""
# 可禁用的功能列表
FEATURES = {
'ws': 'enable_ws', # WebSocket
'sse': 'enable_sse', # Server-Sent Events
'hash_calc': 'calculate_hash', # 文件哈希计算
'stats': 'enable_stats', # 统计功能
'monitor': 'enable_monitor', # 系统监控
'sync': 'enable_sync', # 同步功能
'mirrors': 'enable_mirrors', # 加速源
}
# 预设配置
PRESETS = {
'ultra_low': {
'description': '极低端设备 (<256MB RAM)',
'workers': 1,
'max_cache_size': 32 * 1024 * 1024, # 32MB
'chunk_size': 16 * 1024, # 16KB
'buffer_size': 32 * 1024, # 32KB
'db_pool_size': 1,
'max_connections': 3,
'monitor_interval': 60,
'gc_interval': 180, # 3分钟
'timeout': 15,
'disable_optional_features': ['ws', 'sse', 'hash_calc', 'stats', 'monitor']
},
'low': {
'description': '低端设备 (256-512MB RAM)',
'workers': 1,
'max_cache_size': 64 * 1024 * 1024, # 64MB
'chunk_size': 32 * 1024, # 32KB
'buffer_size': 64 * 1024, # 64KB
'db_pool_size': 1,
'max_connections': 5,
'monitor_interval': 30,
'gc_interval': 300, # 5分钟
'timeout': 20,
'disable_optional_features': ['ws', 'sse', 'hash_calc', 'stats']
},
'medium': {
'description': '中等设备 (512MB-1GB RAM)',
'workers': 2,
'max_cache_size': 128 * 1024 * 1024, # 128MB
'chunk_size': 64 * 1024, # 64KB
'buffer_size': 128 * 1024, # 128KB
'db_pool_size': 2,
'max_connections': 15,
'monitor_interval': 15,
'gc_interval': 600, # 10分钟
'timeout': 30,
'disable_optional_features': []
},
'high': {
'description': '高端设备 (1GB+ RAM)',
'workers': 4,
'max_cache_size': 256 * 1024 * 1024, # 256MB
'chunk_size': 128 * 1024, # 128KB
'buffer_size': 256 * 1024, # 256KB
'db_pool_size': 4,
'max_connections': 50,
'monitor_interval': 5,
'gc_interval': 900, # 15分钟
'timeout': 30,
'disable_optional_features': []
},
'performance': {
'description': '高性能设备 (4GB+ RAM)',
'workers': 8,
'max_cache_size': 1024 * 1024 * 1024, # 1GB
'chunk_size': 256 * 1024, # 256KB
'buffer_size': 512 * 1024, # 512KB
'db_pool_size': 8,
'max_connections': 200,
'monitor_interval': 3,
'gc_interval': 1800, # 30分钟
'timeout': 60,
'disable_optional_features': []
}
}
def __init__(self, preset: str = 'auto', custom_config: dict = None):
"""
初始化配置
Args:
preset: 预设 ('ultra_low', 'low', 'medium', 'high', 'auto')
custom_config: 自定义配置
"""
if preset == 'auto':
preset = self._detect_preset()
self.preset = preset
self.config = self.PRESETS.get(preset, self.PRESETS['low']).copy()
if custom_config:
self.config.update(custom_config)
def _detect_preset(self) -> str:
"""自动检测设备配置
检测逻辑:
1. 获取总内存和可用内存
2. 计算可用内存占比
3. 结合总内存和可用内存占比综合判断
"""
try:
import psutil
mem = psutil.virtual_memory()
total_ram = mem.total
available_ram = mem.available
percent_used = mem.percent # 已使用百分比
# 转换为MB
total_mb = total_ram / (1024 * 1024)
# 根据总内存和可用内存占比综合判断
if total_mb < 200:
# 低于 200MB 总内存
return 'ultra_low'
elif total_mb < 400:
# 200MB - 400MB
return 'ultra_low'
elif total_mb < 700:
# 400MB - 700MB
if percent_used > 80:
return 'ultra_low'
return 'low'
elif total_mb < 1200:
# 700MB - 1.2GB
if percent_used > 70:
return 'ultra_low'
elif percent_used > 50:
return 'low'
return 'medium'
elif total_mb < 2500:
# 1.2GB - 2.5GB
if percent_used > 70:
return 'low'
elif percent_used > 40:
return 'medium'
return 'high'
elif total_mb < 5000:
# 2.5GB - 5GB
if percent_used > 60:
return 'medium'
return 'high'
else:
# 5GB+
return 'performance'
except ImportError:
# 如果没有 psutil,使用保守的 low 配置
return 'low'
except Exception:
return 'low'
def get_device_info(self) -> dict:
"""获取设备详细信息用于显示"""
try:
import psutil
mem = psutil.virtual_memory()
cpu_count = psutil.cpu_count(logical=True) or 1
return {
'total_ram_mb': mem.total / (1024 * 1024),
'available_ram_mb': mem.available / (1024 * 1024),
'percent_used': mem.percent,
'cpu_count': cpu_count,
'preset': self.preset
}
except Exception:
return {
'total_ram_mb': 0,
'available_ram_mb': 0,
'percent_used': 0,
'cpu_count': 1,
'preset': self.preset
}
def apply_to_config(self, base_config: dict) -> dict:
"""应用配置到基础配置"""
config = base_config.copy()
# 应用通用设置
config['workers'] = self.config.get('workers', 1)
config['max_cache_size'] = self.config.get('max_cache_size', 64 * 1024 * 1024)
config['chunk_size'] = self.config.get('chunk_size', 64 * 1024)
config['buffer_size'] = self.config.get('buffer_size', 128 * 1024)
config['timeout'] = self.config.get('timeout', 30)
# 数据库池
if 'database' not in config:
config['database'] = {}
config['database']['db_pool_size'] = self.config.get('db_pool_size', 2)
# 定时 GC 配置
config['gc_interval'] = self.config.get('gc_interval', 300)
# 禁用可选功能
for feature in self.config.get('disable_optional_features', []):
feature_key = self.FEATURES.get(feature, feature)
if feature_key.startswith('enable_') or feature_key == 'calculate_hash':
config[feature_key] = False
return config
def get_config(self) -> dict:
"""获取配置"""
return self.config.copy()
def get_status(self) -> dict:
"""获取状态"""
return {
'preset': self.preset,
'description': self.config['description'],
'settings': self.config
}
# ==================== 流式处理优化 ====================
class StreamingOptimizer:
"""流式处理优化器"""
def __init__(self, config: dict = None):
self.config = config or {}
self.chunk_size = self.config.get('chunk_size', 128 * 1024)
self.buffer_size = self.config.get('buffer_size', 256 * 1024)
# 内存池
self._chunk_pool = None
self._use_memory_pool = self.config.get('use_memory_pool', True)
def get_optimized_chunk_size(self, file_size: int) -> int:
"""根据文件大小获取优化的块大小"""
if file_size < 1024 * 1024: # < 1MB
return 16 * 1024 # 16KB
elif file_size < 10 * 1024 * 1024: # < 10MB
return 32 * 1024 # 32KB
elif file_size < 100 * 1024 * 1024: # < 100MB
return 64 * 1024 # 64KB
else:
return self.chunk_size
@contextmanager
def memory_efficient_file_read(self, file_path: str, chunk_size: int = None):
"""
内存高效的文件读取
Usage:
with optimizer.memory_efficient_file_read('/path/to/file') as f:
for chunk in f:
process(chunk)
"""
chunk_size = chunk_size or self.chunk_size
file_size = os.path.getsize(file_path)
chunk_size = self.get_optimized_chunk_size(file_size)
file = open(file_path, 'rb')
try:
yield file
finally:
file.close()
@contextmanager
def memory_efficient_file_write(self, file_path: str, chunk_size: int = None):
"""内存高效的文件写入"""
chunk_size = chunk_size or self.chunk_size
file = open(file_path, 'wb')
try:
yield file
finally:
file.close()
# ==================== 架构检测 ====================
class ArchitectureDetector:
"""架构检测器"""
@staticmethod
def get_architecture() -> dict:
"""
获取架构信息
Returns:
dict: 包含架构信息的字典
"""
info = {
'platform': sys.platform,
'architecture': 'unknown',
'machine': 'unknown',
'processor': 'unknown',
'python_version': sys.version,
'byte_order': sys.byteorder
}
# 机器类型
info['machine'] = os.uname().machine if hasattr(os, 'uname') else 'unknown'
# 检测 32位/64位
if info['machine'] in ['x86_64', 'amd64', 'aarch64', 'arm64']:
info['architecture'] = '64bit'
elif info['machine'] in ['i386', 'i686', 'armv7l', 'armv6l']:
info['architecture'] = '32bit'
elif info['machine'] in ['armv8l', 'aarch32']:
info['architecture'] = '32bit' # 32位ARM
# ARM 变体
if info['machine'].startswith('arm'):
if info['machine'] in ['armv7l', 'armv7hl']:
info['arm_variant'] = 'armv7'
elif info['machine'].startswith('armv8'):
info['arm_variant'] = 'armv8'
elif info['machine'].startswith('armv6'):
info['arm_variant'] = 'armv6'
else:
info['arm_variant'] = 'unknown'
# x86 变体
if info['machine'] in ['i386', 'i686']:
info['x86_variant'] = 'i386'
elif info['machine'] == 'x86_64':
info['x86_variant'] = 'x86_64'
return info
@staticmethod
def is_low_end_device() -> bool:
"""检测是否为低端设备"""
try:
import psutil
mem = psutil.virtual_memory()
return mem.total < 1024 * 1024 * 1024 # < 1GB
except Exception:
return False
@staticmethod
def get_recommended_config() -> dict:
"""获取推荐的配置"""
arch = ArchitectureDetector.get_architecture()
if arch['architecture'] == '32bit':
return {
'max_workers': 2,
'max_cache_size': 100 * 1024 * 1024, # 100MB
'enable_threading': True,
'use_processes': False, # 32位进程数有限制
'max_file_handles': 256
}
elif ArchitectureDetector.is_low_end_device():
return {
'max_workers': 1,
'max_cache_size': 50 * 1024 * 1024, # 50MB
'enable_threading': True,
'use_processes': False,
'max_file_handles': 128
}
else:
return {
'max_workers': 4,
'max_cache_size': 500 * 1024 * 1024, # 500MB
'enable_threading': True,
'use_processes': True,
'max_file_handles': 1024
}
# ==================== 兼容性检查 ====================
def check_compatibility() -> dict:
"""
检查系统兼容性
Returns:
dict: 兼容性检查结果
"""
results = {
'compatible': True,
'warnings': [],
'errors': [],
'info': {}
}
# Python 版本检查
if sys.version_info < (3, 8):
results['compatible'] = False
results['errors'].append(f"Python 3.8+ 所需, 当前版本: {sys.version}")
# 架构信息
arch_info = ArchitectureDetector.get_architecture()
results['info']['architecture'] = arch_info
# 检查必需模块
required_modules = [
('os', '标准库'),
('json', '标准库'),
('http', '标准库'),
('sqlite3', '标准库')
]
optional_modules = [
('psutil', '系统监控 (推荐)'),
('sqlalchemy', '数据库 (推荐)'),
(' cryptography', '加密 (推荐)'),
('aiohttp', '异步HTTP (可选)'),
('paramiko', 'SSH/SFTP (可选)')
]
for module, desc in required_modules:
try:
__import__(module)
except ImportError:
results['compatible'] = False
results['errors'].append(f"必需模块缺失: {module} ({desc})")
for module, desc in optional_modules:
try:
__import__(module)
except ImportError:
results['warnings'].append(f"可选模块缺失: {module} ({desc})")
# 内存检查
try:
import psutil
mem = psutil.virtual_memory()
if mem.total < 256 * 1024 * 1024:
results['warnings'].append("内存低于 256MB,可能无法正常运行")
except Exception:
results['warnings'].append("无法检测内存,可能内存不足")
# 磁盘空间检查
try:
disk = psutil.disk_usage('.')
if disk.free < 100 * 1024 * 1024: # 100MB
results['warnings'].append("可用磁盘空间不足 100MB")
except Exception:
pass
return results
# ==================== 便捷函数 ====================
def get_memory_manager(config: dict = None) -> MemoryManager:
"""获取内存管理器"""
return MemoryManager(config)
def get_low_memory_config(preset: str = 'auto') -> LowMemoryConfig:
"""获取低端设备配置"""
return LowMemoryConfig(preset)
def detect_and_configure() -> dict:
"""
自动检测并配置
Returns:
dict: 配置信息
"""
# 检查兼容性
compat = check_compatibility()
if not compat['compatible']:
print("⚠️ 系统兼容性警告:")
for error in compat['errors']:
print(f" - {error}")
# 获取推荐配置
arch_config = ArchitectureDetector.get_recommended_config()
# 获取低端设备配置
low_mem_config = get_low_memory_config('auto')
arch_info = ArchitectureDetector.get_architecture()
return {
'compatible': compat['compatible'],
'architecture': arch_info,
'recommended': arch_config,
'low_memory': low_mem_config.get_status(),
'warnings': compat['warnings']
}