复查修复(三): 调度与扫描
- scan: .tmp/.part 过滤 + list_files 分批遍历(消除 10 万条截断误判) - Scheduler: 持锁只做标记, 锁外异步执行任务(长任务不再冻结调度) - cron 范围/步进边界校验;FTP 下载后大小校验防静默截断
This commit is contained in:
HYC Fixer
@@ -1307,6 +1307,12 @@ class MirrorSyncManager:
|
|||||||
ftp.retrbinary(f'RETR {remote_filename}', f.write, rest=start_pos)
|
ftp.retrbinary(f'RETR {remote_filename}', f.write, rest=start_pos)
|
||||||
else:
|
else:
|
||||||
ftp.retrbinary(f'RETR {remote_filename}', f.write)
|
ftp.retrbinary(f'RETR {remote_filename}', f.write)
|
||||||
|
|
||||||
|
# 校验下载大小与远端一致，防止静默截断
|
||||||
|
actual_size = os.path.getsize(local_path)
|
||||||
|
expected = file_info.get('size')
|
||||||
|
if expected is not None and expected > 0 and actual_size != expected:
|
||||||
|
raise Exception(f"文件大小不匹配: {actual_size} != {expected}")
|
||||||
|
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
+25
-9
@@ -103,16 +103,27 @@ class ScheduledTask:
|
|||||||
result.extend(self._parse_cron_part(sub.strip(), min_val, max_val))
|
result.extend(self._parse_cron_part(sub.strip(), min_val, max_val))
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# 处理范围: "1-5"
|
# 处理范围: "1-5"（校验边界，越界返回空）
|
||||||
if '-' in part:
|
if '-' in part:
|
||||||
start, end = part.split('-')
|
try:
|
||||||
return list(range(int(start), int(end) + 1))
|
start, end = part.split('-')
|
||||||
|
start, end = int(start), int(end)
|
||||||
|
if not (min_val <= start <= end <= max_val):
|
||||||
|
return []
|
||||||
|
return list(range(start, end + 1))
|
||||||
|
except ValueError:
|
||||||
|
return []
|
||||||
|
|
||||||
# 处理步进: "*/5"
|
# 处理步进: "*/5"
|
||||||
if '/' in part:
|
if '/' in part:
|
||||||
base, step = part.split('/')
|
base, step = part.split('/')
|
||||||
base_list = self._parse_cron_part(base or '*', min_val, max_val)
|
base_list = self._parse_cron_part(base or '*', min_val, max_val)
|
||||||
step = int(step)
|
try:
|
||||||
|
step = int(step)
|
||||||
|
if step <= 0:
|
||||||
|
return []
|
||||||
|
except ValueError:
|
||||||
|
return []
|
||||||
return base_list[::step]
|
return base_list[::step]
|
||||||
|
|
||||||
# 单个值
|
# 单个值
|
||||||
@@ -301,18 +312,23 @@ class Scheduler:
|
|||||||
logger.info("定时任务调度器已停止")
|
logger.info("定时任务调度器已停止")
|
||||||
|
|
||||||
def _run_loop(self):
|
def _run_loop(self):
|
||||||
"""运行循环"""
|
"""运行循环（持锁只做标记，锁外异步执行任务，避免长任务冻结调度）"""
|
||||||
while self._running:
|
while self._running:
|
||||||
try:
|
try:
|
||||||
now = datetime.now()
|
now = datetime.now()
|
||||||
|
|
||||||
|
due = []
|
||||||
with self._lock:
|
with self._lock:
|
||||||
for name, task in self.tasks.items():
|
for name, task in self.tasks.items():
|
||||||
if task.should_run_now():
|
if task.should_run_now():
|
||||||
# 使用线程池执行任务
|
due.append(task)
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
# 锁外异步执行，不阻塞后续任务的调度
|
||||||
executor.submit(task.run)
|
for task in due:
|
||||||
|
try:
|
||||||
|
threading.Thread(target=task.run, daemon=True).start()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"调度任务 {task.name} 启动失败: {e}")
|
||||||
|
|
||||||
time.sleep(self.check_interval)
|
time.sleep(self.check_interval)
|
||||||
|
|
||||||
|
|||||||
+502
-493
@@ -1,493 +1,502 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
"""
|
"""
|
||||||
定时同步调度器
|
定时同步调度器
|
||||||
负责本地数据和数据库之间的定时同步
|
负责本地数据和数据库之间的定时同步
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import json
|
import json
|
||||||
import hashlib
|
import hashlib
|
||||||
import threading
|
import threading
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Dict, List, Optional, Callable
|
from typing import Dict, List, Optional, Callable
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
# 添加项目根目录到路径
|
# 添加项目根目录到路径
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from core.database import DatabaseManager, get_db
|
from core.database import DatabaseManager, get_db
|
||||||
from core.scheduler import Scheduler, ScheduledTask
|
from core.scheduler import Scheduler, ScheduledTask
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class SyncScheduler:
|
class SyncScheduler:
|
||||||
"""同步调度器"""
|
"""同步调度器"""
|
||||||
|
|
||||||
def __init__(self, config: dict, db: DatabaseManager = None):
|
def __init__(self, config: dict, db: DatabaseManager = None):
|
||||||
self.config = config
|
self.config = config
|
||||||
self.db = db
|
self.db = db
|
||||||
self._running = False
|
self._running = False
|
||||||
self._executor = ThreadPoolExecutor(max_workers=4)
|
self._executor = ThreadPoolExecutor(max_workers=4)
|
||||||
|
|
||||||
# 同步配置
|
# 同步配置
|
||||||
self.sync_interval = config.get('database', {}).get('sync_interval', 60)
|
self.sync_interval = config.get('database', {}).get('sync_interval', 60)
|
||||||
self.auto_scan = config.get('auto_scan', True)
|
self.auto_scan = config.get('auto_scan', True)
|
||||||
self.scan_interval = config.get('scan_interval', 300) # 5分钟扫描一次
|
self.scan_interval = config.get('scan_interval', 300) # 5分钟扫描一次
|
||||||
|
|
||||||
# 同步状态
|
# 同步状态
|
||||||
self.last_sync_time = 0
|
self.last_sync_time = 0
|
||||||
self.last_scan_time = 0
|
self.last_scan_time = 0
|
||||||
self.sync_in_progress = False
|
self.sync_in_progress = False
|
||||||
self.scan_in_progress = False
|
self.scan_in_progress = False
|
||||||
|
|
||||||
# 回调函数
|
# 回调函数
|
||||||
self.on_file_added: Optional[Callable] = None
|
self.on_file_added: Optional[Callable] = None
|
||||||
self.on_file_deleted: Optional[Callable] = None
|
self.on_file_deleted: Optional[Callable] = None
|
||||||
self.on_file_updated: Optional[Callable] = None
|
self.on_file_updated: Optional[Callable] = None
|
||||||
self.on_sync_complete: Optional[Callable] = None
|
self.on_sync_complete: Optional[Callable] = None
|
||||||
|
|
||||||
# 定时同步执行器（注入 MirrorSyncManager.start_sync 或等效 callable）
|
# 定时同步执行器（注入 MirrorSyncManager.start_sync 或等效 callable）
|
||||||
self.sync_starter: Optional[Callable] = None
|
self.sync_starter: Optional[Callable] = None
|
||||||
|
|
||||||
# 待同步队列
|
# 待同步队列
|
||||||
self._pending_add = [] # 待添加的文件
|
self._pending_add = [] # 待添加的文件
|
||||||
self._pending_update = [] # 待更新的文件
|
self._pending_update = [] # 待更新的文件
|
||||||
self._pending_delete = [] # 待删除的文件
|
self._pending_delete = [] # 待删除的文件
|
||||||
|
|
||||||
# 定时任务调度器
|
# 定时任务调度器
|
||||||
self.task_scheduler = None
|
self.task_scheduler = None
|
||||||
self.scheduled_syncs: Dict[str, dict] = {}
|
self.scheduled_syncs: Dict[str, dict] = {}
|
||||||
|
|
||||||
def _init_scheduled_syncs(self):
|
def _init_scheduled_syncs(self):
|
||||||
"""初始化定时同步任务"""
|
"""初始化定时同步任务"""
|
||||||
if not self.config.get('enable_sync', True):
|
if not self.config.get('enable_sync', True):
|
||||||
return
|
return
|
||||||
|
|
||||||
# 从配置加载定时同步设置
|
# 从配置加载定时同步设置
|
||||||
sync_sources = self.config.get('sync_sources', {})
|
sync_sources = self.config.get('sync_sources', {})
|
||||||
scheduled_sources = {}
|
scheduled_sources = {}
|
||||||
|
|
||||||
for name, source_config in sync_sources.items():
|
for name, source_config in sync_sources.items():
|
||||||
schedule = source_config.get('schedule', {})
|
schedule = source_config.get('schedule', {})
|
||||||
if schedule.get('enabled', False):
|
if schedule.get('enabled', False):
|
||||||
scheduled_sources[name] = {
|
scheduled_sources[name] = {
|
||||||
'type': schedule.get('type', 'interval'), # 'cron' 或 'interval'
|
'type': schedule.get('type', 'interval'), # 'cron' 或 'interval'
|
||||||
'config': {
|
'config': {
|
||||||
'cron': schedule.get('cron'),
|
'cron': schedule.get('cron'),
|
||||||
'interval': schedule.get('interval', {}),
|
'interval': schedule.get('interval', {}),
|
||||||
'enabled': True
|
'enabled': True
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if scheduled_sources:
|
if scheduled_sources:
|
||||||
self.task_scheduler = Scheduler()
|
self.task_scheduler = Scheduler()
|
||||||
for name, sched_config in scheduled_sources.items():
|
for name, sched_config in scheduled_sources.items():
|
||||||
self.task_scheduler.add_task(
|
self.task_scheduler.add_task(
|
||||||
name=f"sync_{name}",
|
name=f"sync_{name}",
|
||||||
task_type=sched_config['type'],
|
task_type=sched_config['type'],
|
||||||
config=sched_config['config'],
|
config=sched_config['config'],
|
||||||
callback=self._create_sync_callback(name)
|
callback=self._create_sync_callback(name)
|
||||||
)
|
)
|
||||||
self.scheduled_syncs = scheduled_sources
|
self.scheduled_syncs = scheduled_sources
|
||||||
|
|
||||||
def _create_sync_callback(self, source_name: str):
|
def _create_sync_callback(self, source_name: str):
|
||||||
"""创建同步回调函数"""
|
"""创建同步回调函数"""
|
||||||
def sync_callback(task_name: str, config: dict):
|
def sync_callback(task_name: str, config: dict):
|
||||||
logger.info(f"定时同步任务触发: {source_name}")
|
logger.info(f"定时同步任务触发: {source_name}")
|
||||||
# 委托给注入的同步执行器（MirrorSyncManager.start_sync）
|
# 委托给注入的同步执行器（MirrorSyncManager.start_sync）
|
||||||
starter = self.sync_starter or self._default_sync_starter()
|
starter = self.sync_starter or self._default_sync_starter()
|
||||||
if starter:
|
if starter:
|
||||||
try:
|
try:
|
||||||
return bool(starter(source_name))
|
return bool(starter(source_name))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"触发同步 {source_name} 失败: {e}")
|
logger.error(f"触发同步 {source_name} 失败: {e}")
|
||||||
return False
|
return False
|
||||||
logger.error(f"无法触发同步 {source_name}: 未配置同步执行器(sync_starter)")
|
logger.error(f"无法触发同步 {source_name}: 未配置同步执行器(sync_starter)")
|
||||||
return False
|
return False
|
||||||
return sync_callback
|
return sync_callback
|
||||||
|
|
||||||
def _default_sync_starter(self):
|
def _default_sync_starter(self):
|
||||||
"""尝试从配置获取 MirrorSyncManager 实例（server.py 启动时注入）"""
|
"""尝试从配置获取 MirrorSyncManager 实例（server.py 启动时注入）"""
|
||||||
mgr = self.config.get('_sync_manager')
|
mgr = self.config.get('_sync_manager')
|
||||||
if mgr and hasattr(mgr, 'start_sync'):
|
if mgr and hasattr(mgr, 'start_sync'):
|
||||||
return mgr.start_sync
|
return mgr.start_sync
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def start(self):
|
def start(self):
|
||||||
"""启动同步调度器"""
|
"""启动同步调度器"""
|
||||||
if self._running:
|
if self._running:
|
||||||
logger.warning("SyncScheduler 已经运行中")
|
logger.warning("SyncScheduler 已经运行中")
|
||||||
return
|
return
|
||||||
|
|
||||||
self._running = True
|
self._running = True
|
||||||
self._executor.submit(self._sync_loop)
|
self._executor.submit(self._sync_loop)
|
||||||
self._executor.submit(self._scan_loop)
|
self._executor.submit(self._scan_loop)
|
||||||
|
|
||||||
# 初始化并启动定时同步
|
# 初始化并启动定时同步
|
||||||
self._init_scheduled_syncs()
|
self._init_scheduled_syncs()
|
||||||
if self.task_scheduler:
|
if self.task_scheduler:
|
||||||
self.task_scheduler.start()
|
self.task_scheduler.start()
|
||||||
|
|
||||||
logger.info(f"同步调度器已启动，间隔: {self.sync_interval}秒")
|
logger.info(f"同步调度器已启动，间隔: {self.sync_interval}秒")
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
"""停止同步调度器"""
|
"""停止同步调度器"""
|
||||||
self._running = False
|
self._running = False
|
||||||
|
|
||||||
# 停止定时任务调度器
|
# 停止定时任务调度器
|
||||||
if self.task_scheduler:
|
if self.task_scheduler:
|
||||||
self.task_scheduler.stop()
|
self.task_scheduler.stop()
|
||||||
self.task_scheduler = None
|
self.task_scheduler = None
|
||||||
|
|
||||||
self._executor.shutdown(wait=True)
|
self._executor.shutdown(wait=True)
|
||||||
logger.info("同步调度器已停止")
|
logger.info("同步调度器已停止")
|
||||||
|
|
||||||
def _sync_loop(self):
|
def _sync_loop(self):
|
||||||
"""同步循环"""
|
"""同步循环"""
|
||||||
while self._running:
|
while self._running:
|
||||||
try:
|
try:
|
||||||
if time.time() - self.last_sync_time >= self.sync_interval:
|
if time.time() - self.last_sync_time >= self.sync_interval:
|
||||||
self.perform_sync()
|
self.perform_sync()
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"同步循环错误: {e}")
|
logger.error(f"同步循环错误: {e}")
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
|
|
||||||
def _scan_loop(self):
|
def _scan_loop(self):
|
||||||
"""扫描循环 - 检测本地文件变化"""
|
"""扫描循环 - 检测本地文件变化"""
|
||||||
while self._running:
|
while self._running:
|
||||||
try:
|
try:
|
||||||
if self.auto_scan and time.time() - self.last_scan_time >= self.scan_interval:
|
if self.auto_scan and time.time() - self.last_scan_time >= self.scan_interval:
|
||||||
self.scan_local_files()
|
self.scan_local_files()
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"扫描循环错误: {e}")
|
logger.error(f"扫描循环错误: {e}")
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
|
|
||||||
def queue_add(self, file_info: dict):
|
def queue_add(self, file_info: dict):
|
||||||
"""队列添加文件"""
|
"""队列添加文件"""
|
||||||
self._pending_add.append(file_info)
|
self._pending_add.append(file_info)
|
||||||
|
|
||||||
def queue_update(self, file_info: dict):
|
def queue_update(self, file_info: dict):
|
||||||
"""队列更新文件"""
|
"""队列更新文件"""
|
||||||
self._pending_update.append(file_info)
|
self._pending_update.append(file_info)
|
||||||
|
|
||||||
def queue_delete(self, file_id: str):
|
def queue_delete(self, file_id: str):
|
||||||
"""队列删除文件"""
|
"""队列删除文件"""
|
||||||
self._pending_delete.append(file_id)
|
self._pending_delete.append(file_id)
|
||||||
|
|
||||||
def perform_sync(self):
|
def perform_sync(self):
|
||||||
"""执行同步"""
|
"""执行同步"""
|
||||||
if self.sync_in_progress:
|
if self.sync_in_progress:
|
||||||
logger.warning("同步已在进行中，跳过")
|
logger.warning("同步已在进行中，跳过")
|
||||||
return
|
return
|
||||||
|
|
||||||
self.sync_in_progress = True
|
self.sync_in_progress = True
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
logger.info("开始执行数据库同步...")
|
logger.info("开始执行数据库同步...")
|
||||||
|
|
||||||
# 同步待添加的文件
|
# 同步待添加的文件
|
||||||
added = 0
|
added = 0
|
||||||
for file_info in self._pending_add[:]:
|
for file_info in self._pending_add[:]:
|
||||||
try:
|
try:
|
||||||
self._sync_add_file(file_info)
|
self._sync_add_file(file_info)
|
||||||
self._pending_add.remove(file_info)
|
self._pending_add.remove(file_info)
|
||||||
added += 1
|
added += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"同步添加文件失败: {e}")
|
logger.error(f"同步添加文件失败: {e}")
|
||||||
|
|
||||||
# 同步待更新的文件
|
# 同步待更新的文件
|
||||||
updated = 0
|
updated = 0
|
||||||
for file_info in self._pending_update[:]:
|
for file_info in self._pending_update[:]:
|
||||||
try:
|
try:
|
||||||
self._sync_update_file(file_info)
|
self._sync_update_file(file_info)
|
||||||
self._pending_update.remove(file_info)
|
self._pending_update.remove(file_info)
|
||||||
updated += 1
|
updated += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"同步更新文件失败: {e}")
|
logger.error(f"同步更新文件失败: {e}")
|
||||||
|
|
||||||
# 同步待删除的文件
|
# 同步待删除的文件
|
||||||
deleted = 0
|
deleted = 0
|
||||||
for file_id in self._pending_delete[:]:
|
for file_id in self._pending_delete[:]:
|
||||||
try:
|
try:
|
||||||
self._sync_delete_file(file_id)
|
self._sync_delete_file(file_id)
|
||||||
self._pending_delete.remove(file_id)
|
self._pending_delete.remove(file_id)
|
||||||
deleted += 1
|
deleted += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"同步删除文件失败: {e}")
|
logger.error(f"同步删除文件失败: {e}")
|
||||||
|
|
||||||
# 同步统计
|
# 同步统计
|
||||||
self.db.reset_pending_count()
|
self.db.reset_pending_count()
|
||||||
|
|
||||||
self.last_sync_time = time.time()
|
self.last_sync_time = time.time()
|
||||||
duration = time.time() - start_time
|
duration = time.time() - start_time
|
||||||
|
|
||||||
logger.info(f"同步完成: 添加{added}, 更新{updated}, 删除{deleted}, 耗时{duration:.2f}秒")
|
logger.info(f"同步完成: 添加{added}, 更新{updated}, 删除{deleted}, 耗时{duration:.2f}秒")
|
||||||
|
|
||||||
# 回调
|
# 回调
|
||||||
if self.on_sync_complete:
|
if self.on_sync_complete:
|
||||||
self.on_sync_complete({
|
self.on_sync_complete({
|
||||||
'added': added,
|
'added': added,
|
||||||
'updated': updated,
|
'updated': updated,
|
||||||
'deleted': deleted,
|
'deleted': deleted,
|
||||||
'duration': duration
|
'duration': duration
|
||||||
})
|
})
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"同步过程错误: {e}")
|
logger.error(f"同步过程错误: {e}")
|
||||||
finally:
|
finally:
|
||||||
self.sync_in_progress = False
|
self.sync_in_progress = False
|
||||||
|
|
||||||
def scan_local_files(self):
|
def scan_local_files(self):
|
||||||
"""扫描本地文件"""
|
"""扫描本地文件"""
|
||||||
if self.scan_in_progress:
|
if self.scan_in_progress:
|
||||||
return
|
return
|
||||||
|
|
||||||
self.scan_in_progress = True
|
self.scan_in_progress = True
|
||||||
|
|
||||||
try:
|
try:
|
||||||
base_dir = self.config.get('base_dir', './downloads')
|
base_dir = self.config.get('base_dir', './downloads')
|
||||||
if not os.path.exists(base_dir):
|
if not os.path.exists(base_dir):
|
||||||
self.last_scan_time = time.time()
|
self.last_scan_time = time.time()
|
||||||
return
|
return
|
||||||
|
|
||||||
# 扫描文件
|
# 扫描文件（过滤传输中的临时文件）
|
||||||
scanned_files = []
|
scanned_files = []
|
||||||
for root, dirs, files in os.walk(base_dir):
|
for root, dirs, files in os.walk(base_dir):
|
||||||
for filename in files:
|
for filename in files:
|
||||||
full_path = os.path.join(root, filename)
|
if filename.endswith(('.tmp', '.part')):
|
||||||
rel_path = os.path.relpath(full_path, base_dir).replace("\\", "/")
|
continue
|
||||||
|
full_path = os.path.join(root, filename)
|
||||||
stat = os.stat(full_path)
|
rel_path = os.path.relpath(full_path, base_dir).replace("\\", "/")
|
||||||
file_info = {
|
|
||||||
'path': rel_path,
|
stat = os.stat(full_path)
|
||||||
'name': filename,
|
file_info = {
|
||||||
'size': stat.st_size,
|
'path': rel_path,
|
||||||
'mtime': stat.st_mtime,
|
'name': filename,
|
||||||
'ctime': stat.st_ctime
|
'size': stat.st_size,
|
||||||
}
|
'mtime': stat.st_mtime,
|
||||||
scanned_files.append(file_info)
|
'ctime': stat.st_ctime
|
||||||
|
}
|
||||||
# 与数据库对比
|
scanned_files.append(file_info)
|
||||||
db_files = self.db.list_files(limit=100000)
|
|
||||||
db_paths = {f.path for f in db_files if not f.is_dir}
|
# 与数据库对比（分批遍历，避免 limit 截断导致误判新增/删除）
|
||||||
|
db_paths = set()
|
||||||
# 检测新增
|
offset = 0
|
||||||
local_paths = {f['path'] for f in scanned_files}
|
BATCH = 5000
|
||||||
new_paths = local_paths - db_paths
|
while True:
|
||||||
|
batch = self.db.list_files(limit=BATCH, offset=offset)
|
||||||
for path in new_paths:
|
if not batch:
|
||||||
file_info = next((f for f in scanned_files if f['path'] == path), None)
|
break
|
||||||
if file_info:
|
db_paths.update(f.path for f in batch if not f.is_dir)
|
||||||
file_id = hashlib.md5(path.encode()).hexdigest()
|
offset += BATCH
|
||||||
self._sync_add_file({
|
|
||||||
'file_id': file_id,
|
# 检测新增
|
||||||
'path': path,
|
local_paths = {f['path'] for f in scanned_files}
|
||||||
'name': file_info['name'],
|
new_paths = local_paths - db_paths
|
||||||
'size': file_info['size'],
|
|
||||||
'updated_at': file_info['mtime']
|
for path in new_paths:
|
||||||
})
|
file_info = next((f for f in scanned_files if f['path'] == path), None)
|
||||||
|
if file_info:
|
||||||
# 检测删除
|
file_id = hashlib.md5(path.encode()).hexdigest()
|
||||||
deleted_paths = db_paths - local_paths
|
self._sync_add_file({
|
||||||
for path in deleted_paths:
|
'file_id': file_id,
|
||||||
record = self.db.get_file_by_path(path)
|
'path': path,
|
||||||
if record:
|
'name': file_info['name'],
|
||||||
self.db.delete_file(record.file_id)
|
'size': file_info['size'],
|
||||||
|
'updated_at': file_info['mtime']
|
||||||
self.last_scan_time = time.time()
|
})
|
||||||
|
|
||||||
except Exception as e:
|
# 检测删除
|
||||||
logger.error(f"扫描本地文件错误: {e}")
|
deleted_paths = db_paths - local_paths
|
||||||
finally:
|
for path in deleted_paths:
|
||||||
self.scan_in_progress = False
|
record = self.db.get_file_by_path(path)
|
||||||
|
if record:
|
||||||
def _sync_add_file(self, file_info: dict):
|
self.db.delete_file(record.file_id)
|
||||||
"""同步添加文件"""
|
|
||||||
existing = self.db.get_file_by_path(file_info['path'])
|
self.last_scan_time = time.time()
|
||||||
if existing:
|
|
||||||
# 已存在，更新
|
except Exception as e:
|
||||||
self.db.update_file(
|
logger.error(f"扫描本地文件错误: {e}")
|
||||||
existing.file_id,
|
finally:
|
||||||
size=file_info.get('size', 0),
|
self.scan_in_progress = False
|
||||||
updated_at=file_info.get('updated_at', time.time()),
|
|
||||||
hash=file_info.get('hash'),
|
def _sync_add_file(self, file_info: dict):
|
||||||
sync_status='synced'
|
"""同步添加文件"""
|
||||||
)
|
existing = self.db.get_file_by_path(file_info['path'])
|
||||||
else:
|
if existing:
|
||||||
# 新增
|
# 已存在，更新
|
||||||
file_id = file_info.get('file_id') or hashlib.md5(
|
self.db.update_file(
|
||||||
file_info['path'].encode()
|
existing.file_id,
|
||||||
).hexdigest()
|
size=file_info.get('size', 0),
|
||||||
|
updated_at=file_info.get('updated_at', time.time()),
|
||||||
self.db.add_file(
|
hash=file_info.get('hash'),
|
||||||
file_id=file_id,
|
sync_status='synced'
|
||||||
path=file_info['path'],
|
)
|
||||||
name=file_info['name'],
|
else:
|
||||||
size=file_info.get('size', 0),
|
# 新增
|
||||||
hash=file_info.get('hash'),
|
file_id = file_info.get('file_id') or hashlib.md5(
|
||||||
is_dir=False,
|
file_info['path'].encode()
|
||||||
created_at=file_info.get('created_at'),
|
).hexdigest()
|
||||||
updated_at=file_info.get('updated_at', time.time())
|
|
||||||
)
|
self.db.add_file(
|
||||||
|
file_id=file_id,
|
||||||
if self.on_file_added:
|
path=file_info['path'],
|
||||||
self.on_file_added(file_info)
|
name=file_info['name'],
|
||||||
|
size=file_info.get('size', 0),
|
||||||
def _sync_update_file(self, file_info: dict):
|
hash=file_info.get('hash'),
|
||||||
"""同步更新文件"""
|
is_dir=False,
|
||||||
file_id = file_info.get('file_id')
|
created_at=file_info.get('created_at'),
|
||||||
if file_id:
|
updated_at=file_info.get('updated_at', time.time())
|
||||||
self.db.update_file(
|
)
|
||||||
file_id,
|
|
||||||
size=file_info.get('size'),
|
if self.on_file_added:
|
||||||
updated_at=file_info.get('updated_at', time.time()),
|
self.on_file_added(file_info)
|
||||||
hash=file_info.get('hash'),
|
|
||||||
sync_status='synced'
|
def _sync_update_file(self, file_info: dict):
|
||||||
)
|
"""同步更新文件"""
|
||||||
|
file_id = file_info.get('file_id')
|
||||||
if self.on_file_updated:
|
if file_id:
|
||||||
self.on_file_updated(file_info)
|
self.db.update_file(
|
||||||
|
file_id,
|
||||||
def _sync_delete_file(self, file_id: str):
|
size=file_info.get('size'),
|
||||||
"""同步删除文件"""
|
updated_at=file_info.get('updated_at', time.time()),
|
||||||
self.db.delete_file(file_id)
|
hash=file_info.get('hash'),
|
||||||
|
sync_status='synced'
|
||||||
if self.on_file_deleted:
|
)
|
||||||
self.on_file_deleted({'file_id': file_id})
|
|
||||||
|
if self.on_file_updated:
|
||||||
def get_status(self) -> dict:
|
self.on_file_updated(file_info)
|
||||||
"""获取同步状态"""
|
|
||||||
return {
|
def _sync_delete_file(self, file_id: str):
|
||||||
'running': self._running,
|
"""同步删除文件"""
|
||||||
'last_sync_time': self.last_sync_time,
|
self.db.delete_file(file_id)
|
||||||
'last_scan_time': self.last_scan_time,
|
|
||||||
'sync_in_progress': self.sync_in_progress,
|
if self.on_file_deleted:
|
||||||
'scan_in_progress': self.scan_in_progress,
|
self.on_file_deleted({'file_id': file_id})
|
||||||
'pending_add': len(self._pending_add),
|
|
||||||
'pending_update': len(self._pending_update),
|
def get_status(self) -> dict:
|
||||||
'pending_delete': len(self._pending_delete),
|
"""获取同步状态"""
|
||||||
'pending_operations': self.db.get_pending_operations() if self.db else 0
|
return {
|
||||||
}
|
'running': self._running,
|
||||||
|
'last_sync_time': self.last_sync_time,
|
||||||
def force_sync(self):
|
'last_scan_time': self.last_scan_time,
|
||||||
"""强制立即同步"""
|
'sync_in_progress': self.sync_in_progress,
|
||||||
self.last_sync_time = 0
|
'scan_in_progress': self.scan_in_progress,
|
||||||
self.perform_sync()
|
'pending_add': len(self._pending_add),
|
||||||
|
'pending_update': len(self._pending_update),
|
||||||
|
'pending_delete': len(self._pending_delete),
|
||||||
# ==================== 文件操作包装器 ====================
|
'pending_operations': self.db.get_pending_operations() if self.db else 0
|
||||||
|
}
|
||||||
class DatabaseBackedFileOperations:
|
|
||||||
"""数据库支持的文件操作"""
|
def force_sync(self):
|
||||||
|
"""强制立即同步"""
|
||||||
def __init__(self, config: dict, db: DatabaseManager, scheduler: SyncScheduler = None):
|
self.last_sync_time = 0
|
||||||
self.config = config
|
self.perform_sync()
|
||||||
self.db = db
|
|
||||||
self.scheduler = scheduler
|
|
||||||
self.base_dir = config.get('base_dir', './downloads')
|
# ==================== 文件操作包装器 ====================
|
||||||
|
|
||||||
def add_file_record(self, path: str, name: str, size: int = 0,
|
class DatabaseBackedFileOperations:
|
||||||
hash: str = None, is_dir: bool = False) -> dict:
|
"""数据库支持的文件操作"""
|
||||||
"""添加文件记录到数据库"""
|
|
||||||
import hashlib
|
def __init__(self, config: dict, db: DatabaseManager, scheduler: SyncScheduler = None):
|
||||||
|
self.config = config
|
||||||
file_id = hashlib.md5(path.encode()).hexdigest()
|
self.db = db
|
||||||
|
self.scheduler = scheduler
|
||||||
file_info = {
|
self.base_dir = config.get('base_dir', './downloads')
|
||||||
'file_id': file_id,
|
|
||||||
'path': path,
|
def add_file_record(self, path: str, name: str, size: int = 0,
|
||||||
'name': name,
|
hash: str = None, is_dir: bool = False) -> dict:
|
||||||
'size': size,
|
"""添加文件记录到数据库"""
|
||||||
'hash': hash,
|
import hashlib
|
||||||
'is_dir': is_dir,
|
|
||||||
'created_at': time.time(),
|
file_id = hashlib.md5(path.encode()).hexdigest()
|
||||||
'updated_at': time.time()
|
|
||||||
}
|
file_info = {
|
||||||
|
'file_id': file_id,
|
||||||
if self.scheduler:
|
'path': path,
|
||||||
self.scheduler.queue_add(file_info)
|
'name': name,
|
||||||
else:
|
'size': size,
|
||||||
self.db.add_file(
|
'hash': hash,
|
||||||
file_id=file_id,
|
'is_dir': is_dir,
|
||||||
path=path,
|
'created_at': time.time(),
|
||||||
name=name,
|
'updated_at': time.time()
|
||||||
size=size,
|
}
|
||||||
hash=hash,
|
|
||||||
is_dir=is_dir,
|
if self.scheduler:
|
||||||
created_at=time.time(),
|
self.scheduler.queue_add(file_info)
|
||||||
updated_at=time.time()
|
else:
|
||||||
)
|
self.db.add_file(
|
||||||
|
file_id=file_id,
|
||||||
return file_info
|
path=path,
|
||||||
|
name=name,
|
||||||
def update_file_record(self, file_id: str, **kwargs):
|
size=size,
|
||||||
"""更新文件记录"""
|
hash=hash,
|
||||||
if self.scheduler:
|
is_dir=is_dir,
|
||||||
self.scheduler.queue_update({'file_id': file_id, **kwargs})
|
created_at=time.time(),
|
||||||
else:
|
updated_at=time.time()
|
||||||
self.db.update_file(file_id, **kwargs)
|
)
|
||||||
|
|
||||||
def delete_file_record(self, file_id: str, hard: bool = False):
|
return file_info
|
||||||
"""删除文件记录"""
|
|
||||||
if self.scheduler:
|
def update_file_record(self, file_id: str, **kwargs):
|
||||||
self.scheduler.queue_delete(file_id)
|
"""更新文件记录"""
|
||||||
else:
|
if self.scheduler:
|
||||||
self.db.delete_file(file_id, hard=hard)
|
self.scheduler.queue_update({'file_id': file_id, **kwargs})
|
||||||
|
else:
|
||||||
def record_download(self, file_path: str, file_size: int = 0,
|
self.db.update_file(file_id, **kwargs)
|
||||||
client_ip: str = None, duration: float = 0,
|
|
||||||
success: bool = True, error_message: str = None):
|
def delete_file_record(self, file_id: str, hard: bool = False):
|
||||||
"""记录下载"""
|
"""删除文件记录"""
|
||||||
self.db.add_download_record(
|
if self.scheduler:
|
||||||
file_path=file_path,
|
self.scheduler.queue_delete(file_id)
|
||||||
file_size=file_size,
|
else:
|
||||||
client_ip=client_ip,
|
self.db.delete_file(file_id, hard=hard)
|
||||||
duration=duration,
|
|
||||||
success=success,
|
def record_download(self, file_path: str, file_size: int = 0,
|
||||||
error_message=error_message
|
client_ip: str = None, duration: float = 0,
|
||||||
)
|
success: bool = True, error_message: str = None):
|
||||||
|
"""记录下载"""
|
||||||
# 更新下载计数
|
self.db.add_download_record(
|
||||||
record = self.db.get_file_by_path(file_path)
|
file_path=file_path,
|
||||||
if record:
|
file_size=file_size,
|
||||||
self.db.increment_download_count(record.file_id)
|
client_ip=client_ip,
|
||||||
|
duration=duration,
|
||||||
def record_cache_hit(self, cache_key: str, cache_type: str):
|
success=success,
|
||||||
"""记录缓存命中"""
|
error_message=error_message
|
||||||
record = self.db.get_cache_record(cache_key)
|
)
|
||||||
if record:
|
|
||||||
self.db.increment_cache_hits(cache_key)
|
# 更新下载计数
|
||||||
else:
|
record = self.db.get_file_by_path(file_path)
|
||||||
self.db.add_cache_record(
|
if record:
|
||||||
cache_key=cache_key,
|
self.db.increment_download_count(record.file_id)
|
||||||
cache_type=cache_type,
|
|
||||||
hits=1,
|
def record_cache_hit(self, cache_key: str, cache_type: str):
|
||||||
last_hit=time.time()
|
"""记录缓存命中"""
|
||||||
)
|
record = self.db.get_cache_record(cache_key)
|
||||||
|
if record:
|
||||||
|
self.db.increment_cache_hits(cache_key)
|
||||||
# ==================== 便捷函数 ====================
|
else:
|
||||||
|
self.db.add_cache_record(
|
||||||
def get_sync_scheduler(config: dict) -> SyncScheduler:
|
cache_key=cache_key,
|
||||||
"""获取同步调度器"""
|
cache_type=cache_type,
|
||||||
db = get_db(config)
|
hits=1,
|
||||||
return SyncScheduler(config, db)
|
last_hit=time.time()
|
||||||
|
)
|
||||||
|
|
||||||
def init_database_sync(config: dict, db=None) -> tuple:
|
|
||||||
"""初始化数据库和同步"""
|
# ==================== 便捷函数 ====================
|
||||||
if db is None:
|
|
||||||
db = get_db(config)
|
def get_sync_scheduler(config: dict) -> SyncScheduler:
|
||||||
scheduler = SyncScheduler(config, db)
|
"""获取同步调度器"""
|
||||||
file_ops = DatabaseBackedFileOperations(config, db, scheduler)
|
db = get_db(config)
|
||||||
|
return SyncScheduler(config, db)
|
||||||
return db, scheduler, file_ops
|
|
||||||
|
|
||||||
|
def init_database_sync(config: dict, db=None) -> tuple:
|
||||||
|
"""初始化数据库和同步"""
|
||||||
|
if db is None:
|
||||||
|
db = get_db(config)
|
||||||
|
scheduler = SyncScheduler(config, db)
|
||||||
|
file_ops = DatabaseBackedFileOperations(config, db, scheduler)
|
||||||
|
|
||||||
|
return db, scheduler, file_ops
|
||||||
|
|||||||
Reference in New Issue
Block a user