复查修复(二): 同步模块与安全遗留

- 修复 remove_sync_source 引入的 stop_sync KeyError 回归(容错 get) - stop_sync 不再删除存活线程条目(防双 worker);FTP 子目录递归传真名(不再 KeyError) - 远程文件名统一 _safe_remote_name 校验(FTP/SFTP/HTTP 防路径穿越) - temp 任务名加随机后缀防碰撞;temp 状态/线程完成后清理(防 sync_state.json 膨胀) - save_sync_state 快照+原子写(临时文件+os.replace),持锁调用不死锁 - URL 打印脱敏(user:pass@ -> ***@) - _need_sync_http size=0 不再全量重下(仅按存在性) - cron 同一分钟去重;PyPI 进度只更新当前源 - v2 start_sync 不再把 bool 当 task_id - UserRecord.to_dict 脱敏(不返回 password_hash/token),新增 to_dict_private/get_user_with_password - config_hotreload 单配置源(set+persist 与热重载一致);server.py 用 get_all() - 会话创建时顺带清理过期项;JSON stats/历史读改写加锁 - 解压目标目录先校验;FTP RETR 命令注入防护;rsync --delete 目标保护 - git/urlopen 补超时;cleanup_completed_tasks 删旧留新
2026-09-02 00:39:12 +08:00
parent fc72759a92
commit 82875b710a
10 changed files with 957 additions and 810 deletions
@@ -97,10 +97,14 @@ class MirrorSyncManager:
            self.sync_status = {}
    
    def save_sync_state(self):
-        """保存同步状态"""
+        """保存同步状态（快照 + 原子写，可在持锁状态下调用，不会死锁）"""
        try:
-            with open(self.sync_state_file, 'w', encoding='utf-8') as f:
-                json.dump(self.sync_status, f, ensure_ascii=False, indent=2)
+            # dict() 浅拷贝是单条 C 操作，并发修改安全
+            snapshot = dict(self.sync_status)
+            tmp_file = self.sync_state_file + '.tmp'
+            with open(tmp_file, 'w', encoding='utf-8') as f:
+                json.dump(snapshot, f, ensure_ascii=False, indent=2)
+            os.replace(tmp_file, self.sync_state_file)
        except Exception as e:
            print(f"保存同步状态失败: {e}")

@@ -249,18 +253,18 @@ class MirrorSyncManager:
        if name not in self.sync_sources:
            return False

-        if name in self.sync_threads and self.sync_threads[name].is_alive():
-            return True  # 已经在运行
-
-        # 设置运行标志，确保同步循环可以执行
-        self.running = True
-
        with self.sync_lock:
+            existing = self.sync_threads.get(name)
+            if existing and existing.is_alive():
+                return True  # 已经在运行（检查+记录在同一把锁内，防双 worker 竞态）
+
+            # 设置运行标志，确保同步循环可以执行
+            self.running = True
            self.sync_status[name]['status'] = 'syncing'
            self.sync_status[name]['error'] = None

-        thread = Thread(target=self._sync_worker, args=(name,), daemon=True)
-        self.sync_threads[name] = thread
+            thread = Thread(target=self._sync_worker, args=(name,), daemon=True)
+            self.sync_threads[name] = thread
        thread.start()
        return True

@@ -280,12 +284,14 @@ class MirrorSyncManager:
        if not packages or not isinstance(packages, list):
            return {"success": False, "error": "请提供有效的包名列表"}

-        # 生成临时任务ID
+        # 生成临时任务ID（随机后缀防同秒碰撞）
        import time
-        task_id = f"temp_sync_{int(time.time())}"
+        import uuid
+        _ts = int(time.time())
+        task_id = f"temp_sync_{_ts}_{uuid.uuid4().hex[:6]}"

        # 使用临时任务名进行同步
-        temp_name = f"{source_name}_temp_{int(time.time())}"
+        temp_name = f"{source_name}_temp_{_ts}_{uuid.uuid4().hex[:6]}"

        # 设置状态
        with self.sync_lock:
@@ -333,17 +339,27 @@ class MirrorSyncManager:
                    self.sync_status[temp_name]['status'] = 'error'
                    self.sync_status[temp_name]['error'] = str(e)
            print(f"[Temp Sync] 临时同步失败: {e}")
+        finally:
+            # 清理临时条目，防止 sync_state.json 无限膨胀
+            with self.sync_lock:
+                self.sync_status.pop(temp_name, None)
+                self.sync_threads.pop(temp_name, None)
+            self.save_sync_state()

    def stop_sync(self, name):
-        """停止同步指定源 - 立即停止"""
-        # 立即设置状态为停止
+        """停止同步指定源 - 请求停止
+
+        注意: 不删除 sync_threads 条目(删除后 start_sync 会再起一个 worker,
+        与仍在运行的旧线程形成双 worker)。线程结束后 is_alive() 为 False,
+        start_sync 会自然复用该条目。
+        """
+        # 设置状态为停止（源可能已从 sync_status 移除,容错）
        with self.sync_lock:
-            self.sync_status[name]['status'] = 'stopped'
+            status = self.sync_status.get(name)
+            if status is not None:
+                status['status'] = 'stopped'
        # 设置停止标志，让线程提前退出
        self.running = False
-        # 立即返回，不等待线程结束
-        if name in self.sync_threads:
-            del self.sync_threads[name]
        # 短暂等待后重置运行标志
        import time
        time.sleep(0.5)
@@ -364,7 +380,7 @@ class MirrorSyncManager:
        print(f"[SYNC] 开始同步: {name}")
        source_config = self.sync_sources[name]
        sync_type = source_config.get('type', 'http')
-        print(f"[SYNC] 类型: {sync_type}, URL: {source_config.get('url', 'N/A')}")
+        print(f"[SYNC] 类型: {sync_type}, URL: {self._redact_url(source_config.get('url', 'N/A'))}")

        try:
            if sync_type in ('http', 'https'):
@@ -394,8 +410,12 @@ class MirrorSyncManager:
                raise ValueError(f"不支持的同步类型: {sync_type}")
            
            with self.sync_lock:
-                self.sync_status[name]['status'] = 'completed'
-                self.sync_status[name]['last_sync'] = datetime.now().isoformat()
+                if not self.running:
+                    # 被 stop_sync 中断：标记为停止而非完成
+                    self.sync_status[name]['status'] = 'stopped'
+                else:
+                    self.sync_status[name]['status'] = 'completed'
+                    self.sync_status[name]['last_sync'] = datetime.now().isoformat()
        
        except Exception as e:
            with self.sync_lock:
@@ -403,6 +423,9 @@ class MirrorSyncManager:
                self.sync_status[name]['error'] = str(e)
        
        finally:
+            # 清理线程条目（start_sync 通过 is_alive 判断, 清理后允许重新 start）
+            with self.sync_lock:
+                self.sync_threads.pop(name, None)
            self.save_sync_state()
    
    def _sync_http(self, name, config, specific_packages=None):
@@ -420,7 +443,7 @@ class MirrorSyncManager:

        # 获取文件列表
        source_url = config.get('url', '')
-        print(f"[HTTP Sync] 源URL: {source_url}")
+        print(f"[HTTP Sync] 源URL: {self._redact_url(source_url)}")
        file_list = self._get_http_file_list(source_url, config, specific_packages)

        total_files = len(file_list)
@@ -445,10 +468,13 @@ class MirrorSyncManager:
            with self.sync_lock:
                self.sync_status[name]['files_synced'] = synced_count

-            # 获取本地路径
+            # 获取本地路径（先做路径穿越校验）
            filename = file_info.get('name', '')
            if not filename:
                continue
+            if self._safe_remote_name(filename) is None:
+                print(f"[HTTP Sync] 跳过不安全的文件名: {filename!r}")
+                continue

            # 处理子目录
            local_path = os.path.join(target_dir, filename)
@@ -476,6 +502,29 @@ class MirrorSyncManager:

        print(f"[HTTP Sync] 完成: 成功 {synced_count}, 失败 {failed_count}")
    
+    @staticmethod
+    def _safe_remote_name(filename):
+        """校验远程文件名/相对路径，拒绝路径穿越（.. 段、绝对路径、反斜杠）"
+
+        允许 '/' 分隔的多级子路径（HTTP 列表常见），但拒绝任何 '..' 段。
+        """
+        if not filename or filename in ('.', '..'):
+            return None
+        if filename.startswith('/') or '\\' in filename:
+            return None
+        for seg in filename.split('/'):
+            if seg in ('', '.', '..'):
+                return None
+        return filename
+
+    @staticmethod
+    def _redact_url(url):
+        """脱敏 URL 中的 user:pass@ 段（防止凭证进日志/状态文件）"""
+        if not url:
+            return url
+        import re as _re
+        return _re.sub(r'(://)([^/@\s]+)@', r'\1***@', str(url))
+
    def _sync_ftp(self, name, config):
        """FTP同步实现"""
        try:
@@ -519,15 +568,19 @@ class MirrorSyncManager:
            for filename, file_info in remote_files.items():
                if not self.running:
                    break
-                local_path = os.path.join(target_dir, filename)
+                safe_name = self._safe_remote_name(filename)
+                if safe_name is None:
+                    continue
+                local_path = os.path.join(target_dir, safe_name)
                if file_info['is_dir']:
                    sub_config = config.copy()
-                    sub_config['remote_path'] = os.path.join(remote_path, filename).replace('\\', '/')
-                    sub_config['target'] = os.path.join(config.get('target', name), filename)
-                    self._sync_ftp(name + '/' + filename, sub_config)
+                    sub_config['remote_path'] = os.path.join(remote_path, safe_name).replace('\\', '/')
+                    sub_config['target'] = os.path.join(config.get('target', name), safe_name)
+                    # 递归沿用顶层 name，避免子目录用假名导致 sync_status KeyError
+                    self._sync_ftp(name, sub_config)
                else:
-                    if self._need_sync_ftp(ftp, filename, local_path, file_info):
-                        if self._download_ftp_file(ftp, filename, local_path, file_info):
+                    if self._need_sync_ftp(ftp, safe_name, local_path, file_info):
+                        if self._download_ftp_file(ftp, safe_name, local_path, file_info):
                            synced_count += 1

            with self.sync_lock:
@@ -603,13 +656,12 @@ class MirrorSyncManager:
        for file_attr in file_list:
            if not self.running:
                break
-            remote_filename = file_attr.filename
+            remote_filename = self._safe_remote_name(file_attr.filename)
+            if remote_filename is None:
+                continue
            remote_filepath = os.path.join(remote_path, remote_filename).replace('\\', '/')
            local_filepath = os.path.join(local_path, remote_filename)
            
-            if remote_filename in ['.', '..']:
-                continue
-            
            if file_attr.st_mode & 0o40000:
                sub_synced = self._sync_sftp_directory(sftp, remote_filepath, local_filepath, sync_name)
                synced_count += sub_synced
@@ -679,7 +731,13 @@ class MirrorSyncManager:
        import subprocess

        source = config.get('source', config.get('url', ''))
-        target_dir = os.path.join(self.config['base_dir'], config.get('target', name))
+        target_rel = config.get('target', name) or name
+        if target_rel in ('.', './', '') or os.path.isabs(target_rel):
+            raise Exception(f"Rsync 同步目标不安全: {target_rel!r} (禁止同步到 '.' 或绝对路径, 防止 --delete 清空目录)")
+        target_dir = os.path.join(self.config['base_dir'], target_rel)
+        # 目标目录不得等于 base_dir 本身
+        if os.path.realpath(target_dir) == os.path.realpath(self.config['base_dir']):
+            raise Exception("Rsync 同步目标不允许是下载根目录本身 (--delete 会清空全部文件)")
        os.makedirs(target_dir, exist_ok=True)

        print(f"开始Rsync同步 {name} -> {target_dir}")
@@ -725,10 +783,15 @@ class MirrorSyncManager:
        if os.path.exists(os.path.join(target_dir, '.git')):
            # 已存在，执行git pull
            try:
-                subprocess.run(['git', 'fetch', '--all'], cwd=target_dir, check=True, capture_output=True)
-                subprocess.run(['git', 'reset', '--hard', f'origin/{branch}'], cwd=target_dir, check=True, capture_output=True)
+                git_timeout = config.get('git_timeout', 1800)
+                subprocess.run(['git', 'fetch', '--all'], cwd=target_dir, check=True,
+                               capture_output=True, timeout=git_timeout)
+                subprocess.run(['git', 'reset', '--hard', f'origin/{branch}'], cwd=target_dir,
+                               check=True, capture_output=True, timeout=git_timeout)
            except subprocess.CalledProcessError as e:
                raise Exception(f"Git pull失败: {e}")
+            except subprocess.TimeoutExpired:
+                raise Exception(f"Git pull超时({git_timeout}s)")
        else:
            # 克隆新仓库
            cmd = ['git', 'clone']
@@ -739,9 +802,12 @@ class MirrorSyncManager:
            cmd.extend([repo_url, target_dir])

            try:
-                subprocess.run(cmd, check=True, capture_output=True)
+                git_timeout = config.get('git_timeout', 1800)
+                subprocess.run(cmd, check=True, capture_output=True, timeout=git_timeout)
            except subprocess.CalledProcessError as e:
                raise Exception(f"Git clone失败: {e}")
+            except subprocess.TimeoutExpired:
+                raise Exception(f"Git clone超时({git_timeout}s)")

    def _sync_s3(self, name, config):
        """AWS S3兼容存储同步 (S3/OSS/COS/MinIO等)"""
@@ -888,7 +954,7 @@ class MirrorSyncManager:

        if is_pypi:
            print(f"[HTTP] 检测到PyPI索引，使用PyPI专用解析")
-            return self._get_pypi_file_list(base_url, config, specific_packages)
+            return self._get_pypi_file_list(base_url, config, specific_packages, sync_name=name)

        if 'api_url' in config:
            try:
@@ -898,7 +964,7 @@ class MirrorSyncManager:
                    encoded_auth = base64.b64encode(auth_string.encode()).decode()
                    req.add_header('Authorization', f'Basic {encoded_auth}')

-                with urllib.request.urlopen(req) as response:
+                with urllib.request.urlopen(req, timeout=60) as response:
                    api_data = json.loads(response.read().decode())
                    if 'files' in api_data:
                        for file_info in api_data['files']:
@@ -920,7 +986,7 @@ class MirrorSyncManager:

        return file_list

-    def _get_pypi_file_list(self, base_url, config, specific_packages=None):
+    def _get_pypi_file_list(self, base_url, config, specific_packages=None, sync_name=None):
        """获取PyPI文件列表

        Args:
@@ -986,12 +1052,10 @@ class MirrorSyncManager:
            if not self.running:
                break

-            # 更新进度
+            # 更新进度（只更新当前源，避免污染其他源状态）
            with self.sync_lock:
-                if hasattr(self, 'sync_status') and self.sync_status:
-                    for name in self.sync_status:
-                        if 'total_files' in self.sync_status[name]:
-                            self.sync_status[name]['files_synced'] = i
+                if sync_name and sync_name in self.sync_status:
+                    self.sync_status[sync_name]['files_synced'] = i

            # 获取每个包的文件列表
            package_url = f"{package_index_url}{package_name}/"
@@ -1098,8 +1162,11 @@ class MirrorSyncManager:
        """检查HTTP文件是否需要同步"""
        if not os.path.exists(local_path):
            return True
-        local_size = os.path.getsize(local_path)
        remote_size = file_info.get('size', 0)
+        if remote_size <= 0:
+            # 远端大小未知（PyPI/HTML 列表恒为 0）：仅按存在性判断，避免每次全量重下
+            return False
+        local_size = os.path.getsize(local_path)
        return local_size != remote_size
    
    def _need_sync_ftp(self, ftp, remote_filename, local_path, file_info):
@@ -1213,7 +1280,7 @@ class MirrorSyncManager:
                return False
                
        except Exception as e:
-            print(f"下载HTTP文件失败 {url}: {e}")
+            print(f"下载HTTP文件失败 {self._redact_url(url)}: {e}")
            if os.path.exists(temp_path):
                os.remove(temp_path)
            return False
@@ -1230,6 +1297,10 @@ class MirrorSyncManager:
                else:
                    return True
            
+            # 防 FTP 命令注入: 文件名含 CR/LF 时拒绝
+            if any(c in remote_filename for c in ('\r', '\n')):
+                raise Exception(f"文件名含非法字符: {remote_filename!r}")
+
            with open(local_path, mode) as f:
                if start_pos > 0:
                    ftp.voidcmd('TYPE I')