feat(scripts): add incremental update system with version tracking and diff utilities
Add complete incremental update infrastructure: - VERSION: project version tracking file - scripts/version_utils.py: shared version parsing and comparison utilities - scripts/build_update.py: build incremental update package from git diff - scripts/apply_update.py: apply offline incremental update - scripts/merge_config.py: safe config merge logic for update safety - scripts/migrate_db_to_appdata.py: migrate DB from project dir to APPDATA Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
PM-pinou
@@ -0,0 +1,386 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
天璇增量更新应用脚本 — 在离线机上运行
|
||||
|
||||
用于将增量更新包应用到离线机上的项目目录。
|
||||
|
||||
Usage:
|
||||
runtime\\python\\python.exe scripts\\apply_update.py <update.zip>
|
||||
runtime\\python\\python.exe scripts\\apply_update.py --rollback
|
||||
|
||||
安全保证:
|
||||
1. 更新前完整备份当前配置
|
||||
2. 永不覆盖 %APPDATA%/TianXuan/ 下的用户数据
|
||||
3. config.yaml 用户编辑通过合并保留
|
||||
4. 完整 SHA256 校验防止传输损坏
|
||||
5. 可回滚到最近一次备份
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# ── Constants ──────────────────────────────────────────────────────────
|
||||
PROJECT_DIR = Path(__file__).resolve().parent.parent
|
||||
PYTHON = str(PROJECT_DIR / 'runtime' / 'python' / 'python.exe')
|
||||
|
||||
if not os.path.exists(PYTHON):
|
||||
# Fallback: use system python if portable not found
|
||||
PYTHON = sys.executable
|
||||
|
||||
APPDATA_DIR = Path(os.environ.get('APPDATA', str(Path.home() / '.tianxuan'))) / 'TianXuan'
|
||||
BACKUP_DIR = APPDATA_DIR / 'backups'
|
||||
VERSION_FILE = PROJECT_DIR / 'VERSION'
|
||||
CONFIG_FILE = PROJECT_DIR / 'config' / 'config.yaml'
|
||||
SCRIPTS_DIR = PROJECT_DIR / 'scripts'
|
||||
MERGE_SCRIPT = SCRIPTS_DIR / 'merge_config.py'
|
||||
|
||||
# ── Helper Functions ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def log(msg: str):
|
||||
print(f" {msg}")
|
||||
|
||||
|
||||
def step(n: int, total: int, msg: str):
|
||||
print(f"\n[{n}/{total}] {msg}")
|
||||
|
||||
|
||||
def verify_sha256(filepath: Path, expected: str) -> bool:
|
||||
"""Verify file SHA256 matches expected hash."""
|
||||
h = hashlib.sha256()
|
||||
with open(filepath, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(65536), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest() == expected
|
||||
|
||||
|
||||
def clean_pycache(root_dir: Path):
|
||||
"""Recursively remove all __pycache__ directories."""
|
||||
count = 0
|
||||
for dirpath, dirnames, _ in os.walk(str(root_dir)):
|
||||
if '__pycache__' in dirnames:
|
||||
cache_dir = os.path.join(dirpath, '__pycache__')
|
||||
try:
|
||||
shutil.rmtree(cache_dir)
|
||||
count += 1
|
||||
except PermissionError:
|
||||
log(f" [WARN] 无法删除 {cache_dir} (权限拒绝)")
|
||||
if count > 0:
|
||||
log(f" 清理了 {count} 个 __pycache__ 目录")
|
||||
|
||||
|
||||
def get_current_version() -> str:
|
||||
"""Read current version from VERSION file."""
|
||||
if VERSION_FILE.exists():
|
||||
return VERSION_FILE.read_text(encoding='utf-8').strip()
|
||||
return 'v0.0.0'
|
||||
|
||||
|
||||
# ── Main Logic ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def do_rollback():
|
||||
"""Restore from the most recent backup."""
|
||||
if not BACKUP_DIR.exists():
|
||||
print("[ERROR] 没有找到备份目录")
|
||||
return 1
|
||||
|
||||
backups = sorted(BACKUP_DIR.iterdir(), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
if not backups:
|
||||
print("[ERROR] 备份目录为空")
|
||||
return 1
|
||||
|
||||
latest = backups[0]
|
||||
config_backup = latest / 'config.yaml'
|
||||
version_backup = latest / 'VERSION'
|
||||
|
||||
print(f"从 {latest.name} 回滚...")
|
||||
|
||||
if config_backup.exists():
|
||||
shutil.copy2(str(config_backup), str(CONFIG_FILE))
|
||||
log("配置已恢复")
|
||||
|
||||
if version_backup.exists():
|
||||
shutil.copy2(str(version_backup), str(VERSION_FILE))
|
||||
log("VERSION 已恢复")
|
||||
|
||||
# Restore project snapshot (if exists)
|
||||
snapshot_dir = latest / 'project_snapshot'
|
||||
if snapshot_dir.exists():
|
||||
log("恢复项目文件快照...")
|
||||
for item in snapshot_dir.iterdir():
|
||||
dest = PROJECT_DIR / item.name
|
||||
if item.is_dir():
|
||||
if dest.exists():
|
||||
shutil.rmtree(str(dest))
|
||||
shutil.copytree(str(item), str(dest))
|
||||
else:
|
||||
shutil.copy2(str(item), str(dest))
|
||||
log("项目文件已恢复")
|
||||
|
||||
print("\n✅ 回滚完成！请重启天璇服务")
|
||||
return 0
|
||||
|
||||
|
||||
def apply_update(zip_path: Path):
|
||||
"""Apply an update package."""
|
||||
total_steps = 10
|
||||
|
||||
print("=" * 50)
|
||||
print(" 天璇 增量更新 — 应用工具")
|
||||
print("=" * 50)
|
||||
log(f"更新包: {zip_path.name}")
|
||||
log(f"当前版本: {get_current_version()}")
|
||||
log(f"备份目录: {BACKUP_DIR}")
|
||||
|
||||
# Step 1: Validate package
|
||||
step(1, total_steps, "验证更新包完整性")
|
||||
|
||||
if not zip_path.exists():
|
||||
print(f"[ERROR] 更新包不存在: {zip_path}")
|
||||
return 1
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path, 'r') as zf:
|
||||
if zf.testzip() is not None:
|
||||
print(f"[CORRUPT] 更新包损坏！第一个损坏文件: {zf.testzip()}")
|
||||
return 1
|
||||
# Read manifest
|
||||
if 'version.json' not in zf.namelist():
|
||||
print("[ERROR] 更新包缺少 version.json")
|
||||
return 1
|
||||
manifest = json.loads(zf.read('version.json'))
|
||||
except Exception as e:
|
||||
print(f"[ERROR] 无法读取更新包: {e}")
|
||||
return 1
|
||||
|
||||
log(f" {zip_path.stat().st_size / 1024:.1f} KB — 校验通过")
|
||||
|
||||
# Step 2: Check version compatibility
|
||||
step(2, total_steps, "检查版本兼容性")
|
||||
|
||||
current_ver = get_current_version()
|
||||
from_ver = manifest.get('from_version', '')
|
||||
|
||||
if current_ver != from_ver:
|
||||
print(f"[VERSION_MISMATCH] 版本不匹配！")
|
||||
print(f" 当前: {current_ver}")
|
||||
print(f" 更新包起始: {from_ver}")
|
||||
print(f" 更新包目标: {manifest.get('to_version', '?')}")
|
||||
print(f"\n提示: 如需强制应用，请更新 VERSION 文件后重试")
|
||||
return 1
|
||||
|
||||
log(f" 版本一致: {current_ver} → {manifest.get('to_version', '?')}")
|
||||
|
||||
# Step 3: Verify file checksums
|
||||
step(3, total_steps, "校验文件 SHA256")
|
||||
|
||||
with zipfile.ZipFile(zip_path, 'r') as zf:
|
||||
for rel_path, file_info in manifest.get('files', {}).items():
|
||||
arcname = f'files/{rel_path}'
|
||||
if arcname not in zf.namelist():
|
||||
print(f"[ERROR] 包中缺少文件: {arcname}")
|
||||
return 1
|
||||
# Extract to temp and verify
|
||||
zf.extract(arcname, str(PROJECT_DIR))
|
||||
extracted = PROJECT_DIR / arcname
|
||||
if extracted.exists() and not verify_sha256(extracted, file_info['sha256']):
|
||||
print(f"[CORRUPT] 文件校验失败: {rel_path}")
|
||||
extracted.unlink()
|
||||
return 1
|
||||
extracted.unlink()
|
||||
|
||||
log(f" {len(manifest.get('files', {}))} 个文件校验通过")
|
||||
|
||||
# Step 4: Backup current config
|
||||
step(4, total_steps, "备份当前配置")
|
||||
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
version_dir = BACKUP_DIR / f'{manifest.get("to_version", "unknown")}_{timestamp}'
|
||||
version_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if CONFIG_FILE.exists():
|
||||
shutil.copy2(str(CONFIG_FILE), str(version_dir / 'config.yaml'))
|
||||
log("config.yaml 已备份")
|
||||
|
||||
if VERSION_FILE.exists():
|
||||
shutil.copy2(str(VERSION_FILE), str(version_dir / 'VERSION'))
|
||||
log("VERSION 已备份")
|
||||
|
||||
# Step 5: Delete removed files
|
||||
step(5, total_steps, "删除已废弃的文件")
|
||||
|
||||
for rel_path in manifest.get('deleted', []):
|
||||
target = PROJECT_DIR / rel_path
|
||||
if target.exists():
|
||||
try:
|
||||
if target.is_dir():
|
||||
shutil.rmtree(str(target))
|
||||
else:
|
||||
target.unlink()
|
||||
log(f" 删除: {rel_path}")
|
||||
except Exception as e:
|
||||
log(f" [WARN] 删除失败 {rel_path}: {e}")
|
||||
|
||||
# Step 6: Extract changed files
|
||||
# config.yaml is extracted but NOT moved — left in files/ for the merge step
|
||||
step(6, total_steps, "解压更新文件")
|
||||
|
||||
extracted_count = 0
|
||||
new_config_yaml = None # will hold path to extracted new config.yaml
|
||||
with zipfile.ZipFile(zip_path, 'r') as zf:
|
||||
for name in zf.namelist():
|
||||
if not name.startswith('files/'):
|
||||
continue
|
||||
rel_path = name[6:] # strip 'files/' prefix
|
||||
zf.extract(name, str(PROJECT_DIR))
|
||||
|
||||
if rel_path == 'config/config.yaml':
|
||||
# Don't move — keep in files/ for merge step 7
|
||||
new_config_yaml = PROJECT_DIR / name
|
||||
extracted_count += 1
|
||||
continue
|
||||
|
||||
# Move from files/ subdir to project root
|
||||
src = PROJECT_DIR / name
|
||||
dst = PROJECT_DIR / rel_path
|
||||
if src != dst:
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if src.exists():
|
||||
shutil.move(str(src), str(dst))
|
||||
extracted_count += 1
|
||||
|
||||
log(f" 已解压 {extracted_count} 个文件")
|
||||
|
||||
# Step 7: Merge config.yaml
|
||||
step(7, total_steps, "合并配置文件")
|
||||
|
||||
user_config = version_dir / 'config.yaml' # backed-up user config with edits
|
||||
|
||||
if user_config.exists() and new_config_yaml and new_config_yaml.exists() and MERGE_SCRIPT.exists():
|
||||
log("合并用户配置与更新配置...")
|
||||
# --old: user's backed-up config (preserves their edits)
|
||||
# --new: config.yaml from the update package (new keys to add)
|
||||
# --output: the live config file
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(MERGE_SCRIPT),
|
||||
'--old', str(user_config),
|
||||
'--new', str(new_config_yaml),
|
||||
'--output', str(CONFIG_FILE)],
|
||||
capture_output=True, text=True, cwd=str(PROJECT_DIR)
|
||||
)
|
||||
print(result.stdout.strip())
|
||||
if result.returncode != 0:
|
||||
log(f" [WARN] 配置合并异常: {result.stderr.strip()}")
|
||||
elif user_config.exists() and MERGE_SCRIPT.exists():
|
||||
# New config not in package — keep user config unchanged
|
||||
log(" 更新包不含 config.yaml，保留现有配置")
|
||||
# Restore the backup since nothing new to merge
|
||||
shutil.copy2(str(user_config), str(CONFIG_FILE))
|
||||
else:
|
||||
log(" 无备份配置，保留现有 config.yaml")
|
||||
|
||||
# Clean up extracted files/ directory
|
||||
files_dir = PROJECT_DIR / 'files'
|
||||
if files_dir.exists():
|
||||
shutil.rmtree(str(files_dir))
|
||||
|
||||
# Step 8: Clean __pycache__
|
||||
step(8, total_steps, "清理 Python 缓存")
|
||||
|
||||
clean_pycache(PROJECT_DIR)
|
||||
# Also clean in runtime (portable python)
|
||||
runtime_dir = PROJECT_DIR / 'runtime'
|
||||
if runtime_dir.exists():
|
||||
clean_pycache(runtime_dir)
|
||||
|
||||
# Step 9: Run Django migrations (always, since migrate is idempotent)
|
||||
step(9, total_steps, "运行数据库迁移")
|
||||
|
||||
python_exe = PYTHON
|
||||
migrate_cmd = [python_exe, 'manage.py', 'migrate']
|
||||
log(f" 执行: {' '.join(migrate_cmd)}")
|
||||
|
||||
result = subprocess.run(migrate_cmd, capture_output=True, text=True, cwd=str(PROJECT_DIR))
|
||||
print(result.stdout.strip())
|
||||
if result.returncode != 0:
|
||||
print(f"[MIGRATE_ERROR] 迁移失败: {result.stderr.strip()}")
|
||||
print(" 请运行 python scripts/apply_update.py --rollback 回滚")
|
||||
return 1
|
||||
|
||||
if 'Applying' in result.stdout or manifest.get('run_migrate'):
|
||||
log(" 数据库 schema 已更新 ✓")
|
||||
else:
|
||||
log(" 数据库已是最新 ✓")
|
||||
|
||||
# ── Ensure tls_ref_data table ─────────────────────────────────────
|
||||
# import_tlsdb is idempotent — safe to run every update.
|
||||
log(" 检查 tls_ref_data 引用数据表...")
|
||||
import_result = subprocess.run(
|
||||
[python_exe, 'manage.py', 'import_tlsdb'],
|
||||
capture_output=True, text=True, cwd=str(PROJECT_DIR)
|
||||
)
|
||||
if import_result.returncode == 0:
|
||||
log(" 引用数据已就绪 ✓")
|
||||
else:
|
||||
# import_tlsdb may warn about existing data; that's fine
|
||||
if 'already exists' in import_result.stderr.lower() or '0 row' in import_result.stdout.lower():
|
||||
log(" 引用数据已存在 ✓")
|
||||
else:
|
||||
log(f" [INFO] import_tlsdb 输出: {import_result.stdout.strip()[:200]}")
|
||||
|
||||
# Step 10: Update VERSION and finalize
|
||||
step(10, total_steps, "完成更新")
|
||||
|
||||
to_version = manifest.get('to_version', '')
|
||||
if to_version:
|
||||
VERSION_FILE.write_text(to_version + '\n', encoding='utf-8')
|
||||
log(f"VERSION → {to_version}")
|
||||
|
||||
# Print runtime warning if needed
|
||||
if manifest.get('runtime_changed'):
|
||||
print("\n" + "!" * 50)
|
||||
print(" ⚠️ 依赖已变更！")
|
||||
print(" 请手动将开发机的 runtime/ 目录同步到离线机")
|
||||
print(" (使用 U 盘复制 %APPDATA%/TianXuan/runtime/ 或直接覆盖)")
|
||||
print("!" * 50)
|
||||
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f"✅ 更新完成！")
|
||||
print(f" {from_ver} → {to_version}")
|
||||
print(f" {extracted_count} 个文件已更新")
|
||||
print(f" {len(manifest.get('deleted', []))} 个文件已删除")
|
||||
print(f"\n 请关闭当前窗口，重新双击 run.bat 启动新版本")
|
||||
print(f"{'=' * 50}")
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("用法:")
|
||||
print(" python scripts/apply_update.py <update.zip> — 应用更新")
|
||||
print(" python scripts/apply_update.py --rollback — 回滚到最近备份")
|
||||
return 1
|
||||
|
||||
if sys.argv[1] == '--rollback':
|
||||
return do_rollback()
|
||||
|
||||
zip_path = Path(sys.argv[1])
|
||||
if not zip_path.exists():
|
||||
# Try relative to project dir
|
||||
zip_path = PROJECT_DIR / sys.argv[1]
|
||||
if not zip_path.exists():
|
||||
print(f"[ERROR] 找不到更新包: {sys.argv[1]}")
|
||||
return 1
|
||||
|
||||
return apply_update(zip_path.resolve())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
Build incremental update package for 天璇 (TianXuan).
|
||||
|
||||
Usage:
|
||||
python scripts/build_update.py
|
||||
|
||||
Detects changes since the last git tag, creates a ZIP package containing
|
||||
only modified/new files, ready to be applied on the offline machine.
|
||||
|
||||
Output: tianxuan_update_<from_version>-<to_version>.zip
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path for sibling imports
|
||||
PROJECT_DIR = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT_DIR))
|
||||
|
||||
from scripts.version_utils import (
|
||||
get_current_version,
|
||||
set_current_version,
|
||||
get_latest_tag,
|
||||
get_changed_files,
|
||||
bump_version,
|
||||
is_working_tree_clean,
|
||||
)
|
||||
|
||||
# Files/dirs to exclude from update packages (in addition to .gitignore)
|
||||
EXCLUDE_PATTERNS = {
|
||||
'.git', '.venv', '__pycache__', '*.pyc', '.pytest_cache',
|
||||
'.omo', '.codegraph', '.playwright-mcp',
|
||||
}
|
||||
|
||||
GITIGNORE_PATTERNS = {
|
||||
'runtime/', '.venv', '*.sqlite3*', 'logs/', '*.log',
|
||||
}
|
||||
|
||||
|
||||
def should_exclude(path: str) -> bool:
|
||||
"""Check if a file path should be excluded from the update package."""
|
||||
path_lower = path.lower()
|
||||
for pat in EXCLUDE_PATTERNS:
|
||||
if pat.endswith('/'):
|
||||
if path_lower.startswith(pat) or f'/{pat.lower()}' in path_lower:
|
||||
return True
|
||||
else:
|
||||
if path_lower == pat.lower():
|
||||
return True
|
||||
# Exclude .gitignore patterns that are not in git
|
||||
for pat in GITIGNORE_PATTERNS:
|
||||
if pat.endswith('/'):
|
||||
if path_lower.startswith(pat) or f'/{pat.lower()}' in path_lower:
|
||||
return True
|
||||
elif path_lower.endswith(pat.lstrip('*')):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def compute_sha256(filepath: Path) -> str:
|
||||
"""Compute SHA256 hash of a file."""
|
||||
h = hashlib.sha256()
|
||||
with open(filepath, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(65536), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def has_migration_changes(added_modified: list[str]) -> bool:
|
||||
"""Check if any migration files changed."""
|
||||
for path in added_modified:
|
||||
if '/migrations/' in path and path.endswith('.py'):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def has_tlsdb_changes(added_modified: list[str]) -> bool:
|
||||
"""Check if TlsDB.csv changed."""
|
||||
for path in added_modified:
|
||||
if path == 'TlsDB.csv':
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def has_dependency_changes(added_modified: list[str]) -> bool:
|
||||
"""Check if pyproject.toml or requirements.txt changed."""
|
||||
for path in added_modified:
|
||||
if path in ('pyproject.toml', 'requirements.txt', 'uv.lock'):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def detect_runtime_changes(project_dir: Path) -> bool:
|
||||
"""Detect if portable Python runtime packages have changed.
|
||||
|
||||
Uses pip list output to snapshot installed packages, comparing
|
||||
against the previous build's snapshot. This avoids depending on
|
||||
git (which can't track runtime/ as it's gitignored).
|
||||
|
||||
Returns True if any package changed.
|
||||
"""
|
||||
python_exe = str(project_dir / 'runtime' / 'python' / 'python.exe')
|
||||
if not os.path.exists(python_exe):
|
||||
return False # No runtime installed — skip
|
||||
|
||||
# Get current pip list
|
||||
result = subprocess.run(
|
||||
[python_exe, '-m', 'pip', 'list', '--format=json'],
|
||||
capture_output=True, text=True, cwd=str(project_dir), timeout=60
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return False # Can't determine — assume no change
|
||||
|
||||
try:
|
||||
current = json.loads(result.stdout)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
|
||||
# Build a dict of name -> version
|
||||
current_packages = {pkg['name'].lower(): pkg['version'] for pkg in current}
|
||||
|
||||
# Compare with previous snapshot
|
||||
snapshot_dir = project_dir / '.update_cache'
|
||||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
snapshot_file = snapshot_dir / 'runtime_snapshot.json'
|
||||
|
||||
if not snapshot_file.exists():
|
||||
# First build — save and report no change
|
||||
json.dump(current_packages, open(snapshot_file, 'w'), indent=2)
|
||||
return False
|
||||
|
||||
try:
|
||||
previous = json.loads(open(snapshot_file).read())
|
||||
except (json.JSONDecodeError, OSError):
|
||||
json.dump(current_packages, open(snapshot_file, 'w'), indent=2)
|
||||
return False
|
||||
|
||||
# Detect changes
|
||||
changed = {}
|
||||
for name, version in current_packages.items():
|
||||
prev_ver = previous.get(name)
|
||||
if prev_ver is None:
|
||||
changed[name] = f'+{version}' # New package
|
||||
elif prev_ver != version:
|
||||
changed[name] = f'{prev_ver}→{version}' # Version changed
|
||||
|
||||
for name in previous:
|
||||
if name not in current_packages:
|
||||
changed[name] = '-removed' # Package removed
|
||||
|
||||
# Update snapshot
|
||||
json.dump(current_packages, open(snapshot_file, 'w'), indent=2)
|
||||
|
||||
if changed:
|
||||
print(f"\n[RUNTIME] {len(changed)} 个依赖包变更:")
|
||||
for name, diff in sorted(changed.items()):
|
||||
print(f" {name}: {diff}")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def build_update_package():
|
||||
print("=" * 50)
|
||||
print(" 天璇 增量更新包构建工具")
|
||||
print("=" * 50)
|
||||
|
||||
# Step 1: Check working tree
|
||||
if not is_working_tree_clean():
|
||||
print("\n[ERROR] 工作树有未提交的变更！")
|
||||
print(" 请先提交或贮藏 (git stash) 后再构建")
|
||||
subprocess.run(['git', 'status', '--short'], cwd=PROJECT_DIR)
|
||||
sys.exit(1)
|
||||
|
||||
# Step 2: Determine versions
|
||||
current_version = get_current_version()
|
||||
latest_tag = get_latest_tag()
|
||||
|
||||
print(f"\n当前 VERSION: {current_version}")
|
||||
print(f"最新 git tag: {latest_tag or '(无)'}")
|
||||
|
||||
# If no tags exist, use VERSION content
|
||||
from_version = latest_tag if latest_tag else current_version
|
||||
to_version = bump_version(current_version)
|
||||
|
||||
# Step 3: Get changed files
|
||||
if latest_tag:
|
||||
added_modified, deleted = get_changed_files(latest_tag)
|
||||
else:
|
||||
# First build — include everything except gitignored/excluded
|
||||
print("\n[INFO] 首次构建，包含所有被 git 跟踪的文件")
|
||||
result = subprocess.run(
|
||||
['git', 'ls-files'],
|
||||
capture_output=True, text=True, cwd=PROJECT_DIR, timeout=30
|
||||
)
|
||||
added_modified = [f.strip() for f in result.stdout.strip().split('\n') if f.strip()]
|
||||
deleted = []
|
||||
|
||||
# Filter excluded
|
||||
added_modified = [f for f in added_modified if not should_exclude(f)]
|
||||
deleted = [f for f in deleted if not should_exclude(f)]
|
||||
|
||||
if not added_modified and not deleted:
|
||||
print("\n[NO_CHANGES] 自上次构建以来没有变更文件")
|
||||
print(f" VERSION: {current_version}")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"\n变更文件: {len(added_modified)} 个修改/新增, {len(deleted)} 个删除")
|
||||
|
||||
# Step 4: Build manifest
|
||||
files_manifest = {}
|
||||
for rel_path in added_modified:
|
||||
abs_path = PROJECT_DIR / rel_path
|
||||
if abs_path.exists() and abs_path.is_file():
|
||||
files_manifest[rel_path] = {
|
||||
'sha256': compute_sha256(abs_path),
|
||||
'size': abs_path.stat().st_size,
|
||||
}
|
||||
|
||||
# Runtime change detection (runtime/ is gitignored, so git can't track it)
|
||||
runtime_changed = detect_runtime_changes(PROJECT_DIR)
|
||||
if not runtime_changed:
|
||||
# Also check if pyproject.toml was modified (even if pip list didn't change yet)
|
||||
runtime_changed = has_dependency_changes(added_modified)
|
||||
|
||||
manifest = {
|
||||
'from_version': from_version,
|
||||
'to_version': to_version,
|
||||
'built_at': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'files': files_manifest,
|
||||
'deleted': deleted,
|
||||
'run_migrate': has_migration_changes(added_modified),
|
||||
'run_import_tlsdb': has_tlsdb_changes(added_modified),
|
||||
'runtime_changed': runtime_changed,
|
||||
}
|
||||
|
||||
# Step 5: Build ZIP
|
||||
zip_name = f'tianxuan_update_{from_version}-{to_version}.zip'
|
||||
zip_path = PROJECT_DIR / zip_name
|
||||
|
||||
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
# Write manifest
|
||||
zf.writestr('version.json', json.dumps(manifest, indent=2, ensure_ascii=False))
|
||||
|
||||
# Write changed files
|
||||
for rel_path in added_modified:
|
||||
abs_path = PROJECT_DIR / rel_path
|
||||
if abs_path.exists() and abs_path.is_file():
|
||||
arcname = f'files/{rel_path}'
|
||||
zf.write(abs_path, arcname)
|
||||
|
||||
# Write runtime_changed marker if dependencies changed
|
||||
if manifest['runtime_changed']:
|
||||
runtime_msg = (
|
||||
"[RUNTIME] 依赖已变更！\n"
|
||||
f"pyproject.toml 或 requirements.txt 已修改。\n"
|
||||
f"请手动将 runtime/ 目录同步到离线机。\n"
|
||||
)
|
||||
zf.writestr('runtime_changed.txt', runtime_msg)
|
||||
|
||||
# Step 6: Update VERSION
|
||||
set_current_version(to_version)
|
||||
|
||||
# Step 7: Print summary
|
||||
total_size_mb = zip_path.stat().st_size / (1024 * 1024)
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f"✅ 增量更新包构建完成！")
|
||||
print(f" 包: {zip_name}")
|
||||
print(f" 大小: {total_size_mb:.2f} MB")
|
||||
print(f" 从: {from_version}")
|
||||
print(f" 到: {to_version}")
|
||||
print(f" 文件数: {len(files_manifest)}")
|
||||
print(f" 删除: {len(deleted)}")
|
||||
if manifest['run_migrate']:
|
||||
print(f" ⚠️ 包含数据库迁移 (schema 变更)")
|
||||
if manifest['run_import_tlsdb']:
|
||||
print(f" ⚠️ TlsDB.csv 已变更，需重新导入引用数据")
|
||||
if manifest['runtime_changed']:
|
||||
# Write a visible marker file
|
||||
runtime_marker = PROJECT_DIR / 'RUNTIME_CHANGED.txt'
|
||||
runtime_marker.write_text(
|
||||
"依赖已变更！请手动将 runtime/ 目录同步到离线机。\n"
|
||||
f"请在开发机上压缩 runtime/ 目录，然后用 U 盘复制到离线机。\n"
|
||||
f"也可直接复制整个 runtime/ 目录覆盖离线机的对应目录。\n",
|
||||
encoding='utf-8'
|
||||
)
|
||||
print(f" ⚠️ 依赖已变更！请手动将 runtime/ 同步到离线机")
|
||||
else:
|
||||
# Clean up stale marker if no longer changed
|
||||
stale_marker = PROJECT_DIR / 'RUNTIME_CHANGED.txt'
|
||||
if stale_marker.exists():
|
||||
stale_marker.unlink()
|
||||
print(f"\n 复制 {zip_name} 到离线机，然后双击 update.bat")
|
||||
print(f"{'=' * 50}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
build_update_package()
|
||||
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
Config merge utility for incremental updates.
|
||||
|
||||
Preserves user edits in config.yaml while adding new configuration keys
|
||||
introduced by source code updates.
|
||||
|
||||
Usage:
|
||||
python scripts/merge_config.py --old <user_config.yaml> --new <source_config.yaml> --output <merged.yaml>
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
# Fallback when running outside Django context
|
||||
try:
|
||||
import ruamel.yaml as yaml
|
||||
except ImportError:
|
||||
print("[ERROR] PyYAML is required. Install with: pip install pyyaml", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def deep_merge(base: dict, overlay: dict) -> dict:
|
||||
"""Merge overlay into base, preserving base values for existing keys.
|
||||
|
||||
Rules:
|
||||
- Keys in base that are NOT in overlay: preserved unchanged
|
||||
- Keys in overlay that are NOT in base: added from overlay
|
||||
- Keys in BOTH: base value wins (user edits preserved)
|
||||
- Nested dicts: recursively merged with same rules
|
||||
- Lists/scalars: base wins entirely (no element-level merge)
|
||||
"""
|
||||
result = dict(base)
|
||||
for k, v in overlay.items():
|
||||
if k not in result:
|
||||
# New key from update → take new value
|
||||
result[k] = v
|
||||
elif isinstance(v, dict) and isinstance(result.get(k), dict):
|
||||
# Both are dicts → recursive merge
|
||||
result[k] = deep_merge(result[k], v)
|
||||
# else: key exists in both → base wins (user edit preserved)
|
||||
return result
|
||||
|
||||
|
||||
def merge_config(old_path: Path, new_path: Path, output_path: Path) -> int:
|
||||
"""Merge old (user) config with new (source) config.
|
||||
|
||||
Returns:
|
||||
0 on success, 1 on error
|
||||
"""
|
||||
if not old_path.exists():
|
||||
print(f"[CONFIG] No user config at {old_path}, using source config directly")
|
||||
import shutil
|
||||
shutil.copy2(str(new_path), str(output_path))
|
||||
return 0
|
||||
|
||||
if not new_path.exists():
|
||||
print(f"[ERROR] Source config {new_path} not found", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
with open(old_path, 'r', encoding='utf-8') as f:
|
||||
old = yaml.safe_load(f) or {}
|
||||
except Exception as e:
|
||||
print(f"[CONFIG_MERGE_FAILED] Cannot parse user config: {e}", file=sys.stderr)
|
||||
print("[CONFIG] Falling back to source config")
|
||||
import shutil
|
||||
shutil.copy2(str(new_path), str(output_path))
|
||||
return 0
|
||||
|
||||
try:
|
||||
with open(new_path, 'r', encoding='utf-8') as f:
|
||||
new = yaml.safe_load(f) or {}
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Cannot parse source config: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
merged = deep_merge(old, new)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(merged, f, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
# Count preserved vs new keys
|
||||
preserved = sum(1 for k in merged if k in old)
|
||||
added = sum(1 for k in merged if k not in old)
|
||||
print(f"[CONFIG] Merged: {preserved} keys preserved, {added} keys added from update")
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Merge user config with source config for incremental updates')
|
||||
parser.add_argument('--old', required=True, help='Path to user config (existing, may have edits)')
|
||||
parser.add_argument('--new', required=True, help='Path to source config (from update package)')
|
||||
parser.add_argument('--output', required=True, help='Path to write merged config')
|
||||
args = parser.parse_args()
|
||||
|
||||
sys.exit(merge_config(
|
||||
Path(args.old), Path(args.new), Path(args.output)
|
||||
))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,58 @@
|
||||
r"""
|
||||
Migrate existing db.sqlite3 from project directory to %APPDATA%/TianXuan/.
|
||||
|
||||
This script is called during the first startup after the DB path change.
|
||||
It preserves the old database and only copies if the new location is empty.
|
||||
|
||||
Usage:
|
||||
runtime\python\python.exe scripts\migrate_db_to_appdata.py
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def migrate():
|
||||
project_dir = Path(__file__).resolve().parent.parent
|
||||
|
||||
old_db = project_dir / 'db.sqlite3'
|
||||
old_wal = project_dir / 'db.sqlite3-wal'
|
||||
old_shm = project_dir / 'db.sqlite3-shm'
|
||||
|
||||
appdata = Path(os.environ.get('APPDATA', str(Path.home() / '.tianxuan'))) / 'TianXuan'
|
||||
appdata.mkdir(parents=True, exist_ok=True)
|
||||
new_db = appdata / 'db.sqlite3'
|
||||
|
||||
if new_db.exists():
|
||||
print(f"[SKIP] {new_db} already exists — migration already done or data present")
|
||||
return 0
|
||||
|
||||
if not old_db.exists():
|
||||
print(f"[SKIP] No {old_db} to migrate — clean install or already migrated")
|
||||
return 0
|
||||
|
||||
# Copy WAL and SHM files first (WAL must be flushed before main DB)
|
||||
for f in [old_wal, old_shm]:
|
||||
if f.exists():
|
||||
shutil.copy2(str(f), str(appdata / f.name))
|
||||
print(f" Copied {f.name}")
|
||||
|
||||
# Copy main database
|
||||
size_mb = old_db.stat().st_size / (1024 * 1024)
|
||||
print(f"Migrating {old_db.name} ({size_mb:.1f} MB) to {new_db}...")
|
||||
shutil.copy2(str(old_db), str(new_db))
|
||||
|
||||
# Verify
|
||||
if new_db.exists() and new_db.stat().st_size > 0:
|
||||
print(f"[OK] Migration complete: {new_db}")
|
||||
print(f"[INFO] Old database preserved at: {old_db}")
|
||||
print(f"[INFO] You can safely delete the old file after verification")
|
||||
return 0
|
||||
else:
|
||||
print(f"[ERROR] Migration failed: {new_db} not created or empty", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(migrate())
|
||||
@@ -0,0 +1,117 @@
|
||||
"""版本工具函数，在构建端（开发机）和应用端（离线机）共享
|
||||
|
||||
Provides version tracking and git change detection utilities.
|
||||
All functions work standalone without Django imports.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import re
|
||||
|
||||
PROJECT_DIR = Path(__file__).resolve().parent.parent
|
||||
VERSION_FILE = PROJECT_DIR / 'VERSION'
|
||||
|
||||
|
||||
def get_current_version() -> str:
|
||||
"""从 VERSION 文件读取当前版本"""
|
||||
if VERSION_FILE.exists():
|
||||
return VERSION_FILE.read_text(encoding='utf-8').strip()
|
||||
return 'v0.0.0'
|
||||
|
||||
|
||||
def set_current_version(version: str) -> None:
|
||||
"""写入 VERSION 文件"""
|
||||
VERSION_FILE.write_text(version.strip() + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def get_git_tags() -> list[str]:
|
||||
"""获取所有 git tag，按版本号降序排列"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['git', 'tag', '--sort=-version:refname'],
|
||||
capture_output=True, text=True, cwd=PROJECT_DIR, timeout=30
|
||||
)
|
||||
return [t.strip() for t in result.stdout.strip().split('\n') if t.strip()]
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return []
|
||||
|
||||
|
||||
def get_latest_tag() -> str | None:
|
||||
"""获取最新 git tag，如果没有则返回 None"""
|
||||
tags = get_git_tags()
|
||||
return tags[0] if tags else None
|
||||
|
||||
|
||||
def get_changed_files(from_tag: str, to_ref: str = 'HEAD') -> tuple[list[str], list[str]]:
|
||||
"""获取两个 git 引用之间变更的文件列表
|
||||
|
||||
使用 git diff --name-status 以确保能捕获删除的文件。
|
||||
|
||||
Args:
|
||||
from_tag: 起始版本（如 'v1.0'）
|
||||
to_ref: 终止引用（默认 HEAD）
|
||||
|
||||
Returns:
|
||||
(added_modified: list[str], deleted: list[str])
|
||||
文件路径相对于项目根目录
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['git', 'diff', '--name-status', f'{from_tag}..{to_ref}'],
|
||||
capture_output=True, text=True, cwd=PROJECT_DIR, timeout=30
|
||||
)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return [], []
|
||||
|
||||
added_modified: list[str] = []
|
||||
deleted: list[str] = []
|
||||
|
||||
for line in result.stdout.strip().split('\n'):
|
||||
if not line.strip():
|
||||
continue
|
||||
parts = line.split('\t')
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
status = parts[0]
|
||||
path = parts[1]
|
||||
|
||||
# Handle renames: R100 file1\tfile2 → track the new name
|
||||
if status.startswith('R'):
|
||||
if len(parts) >= 3:
|
||||
added_modified.append(parts[2]) # new name
|
||||
deleted.append(parts[1]) # old name
|
||||
continue
|
||||
|
||||
if status == 'D':
|
||||
deleted.append(path)
|
||||
elif status in ('A', 'M', 'C'):
|
||||
added_modified.append(path)
|
||||
|
||||
return added_modified, deleted
|
||||
|
||||
|
||||
def is_working_tree_clean() -> bool:
|
||||
"""检查 git 工作树是否干净（无未提交变更）"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['git', 'diff', '--stat'],
|
||||
capture_output=True, text=True, cwd=PROJECT_DIR, timeout=15
|
||||
)
|
||||
return result.stdout.strip() == ''
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return False
|
||||
|
||||
|
||||
def bump_version(current: str) -> str:
|
||||
"""递增版本号
|
||||
|
||||
v0.0.0 → v1.0 (首次稳定版)
|
||||
v1.0 → v1.1
|
||||
v1.9 → v1.10
|
||||
"""
|
||||
match = re.match(r'v(\d+)\.(\d+)', current)
|
||||
if not match:
|
||||
return 'v1.0'
|
||||
major, minor = int(match.group(1)), int(match.group(2))
|
||||
if major == 0:
|
||||
return 'v1.0'
|
||||
return f'v{major}.{minor + 1}'
|
||||
Reference in New Issue
Block a user