PM-pinou
3617b7aabc
Build: fix incremental package (update_info.json SHA256, __delete__, test exclude, files/ prefix), fix bump_version for 3-segment, PYTHONUTF8 in update.bat Frontend: upload→manual redirect, LLM→cluster_overview redirect, filter UI before clustering, 地球→态势 rename, fix globe drag direction+inertia Pipeline: LLM→auto clustering+PCA, column union handling (keep extra cols, no dropping), FFT spectral analysis tool Quality: fix 24 bare except:pass patterns, enrich LLM context with TlsDB metadata, auto-save workflows with pinning, TlsDB-based column types
361 lines
13 KiB
Python
361 lines
13 KiB
Python
#!/usr/bin/env python
|
||
"""
|
||
Build incremental update package for 天璇 (TianXuan).
|
||
|
||
Usage:
|
||
python scripts/build_update.py
|
||
|
||
Detects changes since the last git tag, creates a ZIP package containing
|
||
only modified/new files, ready to be applied on the offline machine.
|
||
|
||
Output: tianxuan_update_<from_version>-<to_version>.zip
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import zipfile
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
# Add project root to path for sibling imports
|
||
PROJECT_DIR = Path(__file__).resolve().parent.parent
|
||
sys.path.insert(0, str(PROJECT_DIR))
|
||
|
||
from scripts.version_utils import (
|
||
get_current_version,
|
||
set_current_version,
|
||
get_latest_tag,
|
||
get_changed_files,
|
||
bump_version,
|
||
is_working_tree_clean,
|
||
)
|
||
|
||
# Files/dirs to exclude from update packages (in addition to .gitignore)
|
||
EXCLUDE_PATTERNS = {
|
||
'.git', '.venv', '__pycache__', '*.pyc', '.pytest_cache',
|
||
'.omo', '.codegraph', '.playwright-mcp',
|
||
'scripts/gen_test_data.py', 'scripts/gen_complex_test.py',
|
||
}
|
||
|
||
GITIGNORE_PATTERNS = {
|
||
'runtime/', '.venv', '*.sqlite3*', 'logs/', '*.log',
|
||
}
|
||
|
||
|
||
def should_exclude(path: str) -> bool:
|
||
"""Check if a file path should be excluded from the update package."""
|
||
path_lower = path.lower()
|
||
for pat in EXCLUDE_PATTERNS:
|
||
if pat.endswith('/'):
|
||
if path_lower.startswith(pat) or f'/{pat.lower()}' in path_lower:
|
||
return True
|
||
else:
|
||
if path_lower == pat.lower():
|
||
return True
|
||
# Exclude .gitignore patterns that are not in git
|
||
for pat in GITIGNORE_PATTERNS:
|
||
if pat.endswith('/'):
|
||
if path_lower.startswith(pat) or f'/{pat.lower()}' in path_lower:
|
||
return True
|
||
elif path_lower.endswith(pat.lstrip('*')):
|
||
return True
|
||
return False
|
||
|
||
|
||
def compute_sha256(filepath: Path) -> str:
|
||
"""Compute SHA256 hash of a file."""
|
||
h = hashlib.sha256()
|
||
with open(filepath, 'rb') as f:
|
||
for chunk in iter(lambda: f.read(65536), b''):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
def has_migration_changes(added_modified: list[str]) -> bool:
|
||
"""Check if any migration files changed."""
|
||
for path in added_modified:
|
||
if '/migrations/' in path and path.endswith('.py'):
|
||
return True
|
||
return False
|
||
|
||
|
||
def has_tlsdb_changes(added_modified: list[str]) -> bool:
|
||
"""Check if TlsDB.csv changed."""
|
||
for path in added_modified:
|
||
if path == 'TlsDB.csv':
|
||
return True
|
||
return False
|
||
|
||
|
||
def has_dependency_changes(added_modified: list[str]) -> bool:
|
||
"""Check if pyproject.toml or requirements.txt changed."""
|
||
for path in added_modified:
|
||
if path in ('pyproject.toml', 'requirements.txt', 'uv.lock'):
|
||
return True
|
||
return False
|
||
|
||
|
||
def detect_runtime_changes(project_dir: Path) -> bool:
|
||
"""Detect if portable Python runtime packages have changed.
|
||
|
||
Uses pip list output to snapshot installed packages, comparing
|
||
against the previous build's snapshot. This avoids depending on
|
||
git (which can't track runtime/ as it's gitignored).
|
||
|
||
Returns True if any package changed.
|
||
"""
|
||
python_exe = str(project_dir / 'runtime' / 'python' / 'python.exe')
|
||
if not os.path.exists(python_exe):
|
||
return False # No runtime installed — skip
|
||
|
||
# Get current pip list
|
||
result = subprocess.run(
|
||
[python_exe, '-m', 'pip', 'list', '--format=json'],
|
||
capture_output=True, text=True, cwd=str(project_dir), timeout=60
|
||
)
|
||
if result.returncode != 0:
|
||
return False # Can't determine — assume no change
|
||
|
||
try:
|
||
current = json.loads(result.stdout)
|
||
except json.JSONDecodeError:
|
||
return False
|
||
|
||
# Build a dict of name -> version
|
||
current_packages = {pkg['name'].lower(): pkg['version'] for pkg in current}
|
||
|
||
# Compare with previous snapshot
|
||
snapshot_dir = project_dir / '.update_cache'
|
||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||
snapshot_file = snapshot_dir / 'runtime_snapshot.json'
|
||
|
||
if not snapshot_file.exists():
|
||
# First build — save and report no change
|
||
json.dump(current_packages, open(snapshot_file, 'w'), indent=2)
|
||
return False
|
||
|
||
try:
|
||
previous = json.loads(open(snapshot_file).read())
|
||
except (json.JSONDecodeError, OSError):
|
||
json.dump(current_packages, open(snapshot_file, 'w'), indent=2)
|
||
return False
|
||
|
||
# Detect changes
|
||
changed = {}
|
||
for name, version in current_packages.items():
|
||
prev_ver = previous.get(name)
|
||
if prev_ver is None:
|
||
changed[name] = f'+{version}' # New package
|
||
elif prev_ver != version:
|
||
changed[name] = f'{prev_ver}→{version}' # Version changed
|
||
|
||
for name in previous:
|
||
if name not in current_packages:
|
||
changed[name] = '-removed' # Package removed
|
||
|
||
# Update snapshot
|
||
json.dump(current_packages, open(snapshot_file, 'w'), indent=2)
|
||
|
||
if changed:
|
||
print(f"\n[RUNTIME] {len(changed)} 个依赖包变更:")
|
||
for name, diff in sorted(changed.items()):
|
||
print(f" {name}: {diff}")
|
||
return True
|
||
|
||
return False
|
||
|
||
|
||
def build_update_package(auto_mode=False):
|
||
print("=" * 50)
|
||
print(" 天璇 增量更新包构建工具")
|
||
print("=" * 50)
|
||
|
||
# Step 1: Check working tree
|
||
if not is_working_tree_clean():
|
||
if auto_mode:
|
||
# Step A: Auto-commit any uncommitted changes
|
||
print("\n[AUTO] 检测到未提交的变更，自动提交中...")
|
||
subprocess.run(['git', 'add', '-A'], cwd=PROJECT_DIR, check=True)
|
||
status = subprocess.run(
|
||
['git', 'status', '--short'],
|
||
capture_output=True, text=True, cwd=PROJECT_DIR
|
||
)
|
||
file_count = len([l for l in status.stdout.strip().split('\n') if l.strip()])
|
||
version = get_current_version()
|
||
commit_msg = f"auto: update for v{version}"
|
||
subprocess.run(
|
||
['git', 'commit', '-m', commit_msg],
|
||
cwd=PROJECT_DIR, check=True
|
||
)
|
||
print(f" ✅ 已提交 {file_count} 个文件: {commit_msg}")
|
||
else:
|
||
print("\n[ERROR] 工作树有未提交的变更！")
|
||
print(" 请先提交或贮藏 (git stash) 后再构建")
|
||
subprocess.run(['git', 'status', '--short'], cwd=PROJECT_DIR)
|
||
sys.exit(1)
|
||
|
||
# Step 2: Determine versions
|
||
current_version = get_current_version()
|
||
latest_tag = get_latest_tag()
|
||
|
||
print(f"\n当前 VERSION: {current_version}")
|
||
print(f"最新 git tag: {latest_tag or '(无)'}")
|
||
|
||
# If no tags exist, use VERSION content
|
||
from_version = latest_tag if latest_tag else current_version
|
||
to_version = bump_version(current_version)
|
||
|
||
# Step 2.5: Auto bump + commit + tag (auto mode only)
|
||
if auto_mode:
|
||
# Bump VERSION file
|
||
set_current_version(to_version)
|
||
|
||
# Commit the VERSION bump
|
||
subprocess.run(['git', 'add', 'VERSION'], cwd=PROJECT_DIR, check=True)
|
||
subprocess.run(
|
||
['git', 'commit', '-m', f'chore: bump to {to_version}'],
|
||
cwd=PROJECT_DIR, check=True
|
||
)
|
||
print(f" ✅ 已提交版本升级: {current_version} → {to_version}")
|
||
|
||
# Create tag pointing at the bumped commit
|
||
result = subprocess.run(
|
||
['git', 'tag', '-l', to_version],
|
||
capture_output=True, text=True, cwd=PROJECT_DIR
|
||
)
|
||
if to_version not in result.stdout:
|
||
subprocess.run(['git', 'tag', to_version], cwd=PROJECT_DIR, check=True)
|
||
print(f" ✅ 已创建 tag: {to_version}")
|
||
else:
|
||
print(f" ℹ️ Tag {to_version} 已存在，跳过")
|
||
|
||
# Step 3: Get changed files
|
||
if latest_tag:
|
||
added_modified, deleted = get_changed_files(latest_tag)
|
||
else:
|
||
# First build — include everything except gitignored/excluded
|
||
print("\n[INFO] 首次构建，包含所有被 git 跟踪的文件")
|
||
result = subprocess.run(
|
||
['git', 'ls-files'],
|
||
capture_output=True, text=True, cwd=PROJECT_DIR, timeout=30
|
||
)
|
||
added_modified = [f.strip() for f in result.stdout.strip().split('\n') if f.strip()]
|
||
deleted = []
|
||
|
||
# Filter excluded
|
||
added_modified = [f for f in added_modified if not should_exclude(f)]
|
||
deleted = [f for f in deleted if not should_exclude(f)]
|
||
|
||
if not added_modified and not deleted:
|
||
print("\n[NO_CHANGES] 自上次构建以来没有变更文件")
|
||
print(f" VERSION: {current_version}")
|
||
sys.exit(0)
|
||
|
||
print(f"\n变更文件: {len(added_modified)} 个修改/新增, {len(deleted)} 个删除")
|
||
|
||
# Step 4: Build manifest
|
||
files_manifest = {}
|
||
for rel_path in added_modified:
|
||
abs_path = PROJECT_DIR / rel_path
|
||
if abs_path.exists() and abs_path.is_file():
|
||
files_manifest[rel_path] = {
|
||
'sha256': compute_sha256(abs_path),
|
||
'size': abs_path.stat().st_size,
|
||
}
|
||
|
||
# Runtime change detection (runtime/ is gitignored, so git can't track it)
|
||
runtime_changed = detect_runtime_changes(PROJECT_DIR)
|
||
if not runtime_changed:
|
||
# Also check if pyproject.toml was modified (even if pip list didn't change yet)
|
||
runtime_changed = has_dependency_changes(added_modified)
|
||
|
||
# Manual delete list — files to explicitly remove (not tracked by git diff)
|
||
MANUAL_DELETE_LIST = []
|
||
|
||
manifest = {
|
||
'from_version': from_version,
|
||
'to_version': to_version,
|
||
'built_at': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||
'files': files_manifest,
|
||
'deleted': deleted,
|
||
'__delete__': deleted + MANUAL_DELETE_LIST,
|
||
'run_migrate': has_migration_changes(added_modified),
|
||
'run_import_tlsdb': has_tlsdb_changes(added_modified),
|
||
'runtime_changed': runtime_changed,
|
||
}
|
||
|
||
# Compute SHA256 of the manifest (without the sha256 field itself)
|
||
manifest['sha256'] = hashlib.sha256(
|
||
json.dumps(manifest, indent=2, ensure_ascii=False, sort_keys=True).encode('utf-8')
|
||
).hexdigest()
|
||
|
||
# Step 5: Build ZIP
|
||
zip_name = f'tianxuan_update_{from_version}-{to_version}.zip'
|
||
zip_path = PROJECT_DIR / zip_name
|
||
|
||
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||
# Write manifest
|
||
zf.writestr('files/update_info.json', json.dumps(manifest, indent=2, ensure_ascii=False))
|
||
|
||
# Write changed files
|
||
for rel_path in added_modified:
|
||
abs_path = PROJECT_DIR / rel_path
|
||
if abs_path.exists() and abs_path.is_file():
|
||
arcname = f'files/{rel_path}'
|
||
zf.write(abs_path, arcname)
|
||
|
||
# Write runtime_changed marker if dependencies changed
|
||
if manifest['runtime_changed']:
|
||
runtime_msg = (
|
||
"[RUNTIME] 依赖已变更！\n"
|
||
f"pyproject.toml 或 requirements.txt 已修改。\n"
|
||
f"请手动将 runtime/ 目录同步到离线机。\n"
|
||
)
|
||
zf.writestr('runtime_changed.txt', runtime_msg)
|
||
|
||
# Step 6: Update VERSION (skip in auto mode — already done before tag)
|
||
if not auto_mode:
|
||
set_current_version(to_version)
|
||
|
||
# Step 7: Print summary
|
||
total_size_mb = zip_path.stat().st_size / (1024 * 1024)
|
||
print(f"\n{'=' * 50}")
|
||
print(f"✅ 增量更新包构建完成！")
|
||
print(f" 包: {zip_name}")
|
||
print(f" 大小: {total_size_mb:.2f} MB")
|
||
print(f" 从: {from_version}")
|
||
print(f" 到: {to_version}")
|
||
print(f" 文件数: {len(files_manifest)}")
|
||
print(f" 删除: {len(deleted)}")
|
||
if manifest['run_migrate']:
|
||
print(f" ⚠️ 包含数据库迁移 (schema 变更)")
|
||
if manifest['run_import_tlsdb']:
|
||
print(f" ⚠️ TlsDB.csv 已变更，需重新导入引用数据")
|
||
if manifest['runtime_changed']:
|
||
# Write a visible marker file
|
||
runtime_marker = PROJECT_DIR / 'RUNTIME_CHANGED.txt'
|
||
runtime_marker.write_text(
|
||
"依赖已变更！请手动将 runtime/ 目录同步到离线机。\n"
|
||
f"请在开发机上压缩 runtime/ 目录，然后用 U 盘复制到离线机。\n"
|
||
f"也可直接复制整个 runtime/ 目录覆盖离线机的对应目录。\n",
|
||
encoding='utf-8'
|
||
)
|
||
print(f" ⚠️ 依赖已变更！请手动将 runtime/ 同步到离线机")
|
||
else:
|
||
# Clean up stale marker if no longer changed
|
||
stale_marker = PROJECT_DIR / 'RUNTIME_CHANGED.txt'
|
||
if stale_marker.exists():
|
||
stale_marker.unlink()
|
||
print(f"\n 复制 {zip_name} 到离线机，然后双击 update.bat")
|
||
print(f"{'=' * 50}")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
parser = argparse.ArgumentParser(description='Build incremental update package for 天璇')
|
||
parser.add_argument('--auto', '-a', action='store_true', help='One-click mode: auto-commit + auto-tag + build')
|
||
args = parser.parse_args()
|
||
build_update_package(auto_mode=args.auto)
|