feat: harden dedupe runtime and NAS deployment

.gitignore

@@ -3,3 +3,4 @@
 __pycache__/
 .pytest_cache/
 .venv/
+am_music_deduper.egg-info/

Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.12-slim
-RUN apt-get update && apt-get install -y --no-install-recommends libchromaprint-tools && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y --no-install-recommends libchromaprint-tools libchromaprint1 && rm -rf /var/lib/apt/lists/*
 WORKDIR /app
 COPY pyproject.toml README.md ./
 COPY am_deduper ./am_deduper

README.md

@@ -2,38 +2,94 @@
 
 轻量级家庭 NAS 音乐库去重服务，面向 `AllMusic/{username}/` 场景。
 
-## 设计目标
+## 处理模型
 
-- 本地 SQLite 元数据库
+```text
-- L1 极速候选：文件大小 + 头/尾 64 KiB hash
+AllMusic
-- Exact Duplicate 最终使用完整 SHA-256 确认
+  ↓
-- L2 使用 Chromaprint 做音频相似度候选
+文件稳定检测
-- duration 预过滤，降低 CPU/I/O
+  ↓
-- 同一版本才允许自动择优
+L1：size + head/tail 64 KiB
-- loser 进入 `.dedupe/trash/`，默认保留 30 天
+  ↓
-- 不使用软链接；物理去重使用 hard link
+完整 SHA-256
-- inotify 增量处理 + 低优先级全量 reconcile
+  ├─ 相同 → Exact Duplicate → hard link
-- 任何不确定或异常默认保留原文件
+  └─ 不同 → duration ±3s → Chromaprint
+                       ├─ 高相似且版本一致 → hard link
+                       ├─ 中等相似/版本冲突 → review
+                       └─ 低相似 → 保留
+```
 
-## 当前阶段
+### 关键安全规则
 
-第一阶段先完成安全的扫描、SQLite 索引、L1/Exact 去重、trash/quarantine 和测试。
+- SHA-256 是 Exact Duplicate 的最终依据；L1 只负责缩小候选范围。
-L2 Chromaprint 比对与生产 NAS 监听在后续阶段接入。
+- 不根据文件名、歌手、歌名、专辑或来源单独判重。
+- 确认相同版本后，优先使用 hard link 保留每个用户原来的逻辑路径。
+- 用户删除自己的路径不会影响其他 hard link；最后一个 link 删除后数据才释放。
+- 不跨文件系统创建 hard link。
+- 正在写入的文件必须先通过稳定性检查。
+- L2 对版本标记冲突的文件进入 review，不自动合并。
+- `--dry-run` 只报告/更新索引，不修改音乐文件。
+- SQLite 是加速索引，文件系统是最终事实；数据库丢失后可以重新扫描。
 
-## 安全原则
+## CLI
-
-1. 不修改正在写入的文件。
-2. L1 命中必须再做完整 SHA-256 才能判定 exact duplicate。
-3. 不根据歌名/歌手/专辑单独判重。
-4. 不同格式或音频规格不会因为 metadata 相似而直接合并。
-5. 自动处理前保留可恢复路径；失败不删除原文件。
-6. 数据库只是索引，文件系统才是最终事实。
-
-## CLI 预期
 
 ```bash
-am-deduper scan /music/AllMusic
+# 建立/更新索引，不修改音乐文件
-am-deduper dedupe /music/AllMusic
+am-deduper scan /music/AllMusic --db /data/dedupe.db
-am-deduper reconcile /music/AllMusic
+
-am-deduper trash-prune /music/AllMusic --days 30
+# Exact + L2；首次执行建议 dry-run
+am-deduper dedupe /music/AllMusic --db /data/dedupe.db --dry-run
+
+# 确认后执行
+am-deduper dedupe /music/AllMusic --db /data/dedupe.db
+
+# 长驻监听：默认只自动执行安全的 Exact hard-link
+am-deduper watch /music/AllMusic --db /data/dedupe.db
+
+# 开启 L2 自动 acoustic 匹配（建议在 dry-run 验证阈值后再启用）
+am-deduper watch /music/AllMusic --db /data/dedupe.db --l2
 ```
+
+## Docker / Feiniu NAS 部署
+
+wjyhk 负责构建镜像，推送到 Gitea Container Registry；NAS 只负责 pull + run，不在 NAS 上重新 build。
+
+NAS 的 `AllMusic` 与 dedupe 数据建议分开：
+
+```text
+NAS
+├── AllMusic/
+│   ├── admin/
+│   ├── alice/
+│   └── bob/
+└── AppData/
+    └── am-deduper/
+        └── dedupe.db
+```
+
+注意：如果以后启用 trash，trash 目录必须位于 `AllMusic` 所在同一文件系统内，不能把需要原子 rename/hard-link 语义的文件直接移动到 `/data`。
+
+### NAS `.env`
+
+```dotenv
+AM_DEDUPER_IMAGE=<GITEA_REGISTRY>/<OWNER>/am-music-deduper:<TAG>
+ALLMUSIC_PATH=<NAS绝对路径>/AllMusic
+DEDUPE_DATA_PATH=<NAS绝对路径>/AppData/am-deduper
+```
+
+### NAS 部署
+
+```bash
+mkdir -p "$DEDUPE_DATA_PATH"
+docker login <GITEA_REGISTRY>
+docker compose pull
+docker compose up -d
+```
+
+查看日志：
+
+```bash
+docker logs -f am-deduper
+```
+
+首次上线建议先暂停 `watch`，执行一次完整 `scan` + `dedupe --dry-run`，确认候选数量与 review 项，再启用正式 dedupe；L2 自动处理单独验证后再打开 `--l2`。

am_deduper/acoustic.py

@@ -5,6 +5,8 @@ import subprocess
 from pathlib import Path
 
 MAX_SECONDS = 120
+MAX_BIT_ERROR = 2
+MAX_ALIGN_OFFSET = 120
 
 
 def chromaprint(path: Path) -> str | None:
@@ -13,26 +15,36 @@ def chromaprint(path: Path) -> str | None:
             ["fpcalc", "-length", str(MAX_SECONDS), "-json", str(path)],
             capture_output=True, text=True, timeout=180, check=True,
         )
-        return json.loads(p.stdout).get("fingerprint")
+        value = json.loads(p.stdout).get("fingerprint")
+        return value if isinstance(value, str) and value else None
     except (OSError, subprocess.SubprocessError, ValueError, json.JSONDecodeError):
         return None
 
 
+def _popcount(value: int) -> int:
+    return value.bit_count()
+
+
 def similarity(a: str, b: str) -> float:
-    """Conservative normalized positional similarity for equal-length prefixes."""
+    """Compare Chromaprint fingerprints using the AcoustID alignment method.
+
+    This is deliberately bounded to the local fingerprint comparison and does
+    not contact the AcoustID service.
+    """
     if not a or not b:
         return 0.0
-    n = min(len(a), len(b))
-    if not n:
-        return 0.0
-    # Chromaprint JSON fingerprints are comma-separated integer features.
     try:
-        aa, bb = [int(x) for x in a.split(",")], [int(x) for x in b.split(",")]
+        import chromaprint as cp
-    except ValueError:
+        aa = [int(x) for x in cp.decode_fingerprint(a.encode())[0]]
+        bb = [int(x) for x in cp.decode_fingerprint(b.encode())[0]]
+    except (ImportError, AttributeError, TypeError, ValueError):
         return 0.0
-    n = min(len(aa), len(bb))
+    if not aa or not bb:
-    if not n:
         return 0.0
-    # Feature-wise equality is intentionally conservative. A future backend
+
-    # can replace this with a library-specific fingerprint distance.
+    counts = [0] * (len(aa) + len(bb) + 1)
-    return sum(x == y for x, y in zip(aa[:n], bb[:n])) / n
+    for i, av in enumerate(aa):
+        for j in range(max(0, i - MAX_ALIGN_OFFSET), min(len(bb), i + MAX_ALIGN_OFFSET)):
+            if _popcount(av ^ bb[j]) <= MAX_BIT_ERROR:
+                counts[i - j + len(bb)] += 1
+    return max(counts) / min(len(aa), len(bb))

am_deduper/cli.py

@@ -1,18 +1,44 @@
 from __future__ import annotations
+
 import argparse
 from pathlib import Path
+
 from .exact import exact_dedupe
-from .scanner import scan
 from .l2 import acoustic_dedupe
+from .scanner import scan
 from .watcher import watch
 
-def main():
+
- p=argparse.ArgumentParser(prog='am-deduper'); sub=p.add_subparsers(dest='cmd',required=True)
+def main() -> None:
- for name in ('scan','dedupe','reconcile','watch'):
+    parser = argparse.ArgumentParser(prog="am-deduper")
-  sp=sub.add_parser(name); sp.add_argument('root',type=Path); sp.add_argument('--db',type=Path,default=None)
+    sub = parser.add_subparsers(dest="cmd", required=True)
- a=p.parse_args(); db=a.db or a.root/'.dedupe'/'dedupe.db'
+
- if a.cmd=='watch': watch(a.root,db); return
+    def add_common(sp):
- print(f'indexed={scan(a.root,db)} db={db}')
+        sp.add_argument("root", type=Path)
- if a.cmd in ('dedupe','reconcile'):
+        sp.add_argument("--db", type=Path, default=None)
-  ex=exact_dedupe(db,a.root); print(f'exact_hardlinked={len(ex)}')
+        sp.add_argument("--dry-run", action="store_true", help="report actions without changing files")
-  l2=acoustic_dedupe(db,a.root); print(f'l2={l2}')
+
+    add_common(sub.add_parser("scan"))
+    add_common(sub.add_parser("dedupe"))
+    add_common(sub.add_parser("reconcile"))
+    watch_parser = sub.add_parser("watch")
+    watch_parser.add_argument("root", type=Path)
+    watch_parser.add_argument("--db", type=Path, default=None)
+    watch_parser.add_argument("--l2", action="store_true", help="enable acoustic matching in the watcher")
+
+    args = parser.parse_args()
+    db = args.db or (Path("/data/dedupe.db") if args.cmd != "watch" and str(args.root).startswith("/music/") else args.root / ".dedupe" / "dedupe.db")
+
+    if args.cmd == "watch":
+        watch(args.root, db, l2=args.l2)
+        return
+
+    indexed = scan(args.root, db)
+    print(f"indexed={indexed} db={db}")
+    if args.cmd == "scan":
+        return
+
+    exact = exact_dedupe(db, args.root, dry_run=args.dry_run)
+    print(f"exact_hardlinked={len(exact)} dry_run={args.dry_run}")
+    l2 = acoustic_dedupe(db, args.root, dry_run=args.dry_run)
+    print(f"l2={l2} dry_run={args.dry_run}")

am_deduper/exact.py

@@ -39,7 +39,7 @@ def choose_winner(rows):
     return min(rows,key=key)
 
 
-def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str,str]]:
+def exact_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> list[tuple[str,str]]:
     actions=[]
     with connect(db_path) as conn:
         rows=conn.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')").fetchall()
@@ -62,7 +62,8 @@ def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str,str]]:
                     if r["id"]==winner["id"]: continue
                     lp=Path(r["file_path"])
                     if not lp.exists(): continue
-                    if lp.stat().st_ino != wp.stat().st_ino: hardlink_duplicate(lp,wp)
+                    if lp.stat().st_ino != wp.stat().st_ino and not dry_run:
-                    conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",(r["id"],))
+                        hardlink_duplicate(lp,wp)
+                        conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",(r["id"],))
                     actions.append((str(lp),str(wp)))
     return actions

am_deduper/l2.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from pathlib import Path
 from .acoustic import chromaprint, similarity
 from .arbitration import choose_winner, is_version_conflict, score
+from .exact import hardlink_duplicate
 from .db import connect
 from .constants import QUARANTINE_SUBDIR, TRASH_SUBDIR
 
@@ -22,7 +23,7 @@ def find_candidates(conn,row):
     return conn.execute("SELECT * FROM audio_tracks WHERE id!=? AND status IN ('active','hardlinked') AND duration IS NOT NULL AND duration BETWEEN ? AND ?", (row["id"],d-DURATION_DELTA,d+DURATION_DELTA)).fetchall()
 
 
-def acoustic_dedupe(db_path: Path, root: Path) -> dict:
+def acoustic_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> dict:
     stats={"checked":0,"equivalent":0,"review":0,"unique":0,"errors":0}
     with connect(db_path) as conn:
         rows=conn.execute("SELECT * FROM audio_tracks WHERE status='active' ORDER BY id").fetchall()
@@ -47,12 +48,11 @@ def acoustic_dedupe(db_path: Path, root: Path) -> dict:
             winner=choose_winner([row,c]); loser=c if winner["id"]==row["id"] else row
             lp=Path(loser["file_path"]); wp=Path(winner["file_path"])
             if lp.exists() and wp.exists():
-                trash=root/TRASH_SUBDIR
+                # Confirmed same-version acoustic match keeps the user's logical
-                trash.mkdir(parents=True,exist_ok=True)
+                # path. The physical data is shared via a hard link.
-                target=trash/lp.name
+                if lp.stat().st_ino != wp.stat().st_ino:
-                n=1
+                    if not dry_run:
-                while target.exists(): target=trash/f"{lp.stem}.{n}{lp.suffix}"; n+=1
+                        hardlink_duplicate(lp, wp)
-                lp.replace(target)
+                        conn.execute("UPDATE audio_tracks SET status='hardlinked',review_status=?,updated_at=CURRENT_TIMESTAMP WHERE id=?", (f"acoustic={sim:.3f};winner={wp}", loser["id"]))
-                conn.execute("UPDATE audio_tracks SET status='trashed',file_path=?,trashed_at=CURRENT_TIMESTAMP,review_status=?,updated_at=CURRENT_TIMESTAMP WHERE id=?",(str(target),f"equivalent={sim:.3f};winner={wp}",loser["id"]))
                 stats["equivalent"]+=1
     return stats

am_deduper/metadata.py

@@ -28,6 +28,6 @@ def read_metadata(path: Path) -> dict:
         "sample_rate": int(tag.samplerate) if tag.samplerate else None,
         "bit_depth": int(tag.bitdepth) if tag.bitdepth else None,
     })
-    text = " ".join(str(data.get(k) or "") for k in ("title", "album", "file_name"))
+    text = " ".join(str(data.get(k) or "") for k in ("title", "album")) + " " + path.name
     data["version_flags"] = [m for m in VERSION_MARKERS if re.search(rf"(?<!\w){re.escape(m)}(?!\w)", text, re.I)]
     return data

am_deduper/watcher.py

@@ -8,17 +8,20 @@ from watchdog.observers import Observer
 
 from .constants import MUSIC_EXTENSIONS, STABILITY_CHECKS, STABILITY_INTERVAL_SECONDS
 from .db import connect
+from .exact import exact_dedupe
+from .l2 import acoustic_dedupe
 from .scanner import index_file
 
 
 class Handler(FileSystemEventHandler):
-    def __init__(self, root: Path, db_path: Path):
+    def __init__(self, root: Path, db_path: Path, l2: bool = False):
         self.root = root
         self.db_path = db_path
+        self.l2 = l2
 
     def _process(self, filename: str) -> None:
         path = Path(filename)
-        if path.is_symlink() or path.suffix.lower() not in MUSIC_EXTENSIONS:
+        if path.is_symlink() or path.suffix.lower() not in MUSIC_EXTENSIONS or ".dedupe" in path.parts:
             return
         try:
             previous = path.stat().st_size
@@ -33,7 +36,11 @@ class Handler(FileSystemEventHandler):
                     stable = 0
             with connect(self.db_path) as conn:
                 index_file(conn, path)
-        except (FileNotFoundError, PermissionError):
+            # Exact dedupe is byte-for-byte safe. Acoustic dedupe is opt-in.
+            exact_dedupe(self.db_path, self.root)
+            if self.l2:
+                acoustic_dedupe(self.db_path, self.root)
+        except (FileNotFoundError, PermissionError, OSError):
             return
 
     def on_created(self, event):
@@ -45,9 +52,11 @@ class Handler(FileSystemEventHandler):
             self._process(event.dest_path)
 
 
-def watch(root: Path, db_path: Path) -> None:
+
+def watch(root: Path, db_path: Path, l2: bool = False) -> None:
+    root = root.resolve()
     observer = Observer()
-    observer.schedule(Handler(root, db_path), str(root), recursive=True)
+    observer.schedule(Handler(root, db_path, l2=l2), str(root), recursive=True)
     observer.start()
     try:
         while True:

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   am-deduper:
-    build: .
+    image: ${AM_DEDUPER_IMAGE:?Set AM_DEDUPER_IMAGE to the pushed Gitea registry image}
     container_name: am-deduper
     restart: unless-stopped
     command: ["watch", "/music/AllMusic"]
@@ -8,5 +8,6 @@ services:
       TZ: Asia/Shanghai
       PYTHONUNBUFFERED: "1"
     volumes:
-      - /your/nas/path/AllMusic:/music/AllMusic
+      - ${ALLMUSIC_PATH:?Set ALLMUSIC_PATH to the NAS AllMusic directory}:/music/AllMusic
+      - ${DEDUPE_DATA_PATH:?Set DEDUPE_DATA_PATH to the NAS app-data directory}:/data
     stop_grace_period: 30s

pyproject.toml

@@ -6,6 +6,8 @@ requires-python = ">=3.10"
 dependencies = [
   "tinytag>=2.1.1,<3",
   "watchdog>=6,<7",
+  "pyacoustid>=1.3,<2",
+  "pychromaprint>=1.2,<2",
 ]
 
 [project.optional-dependencies]