feat: bootstrap NAS music deduper

am_deduper/exact.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from .db import connect
+from .hashing import sha256
+
+
+def confirm_sha256(conn, row) -> str | None:
+    path = Path(row["file_path"])
+    try:
+        st = path.stat()
+    except FileNotFoundError:
+        return None
+    if st.st_size != row["file_size"]:
+        return None
+    # If the file changed after indexing, refresh the fast hash first and refuse
+    # this pass. The next incremental/reconcile pass will safely retry it.
+    digest = sha256(path)
+    conn.execute(
+        "UPDATE audio_tracks SET sha256=?, updated_at=CURRENT_TIMESTAMP WHERE id=?",
+        (digest, row["id"]),
+    )
+    return digest
+
+
+def hardlink_duplicate(conn, loser: Path, winner: Path) -> None:
+    """Make loser a hard link to winner, atomically replacing loser."""
+    tmp = loser.with_name(loser.name + ".dedupe-linking")
+    if tmp.exists():
+        tmp.unlink()
+    os.link(winner, tmp)
+    os.replace(tmp, loser)
+
+
+def choose_winner(rows):
+    """Prefer public admin paths, then older records, then lexical path."""
+    def key(r):
+        path = r["file_path"]
+        is_admin = "/admin/" in path or path.endswith("/admin")
+        return (0 if is_admin else 1, r["created_at"], path)
+    return min(rows, key=key)
+
+
+def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str, str]]:
+    """Hard-link byte-identical files while preserving every logical user path."""
+    actions = []
+    with connect(db_path) as conn:
+        candidates = conn.execute(
+            """
+            SELECT * FROM audio_tracks
+            WHERE status='active'
+            ORDER BY id
+            """
+        ).fetchall()
+        # Only full-hash files that share the cheap L1 key need to be compared.
+        groups = {}
+        for row in candidates:
+            groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row)
+        for rows in groups.values():
+            if len(rows) < 2:
+                # L1 has no collision candidate. Do not read the whole file;
+                # the full SHA-256 will be computed only when a future file
+                # lands in the same L1 bucket.
+                continue
+            for row in rows:
+                if row["sha256"] is None:
+                    confirm_sha256(conn, row)
+            refreshed = conn.execute(
+                "SELECT * FROM audio_tracks WHERE id IN (%s)" % ",".join("?" * len(rows)),
+                [r["id"] for r in rows],
+            ).fetchall()
+            by_hash = {}
+            for row in refreshed:
+                if row["sha256"]:
+                    by_hash.setdefault(row["sha256"], []).append(row)
+            for same in by_hash.values():
+                if len(same) < 2:
+                    continue
+                winner = choose_winner(same)
+                winpath = Path(winner["file_path"])
+                if not winpath.exists():
+                    continue
+                for row in same:
+                    if row["id"] == winner["id"]:
+                        continue
+                    loser = Path(row["file_path"])
+                    if not loser.exists():
+                        continue
+                    # Preserve the user's logical path. Exact duplicates are NOT
+                    # moved to trash: all users should continue seeing/downloading
+                    # their own path, while the inode/data is shared.
+                    if loser.stat().st_ino != winpath.stat().st_ino:
+                        hardlink_duplicate(conn, loser, winpath)
+                    conn.execute(
+                        "UPDATE audio_tracks SET status='hardlinked', updated_at=CURRENT_TIMESTAMP WHERE id=?",
+                        (row["id"],),
+                    )
+                    actions.append((str(loser), str(winpath)))
+    return actions