102 lines
3.7 KiB
Python
102 lines
3.7 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from .db import connect
|
|
from .hashing import sha256
|
|
|
|
|
|
def confirm_sha256(conn, row) -> str | None:
|
|
path = Path(row["file_path"])
|
|
try:
|
|
st = path.stat()
|
|
except FileNotFoundError:
|
|
return None
|
|
if st.st_size != row["file_size"]:
|
|
return None
|
|
# If the file changed after indexing, refresh the fast hash first and refuse
|
|
# this pass. The next incremental/reconcile pass will safely retry it.
|
|
digest = sha256(path)
|
|
conn.execute(
|
|
"UPDATE audio_tracks SET sha256=?, updated_at=CURRENT_TIMESTAMP WHERE id=?",
|
|
(digest, row["id"]),
|
|
)
|
|
return digest
|
|
|
|
|
|
def hardlink_duplicate(conn, loser: Path, winner: Path) -> None:
|
|
"""Make loser a hard link to winner, atomically replacing loser."""
|
|
tmp = loser.with_name(loser.name + ".dedupe-linking")
|
|
if tmp.exists():
|
|
tmp.unlink()
|
|
os.link(winner, tmp)
|
|
os.replace(tmp, loser)
|
|
|
|
|
|
def choose_winner(rows):
|
|
"""Prefer public admin paths, then older records, then lexical path."""
|
|
def key(r):
|
|
path = r["file_path"]
|
|
is_admin = "/admin/" in path or path.endswith("/admin")
|
|
return (0 if is_admin else 1, r["created_at"], path)
|
|
return min(rows, key=key)
|
|
|
|
|
|
def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str, str]]:
|
|
"""Hard-link byte-identical files while preserving every logical user path."""
|
|
actions = []
|
|
with connect(db_path) as conn:
|
|
candidates = conn.execute(
|
|
"""
|
|
SELECT * FROM audio_tracks
|
|
WHERE status='active'
|
|
ORDER BY id
|
|
"""
|
|
).fetchall()
|
|
# Only full-hash files that share the cheap L1 key need to be compared.
|
|
groups = {}
|
|
for row in candidates:
|
|
groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row)
|
|
for rows in groups.values():
|
|
if len(rows) < 2:
|
|
# L1 has no collision candidate. Do not read the whole file;
|
|
# the full SHA-256 will be computed only when a future file
|
|
# lands in the same L1 bucket.
|
|
continue
|
|
for row in rows:
|
|
if row["sha256"] is None:
|
|
confirm_sha256(conn, row)
|
|
refreshed = conn.execute(
|
|
"SELECT * FROM audio_tracks WHERE id IN (%s)" % ",".join("?" * len(rows)),
|
|
[r["id"] for r in rows],
|
|
).fetchall()
|
|
by_hash = {}
|
|
for row in refreshed:
|
|
if row["sha256"]:
|
|
by_hash.setdefault(row["sha256"], []).append(row)
|
|
for same in by_hash.values():
|
|
if len(same) < 2:
|
|
continue
|
|
winner = choose_winner(same)
|
|
winpath = Path(winner["file_path"])
|
|
if not winpath.exists():
|
|
continue
|
|
for row in same:
|
|
if row["id"] == winner["id"]:
|
|
continue
|
|
loser = Path(row["file_path"])
|
|
if not loser.exists():
|
|
continue
|
|
# Preserve the user's logical path. Exact duplicates are NOT
|
|
# moved to trash: all users should continue seeing/downloading
|
|
# their own path, while the inode/data is shared.
|
|
if loser.stat().st_ino != winpath.stat().st_ino:
|
|
hardlink_duplicate(conn, loser, winpath)
|
|
conn.execute(
|
|
"UPDATE audio_tracks SET status='hardlinked', updated_at=CURRENT_TIMESTAMP WHERE id=?",
|
|
(row["id"],),
|
|
)
|
|
actions.append((str(loser), str(winpath)))
|
|
return actions
|