feat: add acoustic dedupe pipeline and docker runtime
This commit is contained in:
OpenClaw Agent
+45
-78
@@ -2,100 +2,67 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .db import connect
|
||||
from .hashing import sha256
|
||||
|
||||
|
||||
def confirm_sha256(conn, row) -> str | None:
|
||||
path = Path(row["file_path"])
|
||||
try:
|
||||
st = path.stat()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if st.st_size != row["file_size"]:
|
||||
return None
|
||||
# If the file changed after indexing, refresh the fast hash first and refuse
|
||||
# this pass. The next incremental/reconcile pass will safely retry it.
|
||||
try: st = path.stat()
|
||||
except FileNotFoundError: return None
|
||||
if st.st_size != row["file_size"]: return None
|
||||
digest = sha256(path)
|
||||
conn.execute(
|
||||
"UPDATE audio_tracks SET sha256=?, updated_at=CURRENT_TIMESTAMP WHERE id=?",
|
||||
(digest, row["id"]),
|
||||
)
|
||||
try:
|
||||
st2 = path.stat()
|
||||
except FileNotFoundError: return None
|
||||
if st2.st_size != st.st_size or st2.st_mtime_ns != st.st_mtime_ns: return None
|
||||
conn.execute("UPDATE audio_tracks SET sha256=?,updated_at=CURRENT_TIMESTAMP WHERE id=?", (digest,row["id"]))
|
||||
return digest
|
||||
|
||||
|
||||
def hardlink_duplicate(conn, loser: Path, winner: Path) -> None:
|
||||
"""Make loser a hard link to winner, atomically replacing loser."""
|
||||
def hardlink_duplicate(loser: Path, winner: Path) -> None:
|
||||
tmp = loser.with_name(loser.name + ".dedupe-linking")
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
os.link(winner, tmp)
|
||||
os.replace(tmp, loser)
|
||||
try:
|
||||
if tmp.exists(): tmp.unlink()
|
||||
os.link(winner, tmp)
|
||||
os.replace(tmp, loser)
|
||||
finally:
|
||||
if tmp.exists():
|
||||
try: tmp.unlink()
|
||||
except OSError: pass
|
||||
|
||||
|
||||
def choose_winner(rows):
|
||||
"""Prefer public admin paths, then older records, then lexical path."""
|
||||
def key(r):
|
||||
path = r["file_path"]
|
||||
is_admin = "/admin/" in path or path.endswith("/admin")
|
||||
return (0 if is_admin else 1, r["created_at"], path)
|
||||
return min(rows, key=key)
|
||||
p=str(r["file_path"]); admin="/admin/" in p or p.endswith("/admin")
|
||||
return (0 if admin else 1, r["created_at"], p)
|
||||
return min(rows,key=key)
|
||||
|
||||
|
||||
def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str, str]]:
|
||||
"""Hard-link byte-identical files while preserving every logical user path."""
|
||||
actions = []
|
||||
def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str,str]]:
|
||||
actions=[]
|
||||
with connect(db_path) as conn:
|
||||
candidates = conn.execute(
|
||||
"""
|
||||
SELECT * FROM audio_tracks
|
||||
WHERE status='active'
|
||||
ORDER BY id
|
||||
"""
|
||||
).fetchall()
|
||||
# Only full-hash files that share the cheap L1 key need to be compared.
|
||||
groups = {}
|
||||
for row in candidates:
|
||||
groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row)
|
||||
for rows in groups.values():
|
||||
if len(rows) < 2:
|
||||
# L1 has no collision candidate. Do not read the whole file;
|
||||
# the full SHA-256 will be computed only when a future file
|
||||
# lands in the same L1 bucket.
|
||||
continue
|
||||
for row in rows:
|
||||
if row["sha256"] is None:
|
||||
confirm_sha256(conn, row)
|
||||
refreshed = conn.execute(
|
||||
"SELECT * FROM audio_tracks WHERE id IN (%s)" % ",".join("?" * len(rows)),
|
||||
[r["id"] for r in rows],
|
||||
).fetchall()
|
||||
by_hash = {}
|
||||
for row in refreshed:
|
||||
if row["sha256"]:
|
||||
by_hash.setdefault(row["sha256"], []).append(row)
|
||||
for same in by_hash.values():
|
||||
if len(same) < 2:
|
||||
continue
|
||||
winner = choose_winner(same)
|
||||
winpath = Path(winner["file_path"])
|
||||
if not winpath.exists():
|
||||
continue
|
||||
for row in same:
|
||||
if row["id"] == winner["id"]:
|
||||
continue
|
||||
loser = Path(row["file_path"])
|
||||
if not loser.exists():
|
||||
continue
|
||||
# Preserve the user's logical path. Exact duplicates are NOT
|
||||
# moved to trash: all users should continue seeing/downloading
|
||||
# their own path, while the inode/data is shared.
|
||||
if loser.stat().st_ino != winpath.stat().st_ino:
|
||||
hardlink_duplicate(conn, loser, winpath)
|
||||
conn.execute(
|
||||
"UPDATE audio_tracks SET status='hardlinked', updated_at=CURRENT_TIMESTAMP WHERE id=?",
|
||||
(row["id"],),
|
||||
)
|
||||
actions.append((str(loser), str(winpath)))
|
||||
rows=conn.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')").fetchall()
|
||||
groups={}
|
||||
for r in rows: groups.setdefault((r["file_size"],r["fast_hash"]),[]).append(r)
|
||||
for candidates in groups.values():
|
||||
if len(candidates)<2: continue
|
||||
for r in candidates:
|
||||
if not r["sha256"]: confirm_sha256(conn,r)
|
||||
ids=[r["id"] for r in candidates]
|
||||
refreshed=conn.execute("SELECT * FROM audio_tracks WHERE id IN (%s)"%','.join('?'*len(ids)),ids).fetchall()
|
||||
by={}
|
||||
for r in refreshed:
|
||||
if r["sha256"]: by.setdefault(r["sha256"],[]).append(r)
|
||||
for same in by.values():
|
||||
if len(same)<2: continue
|
||||
winner=choose_winner(same); wp=Path(winner["file_path"])
|
||||
if not wp.exists(): continue
|
||||
for r in same:
|
||||
if r["id"]==winner["id"]: continue
|
||||
lp=Path(r["file_path"])
|
||||
if not lp.exists(): continue
|
||||
if lp.stat().st_ino != wp.stat().st_ino: hardlink_duplicate(lp,wp)
|
||||
conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",(r["id"],))
|
||||
actions.append((str(lp),str(wp)))
|
||||
return actions
|
||||
|
||||
Reference in New Issue
Block a user