openclaw/am-deduper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: automate exact dedupe and improve web feedback

Browse Source
This commit is contained in:
OpenClaw Agent
2026-09-18 07:30:38 +08:00
parent 0fbadf0c9c
commit f10e92a5cd
8 changed files with 283 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
am_deduper/exact.py
+40 -41
View File
@@ -5,6 +5,7 @@ from pathlib import Path
from .db import connect
from .hashing import sha256
from .locking import exclusive
def confirm_sha256(conn, row) -> str | None:
@@ -59,50 +60,48 @@ def choose_winner(rows):
def exact_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> list[tuple[str, str]]:
actions: list[tuple[str, str]] = []
root = root.resolve()
with connect(db_path) as conn:
rows = conn.execute(
"SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')"
).fetchall()
groups: dict[tuple[int, str], list] = {}
for row in rows:
groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row)
for candidates in groups.values():
if len(candidates) < 2:
continue
for row in candidates:
if not row["sha256"]:
confirm_sha256(conn, row)
ids = [row["id"] for row in candidates]
placeholders = ",".join("?" * len(ids))
refreshed = conn.execute(
f"SELECT * FROM audio_tracks WHERE id IN ({placeholders})", ids
with exclusive(db_path):
with connect(db_path) as conn:
rows = conn.execute(
"SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')"
).fetchall()
by_sha: dict[str, list] = {}
for row in refreshed:
if row["sha256"]:
by_sha.setdefault(row["sha256"], []).append(row)
groups: dict[tuple[int, str], list] = {}
for row in rows:
groups.setdefault((row["file_size"], row["fast_hash"]), []).append(row)
for same in by_sha.values():
if len(same) < 2:
for candidates in groups.values():
if len(candidates) < 2:
continue
winner = choose_winner(same)
wp = Path(winner["file_path"])
if not wp.exists() or not wp.is_relative_to(root):
continue
for row in same:
if row["id"] == winner["id"]:
for row in candidates:
if not row["sha256"]:
confirm_sha256(conn, row)
ids = [row["id"] for row in candidates]
placeholders = ",".join("?" * len(ids))
refreshed = conn.execute(
f"SELECT * FROM audio_tracks WHERE id IN ({placeholders})", ids
).fetchall()
by_sha: dict[str, list] = {}
for row in refreshed:
if row["sha256"]:
by_sha.setdefault(row["sha256"], []).append(row)
for same in by_sha.values():
if len(same) < 2:
continue
lp = Path(row["file_path"])
if not lp.exists() or not lp.is_relative_to(root):
winner = choose_winner(same)
wp = Path(winner["file_path"])
if not wp.exists() or not wp.is_relative_to(root):
continue
if lp.stat().st_ino != wp.stat().st_ino:
if not dry_run:
hardlink_duplicate(lp, wp)
conn.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ('exact-hardlink',str(lp),str(wp),'sha256 exact duplicate'))
conn.execute(
"UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",
(row["id"],),
)
actions.append((str(lp), str(wp)))
for row in same:
if row["id"] == winner["id"]:
continue
lp = Path(row["file_path"])
if not lp.exists() or not lp.is_relative_to(root):
continue
if lp.stat().st_ino != wp.stat().st_ino:
if not dry_run:
hardlink_duplicate(lp, wp)
conn.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ('exact-hardlink',str(lp),str(wp),'sha256 exact duplicate'))
conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?", (row["id"],))
actions.append((str(lp), str(wp)))
return actions