openclaw/am-deduper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden dedupe runtime and NAS deployment

Browse Source
This commit is contained in:
OpenClaw Agent
2026-09-18 05:06:46 +08:00
parent 04e6f351e8
commit 7219d93b99
11 changed files with 180 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
am_deduper/acoustic.py
+25 -13
View File
@@ -5,6 +5,8 @@ import subprocess
from pathlib import Path
MAX_SECONDS = 120
MAX_BIT_ERROR = 2
MAX_ALIGN_OFFSET = 120
def chromaprint(path: Path) -> str | None:
@@ -13,26 +15,36 @@ def chromaprint(path: Path) -> str | None:
["fpcalc", "-length", str(MAX_SECONDS), "-json", str(path)],
capture_output=True, text=True, timeout=180, check=True,
)
return json.loads(p.stdout).get("fingerprint")
value = json.loads(p.stdout).get("fingerprint")
return value if isinstance(value, str) and value else None
except (OSError, subprocess.SubprocessError, ValueError, json.JSONDecodeError):
return None
def _popcount(value: int) -> int:
return value.bit_count()
def similarity(a: str, b: str) -> float:
"""Conservative normalized positional similarity for equal-length prefixes."""
"""Compare Chromaprint fingerprints using the AcoustID alignment method.
This is deliberately bounded to the local fingerprint comparison and does
not contact the AcoustID service.
"""
if not a or not b:
return 0.0
n = min(len(a), len(b))
if not n:
return 0.0
# Chromaprint JSON fingerprints are comma-separated integer features.
try:
aa, bb = [int(x) for x in a.split(",")], [int(x) for x in b.split(",")]
except ValueError:
import chromaprint as cp
aa = [int(x) for x in cp.decode_fingerprint(a.encode())[0]]
bb = [int(x) for x in cp.decode_fingerprint(b.encode())[0]]
except (ImportError, AttributeError, TypeError, ValueError):
return 0.0
n = min(len(aa), len(bb))
if not n:
if not aa or not bb:
return 0.0
# Feature-wise equality is intentionally conservative. A future backend
# can replace this with a library-specific fingerprint distance.
return sum(x == y for x, y in zip(aa[:n], bb[:n])) / n
counts = [0] * (len(aa) + len(bb) + 1)
for i, av in enumerate(aa):
for j in range(max(0, i - MAX_ALIGN_OFFSET), min(len(bb), i + MAX_ALIGN_OFFSET)):
if _popcount(av ^ bb[j]) <= MAX_BIT_ERROR:
counts[i - j + len(bb)] += 1
return max(counts) / min(len(aa), len(bb))
am_deduper/cli.py
+37 -11
View File
@@ -1,18 +1,44 @@
from __future__ import annotations
import argparse
from pathlib import Path
from .exact import exact_dedupe
from .scanner import scan
from .l2 import acoustic_dedupe
from .scanner import scan
from .watcher import watch
def main():
p=argparse.ArgumentParser(prog='am-deduper'); sub=p.add_subparsers(dest='cmd',required=True)
for name in ('scan','dedupe','reconcile','watch'):
sp=sub.add_parser(name); sp.add_argument('root',type=Path); sp.add_argument('--db',type=Path,default=None)
a=p.parse_args(); db=a.db or a.root/'.dedupe'/'dedupe.db'
if a.cmd=='watch': watch(a.root,db); return
print(f'indexed={scan(a.root,db)} db={db}')
if a.cmd in ('dedupe','reconcile'):
ex=exact_dedupe(db,a.root); print(f'exact_hardlinked={len(ex)}')
l2=acoustic_dedupe(db,a.root); print(f'l2={l2}')
def main() -> None:
parser = argparse.ArgumentParser(prog="am-deduper")
sub = parser.add_subparsers(dest="cmd", required=True)
def add_common(sp):
sp.add_argument("root", type=Path)
sp.add_argument("--db", type=Path, default=None)
sp.add_argument("--dry-run", action="store_true", help="report actions without changing files")
add_common(sub.add_parser("scan"))
add_common(sub.add_parser("dedupe"))
add_common(sub.add_parser("reconcile"))
watch_parser = sub.add_parser("watch")
watch_parser.add_argument("root", type=Path)
watch_parser.add_argument("--db", type=Path, default=None)
watch_parser.add_argument("--l2", action="store_true", help="enable acoustic matching in the watcher")
args = parser.parse_args()
db = args.db or (Path("/data/dedupe.db") if args.cmd != "watch" and str(args.root).startswith("/music/") else args.root / ".dedupe" / "dedupe.db")
if args.cmd == "watch":
watch(args.root, db, l2=args.l2)
return
indexed = scan(args.root, db)
print(f"indexed={indexed} db={db}")
if args.cmd == "scan":
return
exact = exact_dedupe(db, args.root, dry_run=args.dry_run)
print(f"exact_hardlinked={len(exact)} dry_run={args.dry_run}")
l2 = acoustic_dedupe(db, args.root, dry_run=args.dry_run)
print(f"l2={l2} dry_run={args.dry_run}")
am_deduper/exact.py
+4 -3
View File
@@ -39,7 +39,7 @@ def choose_winner(rows):
return min(rows,key=key)
def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str,str]]:
def exact_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> list[tuple[str,str]]:
actions=[]
with connect(db_path) as conn:
rows=conn.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')").fetchall()
@@ -62,7 +62,8 @@ def exact_dedupe(db_path: Path, root: Path) -> list[tuple[str,str]]:
if r["id"]==winner["id"]: continue
lp=Path(r["file_path"])
if not lp.exists(): continue
if lp.stat().st_ino != wp.stat().st_ino: hardlink_duplicate(lp,wp)
conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",(r["id"],))
if lp.stat().st_ino != wp.stat().st_ino and not dry_run:
hardlink_duplicate(lp,wp)
conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",(r["id"],))
actions.append((str(lp),str(wp)))
return actions
am_deduper/l2.py
+8 -8
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
from pathlib import Path
from .acoustic import chromaprint, similarity
from .arbitration import choose_winner, is_version_conflict, score
from .exact import hardlink_duplicate
from .db import connect
from .constants import QUARANTINE_SUBDIR, TRASH_SUBDIR
@@ -22,7 +23,7 @@ def find_candidates(conn,row):
return conn.execute("SELECT * FROM audio_tracks WHERE id!=? AND status IN ('active','hardlinked') AND duration IS NOT NULL AND duration BETWEEN ? AND ?", (row["id"],d-DURATION_DELTA,d+DURATION_DELTA)).fetchall()
def acoustic_dedupe(db_path: Path, root: Path) -> dict:
def acoustic_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> dict:
stats={"checked":0,"equivalent":0,"review":0,"unique":0,"errors":0}
with connect(db_path) as conn:
rows=conn.execute("SELECT * FROM audio_tracks WHERE status='active' ORDER BY id").fetchall()
@@ -47,12 +48,11 @@ def acoustic_dedupe(db_path: Path, root: Path) -> dict:
winner=choose_winner([row,c]); loser=c if winner["id"]==row["id"] else row
lp=Path(loser["file_path"]); wp=Path(winner["file_path"])
if lp.exists() and wp.exists():
trash=root/TRASH_SUBDIR
trash.mkdir(parents=True,exist_ok=True)
target=trash/lp.name
n=1
while target.exists(): target=trash/f"{lp.stem}.{n}{lp.suffix}"; n+=1
lp.replace(target)
conn.execute("UPDATE audio_tracks SET status='trashed',file_path=?,trashed_at=CURRENT_TIMESTAMP,review_status=?,updated_at=CURRENT_TIMESTAMP WHERE id=?",(str(target),f"equivalent={sim:.3f};winner={wp}",loser["id"]))
# Confirmed same-version acoustic match keeps the user's logical
# path. The physical data is shared via a hard link.
if lp.stat().st_ino != wp.stat().st_ino:
if not dry_run:
hardlink_duplicate(lp, wp)
conn.execute("UPDATE audio_tracks SET status='hardlinked',review_status=?,updated_at=CURRENT_TIMESTAMP WHERE id=?", (f"acoustic={sim:.3f};winner={wp}", loser["id"]))
stats["equivalent"]+=1
return stats
am_deduper/metadata.py
+1 -1
View File
@@ -28,6 +28,6 @@ def read_metadata(path: Path) -> dict:
"sample_rate": int(tag.samplerate) if tag.samplerate else None,
"bit_depth": int(tag.bitdepth) if tag.bitdepth else None,
})
text = " ".join(str(data.get(k) or "") for k in ("title", "album", "file_name"))
text = " ".join(str(data.get(k) or "") for k in ("title", "album")) + " " + path.name
data["version_flags"] = [m for m in VERSION_MARKERS if re.search(rf"(?<!\w){re.escape(m)}(?!\w)", text, re.I)]
return data
am_deduper/watcher.py
+14 -5
View File
@@ -8,17 +8,20 @@ from watchdog.observers import Observer
from .constants import MUSIC_EXTENSIONS, STABILITY_CHECKS, STABILITY_INTERVAL_SECONDS
from .db import connect
from .exact import exact_dedupe
from .l2 import acoustic_dedupe
from .scanner import index_file
class Handler(FileSystemEventHandler):
def __init__(self, root: Path, db_path: Path):
def __init__(self, root: Path, db_path: Path, l2: bool = False):
self.root = root
self.db_path = db_path
self.l2 = l2
def _process(self, filename: str) -> None:
path = Path(filename)
if path.is_symlink() or path.suffix.lower() not in MUSIC_EXTENSIONS:
if path.is_symlink() or path.suffix.lower() not in MUSIC_EXTENSIONS or ".dedupe" in path.parts:
return
try:
previous = path.stat().st_size
@@ -33,7 +36,11 @@ class Handler(FileSystemEventHandler):
stable = 0
with connect(self.db_path) as conn:
index_file(conn, path)
except (FileNotFoundError, PermissionError):
# Exact dedupe is byte-for-byte safe. Acoustic dedupe is opt-in.
exact_dedupe(self.db_path, self.root)
if self.l2:
acoustic_dedupe(self.db_path, self.root)
except (FileNotFoundError, PermissionError, OSError):
return
def on_created(self, event):
@@ -45,9 +52,11 @@ class Handler(FileSystemEventHandler):
self._process(event.dest_path)
def watch(root: Path, db_path: Path) -> None:
def watch(root: Path, db_path: Path, l2: bool = False) -> None:
root = root.resolve()
observer = Observer()
observer.schedule(Handler(root, db_path), str(root), recursive=True)
observer.schedule(Handler(root, db_path, l2=l2), str(root), recursive=True)
observer.start()
try:
while True: