openclaw/am-deduper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: automate exact dedupe and improve web feedback

Browse Source
This commit is contained in:
OpenClaw Agent
2026-09-18 07:30:38 +08:00
parent 0fbadf0c9c
commit f10e92a5cd
8 changed files with 283 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5 -1
View File
@@ -43,7 +43,7 @@ am-deduper dedupe /music/AllMusic --db /data/dedupe.db --dry-run
# 确认后执行 # 确认后执行
am-deduper dedupe /music/AllMusic --db /data/dedupe.db am-deduper dedupe /music/AllMusic --db /data/dedupe.db
# 长驻监听：默认只自动执行安全的 Exact hard-link # 长驻监听：启动时先做一次全量索引 + Exact 去重，之后自动执行安全的 Exact hard-link
am-deduper watch /music/AllMusic --db /data/dedupe.db am-deduper watch /music/AllMusic --db /data/dedupe.db
# 开启 L2 acoustic 匹配；建议先 dry-run 验证 # 开启 L2 acoustic 匹配；建议先 dry-run 验证
@@ -123,7 +123,11 @@ http://<NAS-IP>:18080/
- 待审核：查看相似度、候选文件，可人工确认或保留。 - 待审核：查看相似度、候选文件，可人工确认或保留。
- 任务：查看 scan / dry-run / dedupe 状态。 - 任务：查看 scan / dry-run / dedupe 状态。
- 历史：查看 Hard Link 操作，并对仍保持同 inode 的操作执行安全回滚。 - 历史：查看 Hard Link 操作，并对仍保持同 inode 的操作执行安全回滚。
- 监听服务启动时会先做一次全量扫描与 Exact 去重；新文件稳定后也会自动索引并执行 Exact 去重。
- 跨 Web 与 watcher 进程使用数据库锁，避免两个容器同时执行 Hard Link。
- 文件被替换时会自动清空旧 SHA-256 / Chromaprint，防止旧内容哈希污染新文件。
- 扫描会把数据库中已经不存在的文件标记为 `missing`，避免脏索引继续计入统计。 - 扫描会把数据库中已经不存在的文件标记为 `missing`，避免脏索引继续计入统计。
- Web UI 的每个操作都有明确的启动、运行、完成/失败反馈，并保留任务结果。
> Web UI 支持可选管理 Token。生产环境建议在 NAS 的 Compose 同目录 `.env` 中设置 `AM_DEDUPER_TOKEN=<随机长字符串>`；不要把 Token 提交到 Git。设置后 Web UI 会在首次访问 API 时要求输入 Token。若未设置 Token，则仅适合可信内网环境，不要直接暴露到公网。 > Web UI 支持可选管理 Token。生产环境建议在 NAS 的 Compose 同目录 `.env` 中设置 `AM_DEDUPER_TOKEN=<随机长字符串>`；不要把 Token 提交到 Git。设置后 Web UI 会在首次访问 API 时要求输入 Token。若未设置 Token，则仅适合可信内网环境，不要直接暴露到公网。
am_deduper/exact.py
+3 -4
View File
@@ -5,6 +5,7 @@ from pathlib import Path
from .db import connect from .db import connect
from .hashing import sha256 from .hashing import sha256
from .locking import exclusive
def confirm_sha256(conn, row) -> str | None: def confirm_sha256(conn, row) -> str | None:
@@ -59,6 +60,7 @@ def choose_winner(rows):
def exact_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> list[tuple[str, str]]: def exact_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> list[tuple[str, str]]:
actions: list[tuple[str, str]] = [] actions: list[tuple[str, str]] = []
root = root.resolve() root = root.resolve()
with exclusive(db_path):
with connect(db_path) as conn: with connect(db_path) as conn:
rows = conn.execute( rows = conn.execute(
"SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')" "SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked')"
@@ -100,9 +102,6 @@ def exact_dedupe(db_path: Path, root: Path, dry_run: bool = False) -> list[tuple
if not dry_run: if not dry_run:
hardlink_duplicate(lp, wp) hardlink_duplicate(lp, wp)
conn.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ('exact-hardlink',str(lp),str(wp),'sha256 exact duplicate')) conn.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ('exact-hardlink',str(lp),str(wp),'sha256 exact duplicate'))
conn.execute( conn.execute("UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?", (row["id"],))
"UPDATE audio_tracks SET status='hardlinked',updated_at=CURRENT_TIMESTAMP WHERE id=?",
(row["id"],),
)
actions.append((str(lp), str(wp))) actions.append((str(lp), str(wp)))
return actions return actions
am_deduper/locking.py
+18
View File
@@ -0,0 +1,18 @@
from __future__ import annotations
from contextlib import contextmanager
from pathlib import Path
import fcntl
@contextmanager
def exclusive(db_path: Path):
"""Cross-process lock for destructive dedupe operations."""
lock_path = Path(str(db_path) + ".lock")
lock_path.parent.mkdir(parents=True, exist_ok=True)
with lock_path.open("w") as handle:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
am_deduper/scanner.py
+14 -1
View File
@@ -22,14 +22,27 @@ def index_file(conn, path: Path) -> None:
fh = fast_hash(path) fh = fast_hash(path)
meta = read_metadata(path) meta = read_metadata(path)
flags = ",".join(meta.get("version_flags", [])) flags = ",".join(meta.get("version_flags", []))
old = conn.execute("SELECT file_size,fast_hash,inode,device FROM audio_tracks WHERE file_path=?", (str(path),)).fetchone()
identity_changed = old is not None and (old["file_size"], old["fast_hash"], old["inode"], old["device"]) != (st.st_size, fh, st.st_ino, st.st_dev)
conn.execute(""" conn.execute("""
INSERT INTO audio_tracks(file_path,file_size,fast_hash,duration,artist,title,album,format,bitrate,sample_rate,bit_depth,version_flags,inode,device,status) INSERT INTO audio_tracks(file_path,file_size,fast_hash,duration,artist,title,album,format,bitrate,sample_rate,bit_depth,version_flags,inode,device,status)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,'active') VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,'active')
ON CONFLICT(file_path) DO UPDATE SET file_size=excluded.file_size, fast_hash=excluded.fast_hash, ON CONFLICT(file_path) DO UPDATE SET file_size=excluded.file_size, fast_hash=excluded.fast_hash,
duration=excluded.duration,artist=excluded.artist,title=excluded.title,album=excluded.album,format=excluded.format, duration=excluded.duration,artist=excluded.artist,title=excluded.title,album=excluded.album,format=excluded.format,
bitrate=excluded.bitrate,sample_rate=excluded.sample_rate,bit_depth=excluded.bit_depth,version_flags=excluded.version_flags, bitrate=excluded.bitrate,sample_rate=excluded.sample_rate,bit_depth=excluded.bit_depth,version_flags=excluded.version_flags,
inode=excluded.inode,device=excluded.device,updated_at=CURRENT_TIMESTAMP,status='active' inode=excluded.inode,device=excluded.device,updated_at=CURRENT_TIMESTAMP,status='active',
sha256=CASE WHEN excluded.file_size!=audio_tracks.file_size OR excluded.fast_hash!=audio_tracks.fast_hash
OR excluded.inode!=audio_tracks.inode OR excluded.device!=audio_tracks.device
THEN NULL ELSE audio_tracks.sha256 END,
chromaprint=CASE WHEN excluded.file_size!=audio_tracks.file_size OR excluded.fast_hash!=audio_tracks.fast_hash
OR excluded.inode!=audio_tracks.inode OR excluded.device!=audio_tracks.device
THEN NULL ELSE audio_tracks.chromaprint END,
review_status=CASE WHEN excluded.file_size!=audio_tracks.file_size OR excluded.fast_hash!=audio_tracks.fast_hash
OR excluded.inode!=audio_tracks.inode OR excluded.device!=audio_tracks.device
THEN NULL ELSE audio_tracks.review_status END
""", (str(path),st.st_size,fh,meta.get("duration"),meta.get("artist"),meta.get("title"),meta.get("album"),meta.get("format"),meta.get("bitrate"),meta.get("sample_rate"),meta.get("bit_depth"),flags,st.st_ino,st.st_dev)) """, (str(path),st.st_size,fh,meta.get("duration"),meta.get("artist"),meta.get("title"),meta.get("album"),meta.get("format"),meta.get("bitrate"),meta.get("sample_rate"),meta.get("bit_depth"),flags,st.st_ino,st.st_dev))
if identity_changed:
conn.execute("UPDATE audio_tracks SET updated_at=CURRENT_TIMESTAMP WHERE file_path=?", (str(path),))
def scan(root: Path, db_path: Path) -> int: def scan(root: Path, db_path: Path) -> int:
am_deduper/watcher.py
+64 -15
View File
@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import json
import threading import threading
import time import time
from pathlib import Path from pathlib import Path
@@ -11,7 +12,7 @@ from .constants import MUSIC_EXTENSIONS, STABILITY_CHECKS, STABILITY_INTERVAL_SE
from .db import connect from .db import connect
from .exact import exact_dedupe from .exact import exact_dedupe
from .l2 import acoustic_dedupe from .l2 import acoustic_dedupe
from .scanner import index_file from .scanner import index_file, scan
class Handler(FileSystemEventHandler): class Handler(FileSystemEventHandler):
@@ -20,6 +21,17 @@ class Handler(FileSystemEventHandler):
self.db_path = db_path self.db_path = db_path
self.l2 = l2 self.l2 = l2
self._lock = threading.Lock() self._lock = threading.Lock()
self._timers: dict[str, threading.Timer] = {}
self._timers_lock = threading.Lock()
def _record_job(self, kind: str, status: str, job_id: int, result=None, error=None):
with connect(self.db_path) as conn:
conn.execute("UPDATE jobs SET status=?,finished_at=CURRENT_TIMESTAMP,result=?,error=? WHERE id=?",
(status, json.dumps(result, ensure_ascii=False) if result is not None else None, error, job_id))
def _new_job(self, kind: str) -> int:
with connect(self.db_path) as conn:
return conn.execute("INSERT INTO jobs(kind,status) VALUES(?,?)", (kind, "running")).lastrowid
def _process(self, filename: str) -> None: def _process(self, filename: str) -> None:
path = Path(filename) path = Path(filename)
@@ -32,40 +44,75 @@ class Handler(FileSystemEventHandler):
if ".dedupe" in path.parts or path.name.startswith("."): if ".dedupe" in path.parts or path.name.startswith("."):
return return
# watchdog may emit several events while a downloader is still writing.
# Serialize the worker so a file is never indexed while another event
# is changing the same database or replacing a duplicate.
with self._lock: with self._lock:
job_id = self._new_job("auto-exact")
try: try:
previous = path.stat() previous = path.stat()
stable = 0 stable = 0
while stable < STABILITY_CHECKS - 1: while stable < STABILITY_CHECKS - 1:
time.sleep(STABILITY_INTERVAL_SECONDS) time.sleep(STABILITY_INTERVAL_SECONDS)
current = path.stat() current = path.stat()
if (current.st_size, current.st_mtime_ns) == ( if (current.st_size, current.st_mtime_ns) == (previous.st_size, previous.st_mtime_ns):
previous.st_size,
previous.st_mtime_ns,
):
stable += 1 stable += 1
else: else:
previous = current previous = current
stable = 0 stable = 0
with connect(self.db_path) as conn: with connect(self.db_path) as conn:
index_file(conn, path) index_file(conn, path)
# Exact byte-for-byte dedupe is always safe. L2 remains opt-in. actions = exact_dedupe(self.db_path, self.root)
exact_dedupe(self.db_path, self.root) result = {"indexed": 1, "exact_hardlinked": len(actions), "path": str(path)}
if self.l2: if self.l2:
acoustic_dedupe(self.db_path, self.root) result["acoustic"] = acoustic_dedupe(self.db_path, self.root)
except (FileNotFoundError, PermissionError, OSError): self._record_job("auto-exact", "success", job_id, result=result)
return except (FileNotFoundError, PermissionError, OSError) as exc:
self._record_job("auto-exact", "skipped", job_id, error=str(exc))
except Exception as exc:
self._record_job("auto-exact", "failed", job_id, error=str(exc))
def _schedule(self, filename: str) -> None:
with self._timers_lock:
old = self._timers.pop(filename, None)
if old:
old.cancel()
timer = threading.Timer(1.0, self._run_scheduled, args=(filename,))
timer.daemon = True
self._timers[filename] = timer
timer.start()
def _run_scheduled(self, filename: str) -> None:
with self._timers_lock:
self._timers.pop(filename, None)
self._process(filename)
def on_created(self, event): def on_created(self, event):
if not event.is_directory: if not event.is_directory:
self._process(event.src_path) self._schedule(event.src_path)
def on_modified(self, event):
if not event.is_directory:
self._schedule(event.src_path)
def on_moved(self, event): def on_moved(self, event):
if not event.is_directory: if not event.is_directory:
self._process(event.dest_path) self._schedule(event.dest_path)
def _initial_reconcile(root: Path, db_path: Path, l2: bool) -> None:
with connect(db_path) as conn:
job_id = conn.execute("INSERT INTO jobs(kind,status) VALUES(?,?)", ("startup-reconcile", "running")).lastrowid
try:
indexed = scan(root, db_path)
actions = exact_dedupe(db_path, root)
result = {"indexed": indexed, "exact_hardlinked": len(actions)}
if l2:
result["acoustic"] = acoustic_dedupe(db_path, root)
with connect(db_path) as conn:
conn.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?",
(json.dumps(result, ensure_ascii=False), job_id))
except Exception as exc:
with connect(db_path) as conn:
conn.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?", (str(exc), job_id))
raise
def watch(root: Path, db_path: Path, l2: bool = False) -> None: def watch(root: Path, db_path: Path, l2: bool = False) -> None:
@@ -73,9 +120,11 @@ def watch(root: Path, db_path: Path, l2: bool = False) -> None:
if not root.is_dir(): if not root.is_dir():
raise ValueError(f"music root does not exist or is not a directory: {root}") raise ValueError(f"music root does not exist or is not a directory: {root}")
db_path.parent.mkdir(parents=True, exist_ok=True) db_path.parent.mkdir(parents=True, exist_ok=True)
_initial_reconcile(root, db_path, l2)
observer = Observer() observer = Observer()
observer.schedule(Handler(root, db_path, l2=l2), str(root), recursive=True) observer.schedule(Handler(root, db_path, l2=l2), str(root), recursive=True)
observer.start() observer.start()
print(f"AM Deduper watcher active: {root}", flush=True)
try: try:
while True: while True:
time.sleep(1) time.sleep(1)
am_deduper/web.py
+83 -67
View File
@@ -1,8 +1,14 @@
from __future__ import annotations from __future__ import annotations
import json, os, shutil, threading, uuid
import json
import os
import shutil
import threading
import uuid
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
from .db import connect from .db import connect
from .scanner import scan from .scanner import scan
from .exact import exact_dedupe, hardlink_duplicate, choose_winner from .exact import exact_dedupe, hardlink_duplicate, choose_winner
@@ -16,45 +22,56 @@ LOCK=threading.Lock()
HTML = r'''<!doctype html><html lang="zh-CN"><meta charset="utf-8"> HTML = r'''<!doctype html><html lang="zh-CN"><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1"><title>AM Deduper</title> <meta name="viewport" content="width=device-width,initial-scale=1"><title>AM Deduper</title>
<style> <style>
body{font:14px system-ui;margin:0;background:#f5f6f8;color:#202124}header{background:#111827;color:#fff;padding:18px 24px} :root{font:14px system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#172033;background:#f4f6f8}*{box-sizing:border-box}
main{max-width:1280px;margin:18px auto;padding:0 14px}.grid{display:grid;grid-template-columns:repeat(5,1fr);gap:10px} body{margin:0}header{background:#111827;color:white;padding:18px 24px;position:sticky;top:0;z-index:5;box-shadow:0 2px 10px #0002}
.card,section{background:#fff;border:1px solid #ddd;border-radius:10px;padding:14px;margin-bottom:14px}.num{font-size:24px;font-weight:700} header .sub{opacity:.7;font-size:12px;margin-top:4px}.wrap{max-width:1400px;margin:18px auto;padding:0 14px}.actions{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px}
nav{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:14px}button{padding:8px 12px;border:1px solid #bbb;border-radius:7px;background:#fff;cursor:pointer} button{border:1px solid #cbd0d8;border-radius:8px;padding:9px 13px;background:white;color:#172033;cursor:pointer}button:hover{background:#f0f2f5}button:disabled{opacity:.5;cursor:wait}
button.primary{background:#111827;color:#fff}button.danger{background:#fee2e2}table{width:100%;border-collapse:collapse}th,td{padding:9px;border-bottom:1px solid #eee;text-align:left;vertical-align:top} button.primary{background:#111827;color:#fff;border-color:#111827}button.warn{background:#fff7ed;border-color:#fed7aa}button.danger{background:#fef2f2;border-color:#fecaca}
.path{font:12px monospace;word-break:break-all}.muted{color:#666}.badge{background:#eee;border-radius:10px;padding:3px 7px} #toast{position:fixed;right:18px;bottom:18px;z-index:20;max-width:420px;padding:12px 16px;border-radius:10px;background:#111827;color:#fff;box-shadow:0 6px 24px #0003;display:none}
@media(max-width:800px){.grid{grid-template-columns:repeat(2,1fr)}table{font-size:12px}} #toast.error{background:#991b1b}.statusbar{background:#fff;border:1px solid #dfe3e8;border-radius:10px;padding:12px 14px;margin-bottom:14px;display:flex;justify-content:space-between;gap:12px;align-items:center}.status-main{font-weight:600}.status-detail{font-size:12px;color:#667085;margin-top:3px}
.grid{display:grid;grid-template-columns:repeat(5,1fr);gap:10px}.card,section{background:#fff;border:1px solid #dfe3e8;border-radius:12px;padding:15px;margin-bottom:14px}.card .label{color:#667085}.num{font-size:26px;font-weight:750;margin-top:4px}.hint{font-size:12px;color:#667085;margin-top:3px}
h2{font-size:17px;margin:0 0 12px}table{width:100%;border-collapse:collapse}th,td{padding:9px;border-bottom:1px solid #edf0f2;text-align:left;vertical-align:top}th{color:#667085;font-weight:600;font-size:12px}.path{font:12px ui-monospace,SFMono-Regular,Menlo,monospace;word-break:break-all}.muted{color:#667085}.badge{display:inline-block;padding:3px 7px;border-radius:999px;background:#eef2f6;font-size:12px}.success{color:#166534}.error{color:#991b1b}.running{color:#1d4ed8}.empty{padding:22px;text-align:center;color:#667085;background:#fafbfc;border-radius:8px}
.jobline{display:flex;align-items:center;gap:8px}.spinner{width:12px;height:12px;border:2px solid #cbd5e1;border-top-color:#2563eb;border-radius:50%;animation:spin .8s linear infinite}@keyframes spin{to{transform:rotate(360deg)}}
details{margin-top:8px}summary{cursor:pointer;color:#475467}.toolbar{display:flex;justify-content:space-between;gap:8px;align-items:center;margin-bottom:8px}.small{font-size:12px}
@media(max-width:900px){.grid{grid-template-columns:repeat(2,1fr)}.statusbar{align-items:flex-start;flex-direction:column}table{display:block;overflow:auto;white-space:nowrap}}
</style> </style>
<header><b>AM Deduper</b> · NAS Music Library</header><main> <header><div><b>AM Deduper</b> · NAS Music Library</div><div class="sub">自动监听 · SHA-256 精确去重 · Hard Link · 人工审核</div></header>
<nav><button class="primary" onclick="run('scan')">扫描索引</button><button onclick="run('dry')">分析 / Dry-run</button> <div class="wrap">
<button onclick="run('dedupe')">执行去重</button><button onclick="load()">刷新</button></nav> <div class="actions"><button class="primary" data-action="scan" onclick="run('scan')">扫描并更新索引</button><button data-action="dry" onclick="run('dry')">分析重复（Dry-run）</button><button class="warn" data-action="dedupe" onclick="run('dedupe')">执行安全去重</button><button onclick="load(true)">刷新</button></div>
<div id="status" class="statusbar"><div><div class="status-main">正在加载…</div><div class="status-detail">读取任务和索引状态</div></div><span class="badge">自动刷新 5 秒</span></div>
<div class="grid" id="stats"></div> <div class="grid" id="stats"></div>
<section><h2>重复组</h2><div id="dupes"></div></section> <section><div class="toolbar"><h2>重复组</h2><span class="small muted" id="dupeCount"></span></div><div id="dupes"></div></section>
<section><h2>待人工审核</h2><div id="review"></div></section> <section><div class="toolbar"><h2>待人工审核</h2><span class="small muted" id="reviewCount"></span></div><div id="review"></div></section>
<section><h2>任务</h2><div id="jobsBox"></div></section> <section><div class="toolbar"><h2>任务执行记录</h2><span class="small muted">包含自动去重任务</span></div><div id="jobsBox"></div></section>
<section><h2>最近操作</h2><div id="historyBox"></div></section> <section><div class="toolbar"><h2>最近操作</h2><span class="small muted">可回滚仍保持同 inode 的 Hard Link</span></div><div id="historyBox"></div></section>
<section><h2>设置</h2><div id="settingsBox" class="muted">加载中...</div></section> <section><h2>运行设置</h2><div id="settingsBox" class="muted">加载中...</div></section>
</main> </div><div id="toast"></div>
<script> <script>
const esc=s=>String(s??'').replace(/[&<>"]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c])); const esc=s=>String(s??'').replace(/[&<>\"]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c]));
async function api(u,o={}){o.headers=Object.assign({},o.headers||{});let t=localStorage.getItem('am_token');if(t)o.headers.Authorization='Bearer '+t;let r=await fetch(u,o);let j=await r.json();if(r.status===401){t=prompt('请输入 AM Deduper 管理 Token');if(t){localStorage.setItem('am_token',t);return api(u,o)} }if(!r.ok)throw Error(j.error||r.status);return j} const labels={scan:'扫描索引',dry:'Dry-run 分析',dedupe:'执行去重'};
async function load(){try{ let busy=false;
let s=await api('/api/stats');stats.innerHTML=s.cards.map(x=>'<div class="card"><div class="muted">'+esc(x.name)+'</div><div class="num">'+esc(x.value)+'</div></div>').join(''); function toast(msg,error=false){const e=document.getElementById('toast');e.textContent=msg;e.className=error?'error':'';e.style.display='block';clearTimeout(window._toast);window._toast=setTimeout(()=>e.style.display='none',4500)}
let d=await api('/api/duplicates');dupes.innerHTML=d.groups.length?d.groups.map(g=>'<div class="card"><b>'+esc(g.type)+' · '+esc(g.savings)+' MB 可节省</b><table><tr><th>文件</th><th>状态</th></tr>'+g.items.map(x=>'<tr><td class="path">'+esc(x.file_path)+'</td><td>'+esc(x.status)+'</td></tr>').join('')+'</table></div>').join(''):'<p class="muted">暂无重复组。</p>'; async function api(u,o={}){o.headers=Object.assign({},o.headers||{});const t=localStorage.getItem('am_token');if(t)o.headers.Authorization='Bearer '+t;let r=await fetch(u,o);let j={};try{j=await r.json()}catch{}if(r.status===401){const nt=prompt('请输入 AM Deduper 管理 Token');if(nt){localStorage.setItem('am_token',nt);return api(u,o)}throw Error('需要管理 Token')}if(!r.ok)throw Error(j.error||('HTTP '+r.status));return j}
function setButtons(v){document.querySelectorAll('[data-action]').forEach(b=>b.disabled=v)}
let q=await api('/api/review');review.innerHTML=q.items.length?'<table><tr><th>相似度</th><th>文件</th><th>候选目标</th><th>操作</th></tr>'+q.items.map(x=>'<tr><td>'+esc(x.similarity||'-')+'</td><td class="path">'+esc(x.file_path)+'<br>'+esc(x.title||'')+' '+esc(x.artist||'')+'</td><td class="path">'+esc(x.candidate_path||'-')+'</td><td><button onclick="reviewAction('+x.id+',\'reject\')">保留</button> '+(x.candidate_id?'<button class="primary" onclick="reviewAction('+x.id+',\'approve\','+x.candidate_id+')">确认重复</button>':'')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无待审核项目。</p>'; function renderStatus(j){const e=document.getElementById('status');if(!j||!j.items?.length){e.innerHTML='<div><div class="status-main">系统空闲</div><div class="status-detail">监听服务会在新音乐稳定写入后自动进行 SHA-256 精确去重。</div></div><span class="badge">运行正常</span>';return}const x=j.items[0],running=x.status==='running';let detail=x.result||x.error||'等待结果';e.innerHTML='<div><div class="status-main '+(running?'running':x.status==='failed'?'error':'')+'">'+(running?'<span class="jobline"><span class="spinner"></span>':'')+esc(labels[x.kind]||x.kind)+(running?'…</span>':'')+'</div><div class="status-detail">'+esc(detail)+'</div></div><span class="badge">'+esc(x.status)+'</span>';}
let j=await api('/api/jobs');jobsBox.innerHTML=j.items.length?'<table><tr><th>ID</th><th>任务</th><th>状态</th><th>开始</th><th>结束</th></tr>'+j.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.kind)+'</td><td>'+esc(x.status)+'</td><td>'+esc(x.started_at)+'</td><td>'+esc(x.finished_at||'-')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无任务。</p>'; function formatResult(r){if(!r)return '';try{const x=JSON.parse(r);const parts=[];if(x.indexed!=null)parts.push('索引 '+x.indexed);if(x.exact_hardlinked!=null)parts.push('精确去重 '+x.exact_hardlinked+' 个');if(x.exact!=null)parts.push('Exact '+x.exact+' 个');if(x.acoustic)parts.push('L2 '+(x.acoustic.equivalent??0)+' 个');return parts.join(' · ')||r}catch{return r}}
async function load(manual=false){try{
let h=await api('/api/history');historyBox.innerHTML=h.items.length?'<table><tr><th>ID</th><th>时间</th><th>操作</th><th>文件</th><th>目标</th><th></th></tr>'+h.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.created_at)+'</td><td>'+esc(x.action)+'</td><td class="path">'+esc(x.file_path)+'</td><td class="path">'+esc(x.winner_path||'')+'</td><td>'+(x.reversible?'<button onclick="rollback('+x.id+')">回滚</button>':'')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无记录。</p>'; const [s,d,q,j,h,cfg]=await Promise.all([api('/api/stats'),api('/api/duplicates'),api('/api/review'),api('/api/jobs'),api('/api/history'),api('/api/settings')]);
}catch(e){alert(e.message)}} document.getElementById('stats').innerHTML=s.cards.map(x=>'<div class="card"><div class="label">'+esc(x.name)+'</div><div class="num">'+esc(x.value)+'</div><div class="hint">'+esc(x.hint||'')+'</div></div>').join('');
async function run(kind){try{await api('/api/jobs',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({kind})});load()}catch(e){alert(e.message)}} document.getElementById('dupeCount').textContent=d.groups.length+' 组';document.getElementById('dupes').innerHTML=d.groups.length?d.groups.map(g=>'<div class="card"><b>'+esc(g.type)+'</b> · 可节省约 '+esc(g.savings)+' MB · '+esc(g.items.length)+' 个逻辑路径<table><tr><th>路径</th><th>状态</th><th>inode</th></tr>'+g.items.map(x=>'<tr><td class="path">'+esc(x.file_path)+'</td><td><span class="badge">'+esc(x.status)+'</span>'+(x.id===g.winner_id?' <span class="badge">保留路径</span>':'')+'</td><td>'+esc(x.inode)+'</td></tr>').join('')+'</table></div>').join(''):'<div class="empty">没有发现 SHA-256 完全重复。新文件会由监听服务自动检查。</div>';
async function reviewAction(id,action,candidate_id){try{await api('/api/review',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id,action,candidate_id})});load()}catch(e){alert(e.message)}} document.getElementById('reviewCount').textContent=q.items.length+' 项';document.getElementById('review').innerHTML=q.items.length?'<table><tr><th>相似度</th><th>文件</th><th>候选目标</th><th>操作</th></tr>'+q.items.map(x=>'<tr><td>'+esc(x.similarity||'-')+'</td><td class="path">'+esc(x.file_path)+'<br>'+esc(x.title||'')+' '+esc(x.artist||'')+'</td><td class="path">'+esc(x.candidate_path||'-')+'</td><td><button onclick="reviewAction('+x.id+',\'reject\')">保留原文件</button> '+(x.candidate_id?'<button class="primary" onclick="reviewAction('+x.id+',\'approve\','+x.candidate_id+')">确认硬链接</button>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无人工审核项目。</div>';
async function rollback(id){if(!confirm('确认回滚此 Hard Link？'))return;try{await api('/api/rollback',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id})});load()}catch(e){alert(e.message)}} document.getElementById('jobsBox').innerHTML=j.items.length?'<table><tr><th>ID</th><th>任务</th><th>状态</th><th>时间</th><th>结果</th></tr>'+j.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(labels[x.kind]||x.kind)+'</td><td class="'+(x.status==='failed'?'error':x.status==='running'?'running':'success')+'">'+esc(x.status)+'</td><td>'+esc(x.started_at)+'<br>'+esc(x.finished_at||'进行中')+'</td><td>'+esc(formatResult(x.result)||x.error||'-')+(x.error?'<details><summary>错误详情</summary>'+esc(x.error)+'</details>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无任务。启动服务后会自动记录 startup-reconcile。</div>';
let cfg=await api('/api/settings');settingsBox.innerHTML='<div>音乐目录：<span class="path">'+esc(cfg.root)+'</span></div><div>数据库：<span class="path">'+esc(cfg.db)+'</span></div><div>认证：'+esc(cfg.auth)+'</div><div>L2：相似度 ≥ '+esc(cfg.l2_min)+' 自动判定，'+esc(cfg.l2_review)+' 起进入审核；时长窗口 ±'+esc(cfg.duration_delta)+' 秒</div>'; document.getElementById('historyBox').innerHTML=h.items.length?'<table><tr><th>ID</th><th>时间</th><th>操作</th><th>文件</th><th>目标</th><th></th></tr>'+h.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.created_at)+'</td><td>'+esc(x.action)+'</td><td class="path">'+esc(x.file_path)+'</td><td class="path">'+esc(x.winner_path||'')+'</td><td>'+(x.reversible?'<button onclick="rollback('+x.id+')">回滚</button>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无去重操作记录。</div>';
}catch(e){alert(e.message)}} document.getElementById('settingsBox').innerHTML='<div>音乐目录：<span class="path">'+esc(cfg.root)+'</span></div><div>数据库：<span class="path">'+esc(cfg.db)+'</span></div><div>认证：'+esc(cfg.auth)+'</div><div>自动精确去重：<b>已启用</b>（watch 启动时会先扫描一次，之后监听新文件）</div><div>L2：相似度 ≥ '+esc(cfg.l2_min)+' 自动判定，'+esc(cfg.l2_review)+' 起进入审核；时长窗口 ±'+esc(cfg.duration_delta)+' 秒</div>';
load();setInterval(load,5000); renderStatus(j);if(manual)toast('已刷新');
}catch(e){renderStatus(null);toast(e.message,true)}}
async function run(kind){if(busy)return;if(kind==='dedupe'&&!confirm('确认执行安全去重？\n\nSHA-256 完全相同的文件会使用 Hard Link 合并物理存储；逻辑路径会保留。'))return;busy=true;setButtons(true);try{const r=await api('/api/jobs',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({kind})});toast((labels[kind]||kind)+' 已启动，结果会显示在“任务执行记录”中。');await load();const timer=setInterval(async()=>{try{const j=await api('/api/jobs');renderStatus(j);if(!j.items.length||j.items[0].id!==r.job_id||j.items[0].status==='running')return;clearInterval(timer);busy=false;setButtons(false);toast((labels[kind]||kind)+'完成：'+(formatResult(j.items[0].result)||j.items[0].error||j.items[0].status),j.items[0].status==='failed');await load()}catch(e){clearInterval(timer);busy=false;setButtons(false);toast(e.message,true)}},1000)}catch(e){busy=false;setButtons(false);toast(e.message,true)}}
async function reviewAction(id,action,candidate_id){try{await api('/api/review',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id,action,candidate_id})});toast(action==='approve'?'已确认并建立 Hard Link':'已保留原文件');await load()}catch(e){toast(e.message,true)}}
async function rollback(id){if(!confirm('确认回滚此 Hard Link？回滚会恢复 loser 的独立文件内容，并保留两个逻辑路径。'))return;try{await api('/api/rollback',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id})});toast('回滚完成');await load()}catch(e){toast(e.message,true)}}
load();setInterval(()=>load(),5000);
</script></html>''' </script></html>'''
def stats(): def stats():
with connect(DB) as c: with connect(DB) as c:
total = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"] total = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
@@ -62,7 +79,13 @@ def stats():
review = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='review'").fetchone()["n"] review = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='review'").fetchone()["n"]
size = c.execute("SELECT COALESCE(SUM(file_size),0) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"] size = c.execute("SELECT COALESCE(SUM(file_size),0) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
ops = c.execute("SELECT COUNT(*) n FROM operations").fetchone()["n"] ops = c.execute("SELECT COUNT(*) n FROM operations").fetchone()["n"]
return {"cards":[{"name":"音乐文件","value":total},{"name":"Hard Link","value":hard},{"name":"待人工审核","value":review},{"name":"索引大小","value":f"{size/1024/1024/1024:.2f} GB"},{"name":"操作记录","value":ops}]} return {"cards":[
{"name":"逻辑音乐文件","value":total,"hint":"每个用户路径各算 1 个"},
{"name":"Hard Link 文件","value":hard,"hint":"物理数据已共享"},
{"name":"待人工审核","value":review,"hint":"L2 相似或版本冲突"},
{"name":"索引逻辑大小","value":f"{size/1024/1024/1024:.2f} GB","hint":"按路径统计，非实际占用"},
{"name":"操作记录","value":ops,"hint":"可追溯/可回滚"}]}
def duplicates(): def duplicates():
with connect(DB) as c: with connect(DB) as c:
@@ -78,6 +101,7 @@ def duplicates():
out.append({"type":"SHA256 完全重复", "savings":savings, "winner_id":winner["id"], "items":ordered}) out.append({"type":"SHA256 完全重复", "savings":savings, "winner_id":winner["id"], "items":ordered})
return {"groups":out[:100]} return {"groups":out[:100]}
def review_items(): def review_items():
with connect(DB) as c: rows = c.execute("SELECT * FROM audio_tracks WHERE status='review' ORDER BY updated_at DESC").fetchall() with connect(DB) as c: rows = c.execute("SELECT * FROM audio_tracks WHERE status='review' ORDER BY updated_at DESC").fetchall()
out = [] out = []
@@ -88,36 +112,41 @@ def review_items():
d["candidate_id"] = cand["id"] if cand else None; d["candidate_path"] = cand["file_path"] if cand else None; out.append(d) d["candidate_id"] = cand["id"] if cand else None; d["candidate_path"] = cand["file_path"] if cand else None; out.append(d)
return {"items":out} return {"items":out}
def history(): def history():
with connect(DB) as c: rows = c.execute("SELECT id,created_at,action,file_path,winner_path,details FROM operations ORDER BY id DESC LIMIT 100").fetchall() with connect(DB) as c: rows = c.execute("SELECT id,created_at,action,file_path,winner_path,details FROM operations ORDER BY id DESC LIMIT 100").fetchall()
return {"items":[dict(r, reversible=r["action"] in ("exact-hardlink","acoustic-hardlink","manual-hardlink")) for r in rows]} return {"items":[dict(r, reversible=r["action"] in ("exact-hardlink","acoustic-hardlink","manual-hardlink")) for r in rows]}
def jobs(): def jobs():
with connect(DB) as c: rows = c.execute("SELECT * FROM jobs ORDER BY id DESC LIMIT 30").fetchall() with connect(DB) as c: rows = c.execute("SELECT * FROM jobs ORDER BY id DESC LIMIT 30").fetchall()
return {"items":[dict(r) for r in rows]} return {"items":[dict(r) for r in rows]}
def settings(): def settings():
from .l2 import MIN_SIMILARITY, REVIEW_SIMILARITY, DURATION_DELTA from .l2 import MIN_SIMILARITY, REVIEW_SIMILARITY, DURATION_DELTA
return {"root":str(ROOT),"db":str(DB),"auth":"Token 已启用" if TOKEN else "未启用（仅建议内网）", return {"root":str(ROOT),"db":str(DB),"auth":"Token 已启用" if TOKEN else "未启用（仅建议内网）","l2_min":MIN_SIMILARITY,"l2_review":REVIEW_SIMILARITY,"duration_delta":DURATION_DELTA}
"l2_min":MIN_SIMILARITY,"l2_review":REVIEW_SIMILARITY,"duration_delta":DURATION_DELTA}
def run_job(kind): def run_job(kind):
with connect(DB) as c: with connect(DB) as c:
cur=c.execute("INSERT INTO jobs(kind,status) VALUES(?,?)",(kind,"running")); jid=cur.lastrowid jid = c.execute("INSERT INTO jobs(kind,status) VALUES(?,?)", (kind,"running")).lastrowid
def work(): def work():
try: try:
with LOCK: with LOCK:
if kind=="scan": result={"count":scan(ROOT,DB)} indexed = scan(ROOT, DB)
if kind == "scan": result = {"indexed": indexed}
elif kind == "dry": elif kind == "dry":
scan(ROOT,DB); result={"exact":len(exact_dedupe(DB,ROOT,True)),"acoustic":acoustic_dedupe(DB,ROOT,True)} result = {"indexed": indexed, "exact": len(exact_dedupe(DB, ROOT, True)), "acoustic": acoustic_dedupe(DB, ROOT, True)}
else: else:
scan(ROOT,DB); result={"exact":len(exact_dedupe(DB,ROOT)),"acoustic":acoustic_dedupe(DB,ROOT)} result = {"indexed": indexed, "exact": len(exact_dedupe(DB, ROOT)), "acoustic": acoustic_dedupe(DB, ROOT)}
with connect(DB) as c: c.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?", (json.dumps(result,ensure_ascii=False),jid)) with connect(DB) as c: c.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?", (json.dumps(result,ensure_ascii=False),jid))
except Exception as e: except Exception as e:
with connect(DB) as c: c.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?", (str(e),jid)) with connect(DB) as c: c.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?", (str(e),jid))
threading.Thread(target=work,daemon=True).start() threading.Thread(target=work,daemon=True).start()
return jid return jid
def rollback(opid): def rollback(opid):
with connect(DB) as c: row = c.execute("SELECT * FROM operations WHERE id=?", (opid,)).fetchone() with connect(DB) as c: row = c.execute("SELECT * FROM operations WHERE id=?", (opid,)).fetchone()
if not row or row["action"] not in ("exact-hardlink","acoustic-hardlink","manual-hardlink"): raise ValueError("operation not reversible") if not row or row["action"] not in ("exact-hardlink","acoustic-hardlink","manual-hardlink"): raise ValueError("operation not reversible")
@@ -129,63 +158,50 @@ def rollback(opid):
c.execute("UPDATE audio_tracks SET status='active',review_status='rolled-back',updated_at=CURRENT_TIMESTAMP WHERE file_path=?", (str(loser),)) c.execute("UPDATE audio_tracks SET status='active',review_status='rolled-back',updated_at=CURRENT_TIMESTAMP WHERE file_path=?", (str(loser),))
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ("rollback",str(loser),str(winner),f"rollback of operation {opid}")) c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ("rollback",str(loser),str(winner),f"rollback of operation {opid}"))
def approve(rid,cid): def approve(rid,cid):
with connect(DB) as c: with connect(DB) as c:
r=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status='review'",(rid,)).fetchone() r=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status='review'",(rid,)).fetchone(); w=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status IN ('active','hardlinked')",(cid,)).fetchone()
w=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status IN ('active','hardlinked')",(cid,)).fetchone()
if not r or not w: raise ValueError("review item or candidate not found") if not r or not w: raise ValueError("review item or candidate not found")
lp,wp=Path(r["file_path"]),Path(w["file_path"]) lp,wp=Path(r["file_path"]),Path(w["file_path"])
if not lp.exists() or not wp.exists() or not lp.is_relative_to(ROOT) or not wp.is_relative_to(ROOT): raise ValueError("invalid library path") if not lp.exists() or not wp.exists() or not lp.is_relative_to(ROOT) or not wp.is_relative_to(ROOT): raise ValueError("invalid library path")
if lp.stat().st_dev != wp.stat().st_dev: raise ValueError("different filesystem") if lp.stat().st_dev != wp.stat().st_dev: raise ValueError("different filesystem")
if lp.stat().st_ino!=wp.stat().st_ino: if lp.stat().st_ino != wp.stat().st_ino: hardlink_duplicate(lp,wp)
hardlink_duplicate(lp,wp)
with connect(DB) as c: with connect(DB) as c:
c.execute("UPDATE audio_tracks SET status='hardlinked',review_status='manual-approved',updated_at=CURRENT_TIMESTAMP WHERE id=?",(rid,)) c.execute("UPDATE audio_tracks SET status='hardlinked',review_status='manual-approved',updated_at=CURRENT_TIMESTAMP WHERE id=?",(rid,))
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)",("manual-hardlink",str(lp),str(wp),"manual review approval")) c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)",("manual-hardlink",str(lp),str(wp),"manual review approval"))
class Handler(BaseHTTPRequestHandler): class Handler(BaseHTTPRequestHandler):
def send(self,status,data,ctype="application/json"): def send(self,status,data,ctype="application/json"):
raw=data.encode() if isinstance(data,str) else json.dumps(data,ensure_ascii=False).encode() raw=data.encode() if isinstance(data,str) else json.dumps(data,ensure_ascii=False).encode(); self.send_response(status); self.send_header("Content-Type",ctype+"; charset=utf-8"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw)
self.send_response(status); self.send_header("Content-Type",ctype+"; charset=utf-8"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw) def authorized(self): return not TOKEN or self.headers.get("Authorization","") == "Bearer "+TOKEN
def authorized(self):
if not TOKEN: return True
return self.headers.get("Authorization","") == "Bearer "+TOKEN
def do_GET(self): def do_GET(self):
p=urlparse(self.path).path p=urlparse(self.path).path
try: try:
if p=="/": return self.send(200,HTML,"text/html") if p=="/": return self.send(200,HTML,"text/html")
if not self.authorized(): return self.send(401,{"error":"unauthorized"}) if not self.authorized(): return self.send(401,{"error":"unauthorized"})
if p=="/api/stats": return self.send(200,stats()) routes={"/api/stats":stats,"/api/duplicates":duplicates,"/api/review":review_items,"/api/history":history,"/api/jobs":jobs,"/api/settings":settings}
if p=="/api/duplicates": return self.send(200,duplicates()) if p in routes: return self.send(200,routes[p]())
if p=="/api/review": return self.send(200,review_items())
if p=="/api/history": return self.send(200,history())
if p=="/api/jobs": return self.send(200,jobs())
if p=="/api/settings": return self.send(200,settings())
return self.send(404,{"error":"not found"}) return self.send(404,{"error":"not found"})
except Exception as e: return self.send(500,{"error":str(e)}) except Exception as e: return self.send(500,{"error":str(e)})
def do_POST(self): def do_POST(self):
p=urlparse(self.path).path p=urlparse(self.path).path
if not self.authorized(): return self.send(401,{"error":"unauthorized"}) if not self.authorized(): return self.send(401,{"error":"unauthorized"})
try: body=json.loads(self.rfile.read(int(self.headers.get("content-length","0"))) or b"{}") try: body=json.loads(self.rfile.read(int(self.headers.get("content-length","0"))) or b"{}")
except Exception: return self.send(400,{"error":"invalid json"}) except Exception: return self.send(400,{"error":"invalid json"})
try: try:
if p=="/api/jobs": if p=="/api/jobs" and body.get("kind") in {"scan","dry","dedupe"}: return self.send(202,{"ok":True,"job_id":run_job(body["kind"])})
if body.get("kind") not in {"scan","dry","dedupe"}: return self.send(400,{"error":"invalid job"})
return self.send(202,{"ok":True,"job_id":run_job(body["kind"])})
if p=="/api/review": if p=="/api/review":
if body.get("action")=="reject": if body.get("action")=="reject":
with connect(DB) as c: c.execute("UPDATE audio_tracks SET status='active',review_status='rejected',updated_at=CURRENT_TIMESTAMP WHERE id=? AND status='review'",(int(body["id"]),)) with connect(DB) as c: c.execute("UPDATE audio_tracks SET status='active',review_status='rejected',updated_at=CURRENT_TIMESTAMP WHERE id=? AND status='review'",(int(body["id"]),))
return self.send(200,{"ok":True}) return self.send(200,{"ok":True})
if body.get("action")=="approve": approve(int(body["id"]),int(body["candidate_id"])); return self.send(200,{"ok":True}) if body.get("action")=="approve": approve(int(body["id"]),int(body["candidate_id"])); return self.send(200,{"ok":True})
return self.send(400,{"error":"invalid action"})
if p=="/api/rollback": rollback(int(body["id"])); return self.send(200,{"ok":True}) if p=="/api/rollback": rollback(int(body["id"])); return self.send(200,{"ok":True})
return self.send(404,{"error":"not found"}) return self.send(400,{"error":"invalid request"})
except Exception as e: return self.send(409,{"error":str(e)}) except Exception as e: return self.send(409,{"error":str(e)})
def log_message(self,*args): pass def log_message(self,*args): pass
def serve(host="0.0.0.0",port=8080):
print(f"AM Deduper UI listening on {host}:{port}",flush=True)
ThreadingHTTPServer((host,port),Handler).serve_forever()
def serve(host="0.0.0.0",port=8080):
print(f"AM Deduper UI listening on {host}:{port}",flush=True); ThreadingHTTPServer((host,port),Handler).serve_forever()
pyproject.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[project] [project]
name = "am-music-deduper" name = "am-music-deduper"
version = "0.1.0" version = "0.2.0"
description = "Lightweight NAS music deduplication service for AllMusic" description = "Lightweight NAS music deduplication service for AllMusic"
requires-python = ">=3.10" requires-python = ">=3.10"
dependencies = [ dependencies = [
tests/test_scanner.py
+21
View File
@@ -0,0 +1,21 @@
from pathlib import Path
from am_deduper.db import connect
from am_deduper.scanner import scan
def test_replacing_file_invalidates_content_hashes(tmp_path: Path):
root = tmp_path / "AllMusic"
(root / "admin").mkdir(parents=True)
path = root / "admin" / "song.flac"
db = root / ".dedupe" / "dedupe.db"
path.write_bytes(b"first" * 1000)
scan(root, db)
with connect(db) as c:
c.execute("UPDATE audio_tracks SET sha256='old-sha', chromaprint='old-fp' WHERE file_path=?", (str(path),))
path.write_bytes(b"second" * 1000)
scan(root, db)
with connect(db) as c:
row = c.execute("SELECT sha256,chromaprint FROM audio_tracks WHERE file_path=?", (str(path),)).fetchone()
assert row["sha256"] is None
assert row["chromaprint"] is None