feat: automate exact dedupe and improve web feedback

This commit is contained in:
OpenClaw Agent
2026-09-18 07:30:38 +08:00
parent 0fbadf0c9c
commit f10e92a5cd
8 changed files with 283 additions and 163 deletions
+120 -104
View File
@@ -1,191 +1,207 @@
from __future__ import annotations
import json, os, shutil, threading, uuid
import json
import os
import shutil
import threading
import uuid
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import urlparse
from .db import connect
from .scanner import scan
from .exact import exact_dedupe, hardlink_duplicate, choose_winner
from .l2 import acoustic_dedupe
ROOT=Path(os.environ.get("AM_DEDUPER_ROOT","/music/AllMusic")).resolve()
DB=Path(os.environ.get("AM_DEDUPER_DB","/data/dedupe.db")).resolve()
TOKEN=os.environ.get("AM_DEDUPER_TOKEN","").strip()
LOCK=threading.Lock()
ROOT = Path(os.environ.get("AM_DEDUPER_ROOT", "/music/AllMusic")).resolve()
DB = Path(os.environ.get("AM_DEDUPER_DB", "/data/dedupe.db")).resolve()
TOKEN = os.environ.get("AM_DEDUPER_TOKEN", "").strip()
LOCK = threading.Lock()
HTML=r'''<!doctype html><html lang="zh-CN"><meta charset="utf-8">
HTML = r'''<!doctype html><html lang="zh-CN"><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1"><title>AM Deduper</title>
<style>
body{font:14px system-ui;margin:0;background:#f5f6f8;color:#202124}header{background:#111827;color:#fff;padding:18px 24px}
main{max-width:1280px;margin:18px auto;padding:0 14px}.grid{display:grid;grid-template-columns:repeat(5,1fr);gap:10px}
.card,section{background:#fff;border:1px solid #ddd;border-radius:10px;padding:14px;margin-bottom:14px}.num{font-size:24px;font-weight:700}
nav{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:14px}button{padding:8px 12px;border:1px solid #bbb;border-radius:7px;background:#fff;cursor:pointer}
button.primary{background:#111827;color:#fff}button.danger{background:#fee2e2}table{width:100%;border-collapse:collapse}th,td{padding:9px;border-bottom:1px solid #eee;text-align:left;vertical-align:top}
.path{font:12px monospace;word-break:break-all}.muted{color:#666}.badge{background:#eee;border-radius:10px;padding:3px 7px}
@media(max-width:800px){.grid{grid-template-columns:repeat(2,1fr)}table{font-size:12px}}
:root{font:14px system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#172033;background:#f4f6f8}*{box-sizing:border-box}
body{margin:0}header{background:#111827;color:white;padding:18px 24px;position:sticky;top:0;z-index:5;box-shadow:0 2px 10px #0002}
header .sub{opacity:.7;font-size:12px;margin-top:4px}.wrap{max-width:1400px;margin:18px auto;padding:0 14px}.actions{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px}
button{border:1px solid #cbd0d8;border-radius:8px;padding:9px 13px;background:white;color:#172033;cursor:pointer}button:hover{background:#f0f2f5}button:disabled{opacity:.5;cursor:wait}
button.primary{background:#111827;color:#fff;border-color:#111827}button.warn{background:#fff7ed;border-color:#fed7aa}button.danger{background:#fef2f2;border-color:#fecaca}
#toast{position:fixed;right:18px;bottom:18px;z-index:20;max-width:420px;padding:12px 16px;border-radius:10px;background:#111827;color:#fff;box-shadow:0 6px 24px #0003;display:none}
#toast.error{background:#991b1b}.statusbar{background:#fff;border:1px solid #dfe3e8;border-radius:10px;padding:12px 14px;margin-bottom:14px;display:flex;justify-content:space-between;gap:12px;align-items:center}.status-main{font-weight:600}.status-detail{font-size:12px;color:#667085;margin-top:3px}
.grid{display:grid;grid-template-columns:repeat(5,1fr);gap:10px}.card,section{background:#fff;border:1px solid #dfe3e8;border-radius:12px;padding:15px;margin-bottom:14px}.card .label{color:#667085}.num{font-size:26px;font-weight:750;margin-top:4px}.hint{font-size:12px;color:#667085;margin-top:3px}
h2{font-size:17px;margin:0 0 12px}table{width:100%;border-collapse:collapse}th,td{padding:9px;border-bottom:1px solid #edf0f2;text-align:left;vertical-align:top}th{color:#667085;font-weight:600;font-size:12px}.path{font:12px ui-monospace,SFMono-Regular,Menlo,monospace;word-break:break-all}.muted{color:#667085}.badge{display:inline-block;padding:3px 7px;border-radius:999px;background:#eef2f6;font-size:12px}.success{color:#166534}.error{color:#991b1b}.running{color:#1d4ed8}.empty{padding:22px;text-align:center;color:#667085;background:#fafbfc;border-radius:8px}
.jobline{display:flex;align-items:center;gap:8px}.spinner{width:12px;height:12px;border:2px solid #cbd5e1;border-top-color:#2563eb;border-radius:50%;animation:spin .8s linear infinite}@keyframes spin{to{transform:rotate(360deg)}}
details{margin-top:8px}summary{cursor:pointer;color:#475467}.toolbar{display:flex;justify-content:space-between;gap:8px;align-items:center;margin-bottom:8px}.small{font-size:12px}
@media(max-width:900px){.grid{grid-template-columns:repeat(2,1fr)}.statusbar{align-items:flex-start;flex-direction:column}table{display:block;overflow:auto;white-space:nowrap}}
</style>
<header><b>AM Deduper</b> · NAS Music Library</header><main>
<nav><button class="primary" onclick="run('scan')">扫描索引</button><button onclick="run('dry')">分析 / Dry-run</button>
<button onclick="run('dedupe')">执行去重</button><button onclick="load()">刷新</button></nav>
<header><div><b>AM Deduper</b> · NAS Music Library</div><div class="sub">自动监听 · SHA-256 精确去重 · Hard Link · 人工审核</div></header>
<div class="wrap">
<div class="actions"><button class="primary" data-action="scan" onclick="run('scan')">扫描并更新索引</button><button data-action="dry" onclick="run('dry')">分析重复(Dry-run)</button><button class="warn" data-action="dedupe" onclick="run('dedupe')">执行安全去重</button><button onclick="load(true)">刷新</button></div>
<div id="status" class="statusbar"><div><div class="status-main">正在加载…</div><div class="status-detail">读取任务和索引状态</div></div><span class="badge">自动刷新 5 秒</span></div>
<div class="grid" id="stats"></div>
<section><h2>重复组</h2><div id="dupes"></div></section>
<section><h2>待人工审核</h2><div id="review"></div></section>
<section><h2>任务</h2><div id="jobsBox"></div></section>
<section><h2>最近操作</h2><div id="historyBox"></div></section>
<section><h2>设置</h2><div id="settingsBox" class="muted">加载中...</div></section>
</main>
<section><div class="toolbar"><h2>重复组</h2><span class="small muted" id="dupeCount"></span></div><div id="dupes"></div></section>
<section><div class="toolbar"><h2>待人工审核</h2><span class="small muted" id="reviewCount"></span></div><div id="review"></div></section>
<section><div class="toolbar"><h2>任务执行记录</h2><span class="small muted">包含自动去重任务</span></div><div id="jobsBox"></div></section>
<section><div class="toolbar"><h2>最近操作</h2><span class="small muted">可回滚仍保持同 inode 的 Hard Link</span></div><div id="historyBox"></div></section>
<section><h2>运行设置</h2><div id="settingsBox" class="muted">加载中...</div></section>
</div><div id="toast"></div>
<script>
const esc=s=>String(s??'').replace(/[&<>"]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c]));
async function api(u,o={}){o.headers=Object.assign({},o.headers||{});let t=localStorage.getItem('am_token');if(t)o.headers.Authorization='Bearer '+t;let r=await fetch(u,o);let j=await r.json();if(r.status===401){t=prompt('请输入 AM Deduper 管理 Token');if(t){localStorage.setItem('am_token',t);return api(u,o)} }if(!r.ok)throw Error(j.error||r.status);return j}
async function load(){try{
let s=await api('/api/stats');stats.innerHTML=s.cards.map(x=>'<div class="card"><div class="muted">'+esc(x.name)+'</div><div class="num">'+esc(x.value)+'</div></div>').join('');
let d=await api('/api/duplicates');dupes.innerHTML=d.groups.length?d.groups.map(g=>'<div class="card"><b>'+esc(g.type)+' · '+esc(g.savings)+' MB 可节省</b><table><tr><th>文件</th><th>状态</th></tr>'+g.items.map(x=>'<tr><td class="path">'+esc(x.file_path)+'</td><td>'+esc(x.status)+'</td></tr>').join('')+'</table></div>').join(''):'<p class="muted">暂无重复组。</p>';
let q=await api('/api/review');review.innerHTML=q.items.length?'<table><tr><th>相似度</th><th>文件</th><th>候选目标</th><th>操作</th></tr>'+q.items.map(x=>'<tr><td>'+esc(x.similarity||'-')+'</td><td class="path">'+esc(x.file_path)+'<br>'+esc(x.title||'')+' '+esc(x.artist||'')+'</td><td class="path">'+esc(x.candidate_path||'-')+'</td><td><button onclick="reviewAction('+x.id+',\'reject\')">保留</button> '+(x.candidate_id?'<button class="primary" onclick="reviewAction('+x.id+',\'approve\','+x.candidate_id+')">确认重复</button>':'')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无待审核项目。</p>';
let j=await api('/api/jobs');jobsBox.innerHTML=j.items.length?'<table><tr><th>ID</th><th>任务</th><th>状态</th><th>开始</th><th>结束</th></tr>'+j.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.kind)+'</td><td>'+esc(x.status)+'</td><td>'+esc(x.started_at)+'</td><td>'+esc(x.finished_at||'-')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无任务。</p>';
let h=await api('/api/history');historyBox.innerHTML=h.items.length?'<table><tr><th>ID</th><th>时间</th><th>操作</th><th>文件</th><th>目标</th><th></th></tr>'+h.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.created_at)+'</td><td>'+esc(x.action)+'</td><td class="path">'+esc(x.file_path)+'</td><td class="path">'+esc(x.winner_path||'')+'</td><td>'+(x.reversible?'<button onclick="rollback('+x.id+')">回滚</button>':'')+'</td></tr>').join('')+'</table>':'<p class="muted">暂无记录。</p>';
}catch(e){alert(e.message)}}
async function run(kind){try{await api('/api/jobs',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({kind})});load()}catch(e){alert(e.message)}}
async function reviewAction(id,action,candidate_id){try{await api('/api/review',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id,action,candidate_id})});load()}catch(e){alert(e.message)}}
async function rollback(id){if(!confirm('确认回滚此 Hard Link?'))return;try{await api('/api/rollback',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id})});load()}catch(e){alert(e.message)}}
let cfg=await api('/api/settings');settingsBox.innerHTML='<div>音乐目录:<span class="path">'+esc(cfg.root)+'</span></div><div>数据库:<span class="path">'+esc(cfg.db)+'</span></div><div>认证:'+esc(cfg.auth)+'</div><div>L2:相似度 ≥ '+esc(cfg.l2_min)+' 自动判定,'+esc(cfg.l2_review)+' 起进入审核;时长窗口 ±'+esc(cfg.duration_delta)+' 秒</div>';
}catch(e){alert(e.message)}}
load();setInterval(load,5000);
const esc=s=>String(s??'').replace(/[&<>\"]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c]));
const labels={scan:'扫描索引',dry:'Dry-run 分析',dedupe:'执行去重'};
let busy=false;
function toast(msg,error=false){const e=document.getElementById('toast');e.textContent=msg;e.className=error?'error':'';e.style.display='block';clearTimeout(window._toast);window._toast=setTimeout(()=>e.style.display='none',4500)}
async function api(u,o={}){o.headers=Object.assign({},o.headers||{});const t=localStorage.getItem('am_token');if(t)o.headers.Authorization='Bearer '+t;let r=await fetch(u,o);let j={};try{j=await r.json()}catch{}if(r.status===401){const nt=prompt('请输入 AM Deduper 管理 Token');if(nt){localStorage.setItem('am_token',nt);return api(u,o)}throw Error('需要管理 Token')}if(!r.ok)throw Error(j.error||('HTTP '+r.status));return j}
function setButtons(v){document.querySelectorAll('[data-action]').forEach(b=>b.disabled=v)}
function renderStatus(j){const e=document.getElementById('status');if(!j||!j.items?.length){e.innerHTML='<div><div class="status-main">系统空闲</div><div class="status-detail">监听服务会在新音乐稳定写入后自动进行 SHA-256 精确去重。</div></div><span class="badge">运行正常</span>';return}const x=j.items[0],running=x.status==='running';let detail=x.result||x.error||'等待结果';e.innerHTML='<div><div class="status-main '+(running?'running':x.status==='failed'?'error':'')+'">'+(running?'<span class="jobline"><span class="spinner"></span>':'')+esc(labels[x.kind]||x.kind)+(running?'…</span>':'')+'</div><div class="status-detail">'+esc(detail)+'</div></div><span class="badge">'+esc(x.status)+'</span>';}
function formatResult(r){if(!r)return '';try{const x=JSON.parse(r);const parts=[];if(x.indexed!=null)parts.push('索引 '+x.indexed);if(x.exact_hardlinked!=null)parts.push('精确去重 '+x.exact_hardlinked+' 个');if(x.exact!=null)parts.push('Exact '+x.exact+' 个');if(x.acoustic)parts.push('L2 '+(x.acoustic.equivalent??0)+' 个');return parts.join(' · ')||r}catch{return r}}
async function load(manual=false){try{
const [s,d,q,j,h,cfg]=await Promise.all([api('/api/stats'),api('/api/duplicates'),api('/api/review'),api('/api/jobs'),api('/api/history'),api('/api/settings')]);
document.getElementById('stats').innerHTML=s.cards.map(x=>'<div class="card"><div class="label">'+esc(x.name)+'</div><div class="num">'+esc(x.value)+'</div><div class="hint">'+esc(x.hint||'')+'</div></div>').join('');
document.getElementById('dupeCount').textContent=d.groups.length+' 组';document.getElementById('dupes').innerHTML=d.groups.length?d.groups.map(g=>'<div class="card"><b>'+esc(g.type)+'</b> · 可节省约 '+esc(g.savings)+' MB · '+esc(g.items.length)+' 个逻辑路径<table><tr><th>路径</th><th>状态</th><th>inode</th></tr>'+g.items.map(x=>'<tr><td class="path">'+esc(x.file_path)+'</td><td><span class="badge">'+esc(x.status)+'</span>'+(x.id===g.winner_id?' <span class="badge">保留路径</span>':'')+'</td><td>'+esc(x.inode)+'</td></tr>').join('')+'</table></div>').join(''):'<div class="empty">没有发现 SHA-256 完全重复。新文件会由监听服务自动检查。</div>';
document.getElementById('reviewCount').textContent=q.items.length+' 项';document.getElementById('review').innerHTML=q.items.length?'<table><tr><th>相似度</th><th>文件</th><th>候选目标</th><th>操作</th></tr>'+q.items.map(x=>'<tr><td>'+esc(x.similarity||'-')+'</td><td class="path">'+esc(x.file_path)+'<br>'+esc(x.title||'')+' '+esc(x.artist||'')+'</td><td class="path">'+esc(x.candidate_path||'-')+'</td><td><button onclick="reviewAction('+x.id+',\'reject\')">保留原文件</button> '+(x.candidate_id?'<button class="primary" onclick="reviewAction('+x.id+',\'approve\','+x.candidate_id+')">确认硬链接</button>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无人工审核项目。</div>';
document.getElementById('jobsBox').innerHTML=j.items.length?'<table><tr><th>ID</th><th>任务</th><th>状态</th><th>时间</th><th>结果</th></tr>'+j.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(labels[x.kind]||x.kind)+'</td><td class="'+(x.status==='failed'?'error':x.status==='running'?'running':'success')+'">'+esc(x.status)+'</td><td>'+esc(x.started_at)+'<br>'+esc(x.finished_at||'进行中')+'</td><td>'+esc(formatResult(x.result)||x.error||'-')+(x.error?'<details><summary>错误详情</summary>'+esc(x.error)+'</details>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无任务。启动服务后会自动记录 startup-reconcile。</div>';
document.getElementById('historyBox').innerHTML=h.items.length?'<table><tr><th>ID</th><th>时间</th><th>操作</th><th>文件</th><th>目标</th><th></th></tr>'+h.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.created_at)+'</td><td>'+esc(x.action)+'</td><td class="path">'+esc(x.file_path)+'</td><td class="path">'+esc(x.winner_path||'')+'</td><td>'+(x.reversible?'<button onclick="rollback('+x.id+')">回滚</button>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无去重操作记录。</div>';
document.getElementById('settingsBox').innerHTML='<div>音乐目录:<span class="path">'+esc(cfg.root)+'</span></div><div>数据库:<span class="path">'+esc(cfg.db)+'</span></div><div>认证:'+esc(cfg.auth)+'</div><div>自动精确去重:<b>已启用</b>(watch 启动时会先扫描一次,之后监听新文件)</div><div>L2:相似度 ≥ '+esc(cfg.l2_min)+' 自动判定,'+esc(cfg.l2_review)+' 起进入审核;时长窗口 ±'+esc(cfg.duration_delta)+' 秒</div>';
renderStatus(j);if(manual)toast('已刷新');
}catch(e){renderStatus(null);toast(e.message,true)}}
async function run(kind){if(busy)return;if(kind==='dedupe'&&!confirm('确认执行安全去重?\n\nSHA-256 完全相同的文件会使用 Hard Link 合并物理存储;逻辑路径会保留。'))return;busy=true;setButtons(true);try{const r=await api('/api/jobs',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({kind})});toast((labels[kind]||kind)+' 已启动,结果会显示在“任务执行记录”中。');await load();const timer=setInterval(async()=>{try{const j=await api('/api/jobs');renderStatus(j);if(!j.items.length||j.items[0].id!==r.job_id||j.items[0].status==='running')return;clearInterval(timer);busy=false;setButtons(false);toast((labels[kind]||kind)+'完成:'+(formatResult(j.items[0].result)||j.items[0].error||j.items[0].status),j.items[0].status==='failed');await load()}catch(e){clearInterval(timer);busy=false;setButtons(false);toast(e.message,true)}},1000)}catch(e){busy=false;setButtons(false);toast(e.message,true)}}
async function reviewAction(id,action,candidate_id){try{await api('/api/review',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id,action,candidate_id})});toast(action==='approve'?'已确认并建立 Hard Link':'已保留原文件');await load()}catch(e){toast(e.message,true)}}
async function rollback(id){if(!confirm('确认回滚此 Hard Link?回滚会恢复 loser 的独立文件内容,并保留两个逻辑路径。'))return;try{await api('/api/rollback',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id})});toast('回滚完成');await load()}catch(e){toast(e.message,true)}}
load();setInterval(()=>load(),5000);
</script></html>'''
def stats():
with connect(DB) as c:
total=c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
hard=c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='hardlinked'").fetchone()["n"]
review=c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='review'").fetchone()["n"]
size=c.execute("SELECT COALESCE(SUM(file_size),0) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
ops=c.execute("SELECT COUNT(*) n FROM operations").fetchone()["n"]
return {"cards":[{"name":"音乐文件","value":total},{"name":"Hard Link","value":hard},{"name":"待人工审核","value":review},{"name":"索引大小","value":f"{size/1024/1024/1024:.2f} GB"},{"name":"操作记录","value":ops}]}
total = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
hard = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='hardlinked'").fetchone()["n"]
review = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='review'").fetchone()["n"]
size = c.execute("SELECT COALESCE(SUM(file_size),0) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
ops = c.execute("SELECT COUNT(*) n FROM operations").fetchone()["n"]
return {"cards":[
{"name":"逻辑音乐文件","value":total,"hint":"每个用户路径各算 1 个"},
{"name":"Hard Link 文件","value":hard,"hint":"物理数据已共享"},
{"name":"待人工审核","value":review,"hint":"L2 相似或版本冲突"},
{"name":"索引逻辑大小","value":f"{size/1024/1024/1024:.2f} GB","hint":"按路径统计,非实际占用"},
{"name":"操作记录","value":ops,"hint":"可追溯/可回滚"}]}
def duplicates():
with connect(DB) as c:
rows=c.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked') AND sha256 IS NOT NULL ORDER BY sha256,id").fetchall()
groups={}
for r in rows: groups.setdefault(r["sha256"],[]).append(dict(r))
out=[]
for _,items in groups.items():
if len(items)>1:
winner=choose_winner(items)
ordered=[winner]+[x for x in items if x["id"]!=winner["id"]]
savings=round(sum(x["file_size"] for x in ordered[1:])/1024/1024,2)
out.append({"type":"SHA256 完全重复","savings":savings,"winner_id":winner["id"],"items":ordered})
rows = c.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked') AND sha256 IS NOT NULL ORDER BY sha256,id").fetchall()
groups = {}
for r in rows: groups.setdefault(r["sha256"], []).append(dict(r))
out = []
for _, items in groups.items():
if len(items) > 1:
winner = choose_winner(items)
ordered = [winner] + [x for x in items if x["id"] != winner["id"]]
savings = round(sum(x["file_size"] for x in ordered[1:]) / 1024 / 1024, 2)
out.append({"type":"SHA256 完全重复", "savings":savings, "winner_id":winner["id"], "items":ordered})
return {"groups":out[:100]}
def review_items():
with connect(DB) as c: rows=c.execute("SELECT * FROM audio_tracks WHERE status='review' ORDER BY updated_at DESC").fetchall()
out=[]
with connect(DB) as c: rows = c.execute("SELECT * FROM audio_tracks WHERE status='review' ORDER BY updated_at DESC").fetchall()
out = []
for r in rows:
d=dict(r); d["similarity"]=(r["review_status"] or "").replace("similarity=","")
d = dict(r); d["similarity"] = (r["review_status"] or "").replace("similarity=", "")
with connect(DB) as c2:
cand=c2.execute("SELECT id,file_path FROM audio_tracks WHERE id!=? AND status IN ('active','hardlinked') AND duration BETWEEN ? AND ? ORDER BY ABS(duration-?) LIMIT 1",(r["id"],(r["duration"] or 0)-3,(r["duration"] or 0)+3,r["duration"] or 0)).fetchone()
d["candidate_id"]=cand["id"] if cand else None; d["candidate_path"]=cand["file_path"] if cand else None; out.append(d)
cand = c2.execute("SELECT id,file_path FROM audio_tracks WHERE id!=? AND status IN ('active','hardlinked') AND duration BETWEEN ? AND ? ORDER BY ABS(duration-?) LIMIT 1", (r["id"], (r["duration"] or 0)-3, (r["duration"] or 0)+3, r["duration"] or 0)).fetchone()
d["candidate_id"] = cand["id"] if cand else None; d["candidate_path"] = cand["file_path"] if cand else None; out.append(d)
return {"items":out}
def history():
with connect(DB) as c: rows=c.execute("SELECT id,created_at,action,file_path,winner_path,details FROM operations ORDER BY id DESC LIMIT 100").fetchall()
with connect(DB) as c: rows = c.execute("SELECT id,created_at,action,file_path,winner_path,details FROM operations ORDER BY id DESC LIMIT 100").fetchall()
return {"items":[dict(r, reversible=r["action"] in ("exact-hardlink","acoustic-hardlink","manual-hardlink")) for r in rows]}
def jobs():
with connect(DB) as c: rows=c.execute("SELECT * FROM jobs ORDER BY id DESC LIMIT 30").fetchall()
with connect(DB) as c: rows = c.execute("SELECT * FROM jobs ORDER BY id DESC LIMIT 30").fetchall()
return {"items":[dict(r) for r in rows]}
def settings():
from .l2 import MIN_SIMILARITY, REVIEW_SIMILARITY, DURATION_DELTA
return {"root":str(ROOT),"db":str(DB),"auth":"Token 已启用" if TOKEN else "未启用(仅建议内网)",
"l2_min":MIN_SIMILARITY,"l2_review":REVIEW_SIMILARITY,"duration_delta":DURATION_DELTA}
return {"root":str(ROOT),"db":str(DB),"auth":"Token 已启用" if TOKEN else "未启用(仅建议内网)","l2_min":MIN_SIMILARITY,"l2_review":REVIEW_SIMILARITY,"duration_delta":DURATION_DELTA}
def run_job(kind):
with connect(DB) as c:
cur=c.execute("INSERT INTO jobs(kind,status) VALUES(?,?)",(kind,"running")); jid=cur.lastrowid
jid = c.execute("INSERT INTO jobs(kind,status) VALUES(?,?)", (kind,"running")).lastrowid
def work():
try:
with LOCK:
if kind=="scan": result={"count":scan(ROOT,DB)}
elif kind=="dry":
scan(ROOT,DB); result={"exact":len(exact_dedupe(DB,ROOT,True)),"acoustic":acoustic_dedupe(DB,ROOT,True)}
indexed = scan(ROOT, DB)
if kind == "scan": result = {"indexed": indexed}
elif kind == "dry":
result = {"indexed": indexed, "exact": len(exact_dedupe(DB, ROOT, True)), "acoustic": acoustic_dedupe(DB, ROOT, True)}
else:
scan(ROOT,DB); result={"exact":len(exact_dedupe(DB,ROOT)),"acoustic":acoustic_dedupe(DB,ROOT)}
with connect(DB) as c: c.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?",(json.dumps(result,ensure_ascii=False),jid))
result = {"indexed": indexed, "exact": len(exact_dedupe(DB, ROOT)), "acoustic": acoustic_dedupe(DB, ROOT)}
with connect(DB) as c: c.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?", (json.dumps(result,ensure_ascii=False),jid))
except Exception as e:
with connect(DB) as c: c.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?",(str(e),jid))
with connect(DB) as c: c.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?", (str(e),jid))
threading.Thread(target=work,daemon=True).start()
return jid
def rollback(opid):
with connect(DB) as c: row=c.execute("SELECT * FROM operations WHERE id=?",(opid,)).fetchone()
with connect(DB) as c: row = c.execute("SELECT * FROM operations WHERE id=?", (opid,)).fetchone()
if not row or row["action"] not in ("exact-hardlink","acoustic-hardlink","manual-hardlink"): raise ValueError("operation not reversible")
loser,winner=Path(row["file_path"]),Path(row["winner_path"])
if not loser.exists() or not winner.exists() or loser.stat().st_ino!=winner.stat().st_ino: raise ValueError("current file state changed; refusing rollback")
tmp=loser.with_name(loser.name+".dedupe-rollback-"+uuid.uuid4().hex)
loser,winner = Path(row["file_path"]),Path(row["winner_path"])
if not loser.exists() or not winner.exists() or loser.stat().st_ino != winner.stat().st_ino: raise ValueError("current file state changed; refusing rollback")
tmp = loser.with_name(loser.name+".dedupe-rollback-"+uuid.uuid4().hex)
shutil.copy2(winner,tmp); os.replace(tmp,loser)
with connect(DB) as c:
c.execute("UPDATE audio_tracks SET status='active',review_status='rolled-back',updated_at=CURRENT_TIMESTAMP WHERE file_path=?",(str(loser),))
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)",("rollback",str(loser),str(winner),f"rollback of operation {opid}"))
c.execute("UPDATE audio_tracks SET status='active',review_status='rolled-back',updated_at=CURRENT_TIMESTAMP WHERE file_path=?", (str(loser),))
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ("rollback",str(loser),str(winner),f"rollback of operation {opid}"))
def approve(rid,cid):
with connect(DB) as c:
r=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status='review'",(rid,)).fetchone()
w=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status IN ('active','hardlinked')",(cid,)).fetchone()
r=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status='review'",(rid,)).fetchone(); w=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status IN ('active','hardlinked')",(cid,)).fetchone()
if not r or not w: raise ValueError("review item or candidate not found")
lp,wp=Path(r["file_path"]),Path(w["file_path"])
if not lp.exists() or not wp.exists() or not lp.is_relative_to(ROOT) or not wp.is_relative_to(ROOT): raise ValueError("invalid library path")
if lp.stat().st_dev!=wp.stat().st_dev: raise ValueError("different filesystem")
if lp.stat().st_ino!=wp.stat().st_ino:
hardlink_duplicate(lp,wp)
if lp.stat().st_dev != wp.stat().st_dev: raise ValueError("different filesystem")
if lp.stat().st_ino != wp.stat().st_ino: hardlink_duplicate(lp,wp)
with connect(DB) as c:
c.execute("UPDATE audio_tracks SET status='hardlinked',review_status='manual-approved',updated_at=CURRENT_TIMESTAMP WHERE id=?",(rid,))
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)",("manual-hardlink",str(lp),str(wp),"manual review approval"))
class Handler(BaseHTTPRequestHandler):
def send(self,status,data,ctype="application/json"):
raw=data.encode() if isinstance(data,str) else json.dumps(data,ensure_ascii=False).encode()
self.send_response(status); self.send_header("Content-Type",ctype+"; charset=utf-8"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw)
def authorized(self):
if not TOKEN: return True
return self.headers.get("Authorization","") == "Bearer "+TOKEN
raw=data.encode() if isinstance(data,str) else json.dumps(data,ensure_ascii=False).encode(); self.send_response(status); self.send_header("Content-Type",ctype+"; charset=utf-8"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw)
def authorized(self): return not TOKEN or self.headers.get("Authorization","") == "Bearer "+TOKEN
def do_GET(self):
p=urlparse(self.path).path
try:
if p=="/": return self.send(200,HTML,"text/html")
if not self.authorized(): return self.send(401,{"error":"unauthorized"})
if p=="/api/stats": return self.send(200,stats())
if p=="/api/duplicates": return self.send(200,duplicates())
if p=="/api/review": return self.send(200,review_items())
if p=="/api/history": return self.send(200,history())
if p=="/api/jobs": return self.send(200,jobs())
if p=="/api/settings": return self.send(200,settings())
routes={"/api/stats":stats,"/api/duplicates":duplicates,"/api/review":review_items,"/api/history":history,"/api/jobs":jobs,"/api/settings":settings}
if p in routes: return self.send(200,routes[p]())
return self.send(404,{"error":"not found"})
except Exception as e: return self.send(500,{"error":str(e)})
def do_POST(self):
p=urlparse(self.path).path
if not self.authorized(): return self.send(401,{"error":"unauthorized"})
try: body=json.loads(self.rfile.read(int(self.headers.get("content-length","0"))) or b"{}")
except Exception: return self.send(400,{"error":"invalid json"})
try:
if p=="/api/jobs":
if body.get("kind") not in {"scan","dry","dedupe"}: return self.send(400,{"error":"invalid job"})
return self.send(202,{"ok":True,"job_id":run_job(body["kind"])})
if p=="/api/jobs" and body.get("kind") in {"scan","dry","dedupe"}: return self.send(202,{"ok":True,"job_id":run_job(body["kind"])})
if p=="/api/review":
if body.get("action")=="reject":
with connect(DB) as c: c.execute("UPDATE audio_tracks SET status='active',review_status='rejected',updated_at=CURRENT_TIMESTAMP WHERE id=? AND status='review'",(int(body["id"]),))
return self.send(200,{"ok":True})
if body.get("action")=="approve": approve(int(body["id"]),int(body["candidate_id"])); return self.send(200,{"ok":True})
return self.send(400,{"error":"invalid action"})
if p=="/api/rollback": rollback(int(body["id"])); return self.send(200,{"ok":True})
return self.send(404,{"error":"not found"})
return self.send(400,{"error":"invalid request"})
except Exception as e: return self.send(409,{"error":str(e)})
def log_message(self,*args): pass
def serve(host="0.0.0.0",port=8080):
print(f"AM Deduper UI listening on {host}:{port}",flush=True)
ThreadingHTTPServer((host,port),Handler).serve_forever()
def serve(host="0.0.0.0",port=8080):
print(f"AM Deduper UI listening on {host}:{port}",flush=True); ThreadingHTTPServer((host,port),Handler).serve_forever()