Files
am-deduper/am_deduper/web.py
T

208 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
import json
import os
import shutil
import threading
import uuid
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import urlparse
from .db import connect
from .scanner import scan
from .exact import exact_dedupe, hardlink_duplicate, choose_winner
from .l2 import acoustic_dedupe
ROOT = Path(os.environ.get("AM_DEDUPER_ROOT", "/music/AllMusic")).resolve()
DB = Path(os.environ.get("AM_DEDUPER_DB", "/data/dedupe.db")).resolve()
TOKEN = os.environ.get("AM_DEDUPER_TOKEN", "").strip()
LOCK = threading.Lock()
HTML = r'''<!doctype html><html lang="zh-CN"><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1"><title>AM Deduper</title>
<style>
:root{font:14px system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#172033;background:#f4f6f8}*{box-sizing:border-box}
body{margin:0}header{background:#111827;color:white;padding:18px 24px;position:sticky;top:0;z-index:5;box-shadow:0 2px 10px #0002}
header .sub{opacity:.7;font-size:12px;margin-top:4px}.wrap{max-width:1400px;margin:18px auto;padding:0 14px}.actions{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px}
button{border:1px solid #cbd0d8;border-radius:8px;padding:9px 13px;background:white;color:#172033;cursor:pointer}button:hover{background:#f0f2f5}button:disabled{opacity:.5;cursor:wait}
button.primary{background:#111827;color:#fff;border-color:#111827}button.warn{background:#fff7ed;border-color:#fed7aa}button.danger{background:#fef2f2;border-color:#fecaca}
#toast{position:fixed;right:18px;bottom:18px;z-index:20;max-width:420px;padding:12px 16px;border-radius:10px;background:#111827;color:#fff;box-shadow:0 6px 24px #0003;display:none}
#toast.error{background:#991b1b}.statusbar{background:#fff;border:1px solid #dfe3e8;border-radius:10px;padding:12px 14px;margin-bottom:14px;display:flex;justify-content:space-between;gap:12px;align-items:center}.status-main{font-weight:600}.status-detail{font-size:12px;color:#667085;margin-top:3px}
.grid{display:grid;grid-template-columns:repeat(5,1fr);gap:10px}.card,section{background:#fff;border:1px solid #dfe3e8;border-radius:12px;padding:15px;margin-bottom:14px}.card .label{color:#667085}.num{font-size:26px;font-weight:750;margin-top:4px}.hint{font-size:12px;color:#667085;margin-top:3px}
h2{font-size:17px;margin:0 0 12px}table{width:100%;border-collapse:collapse}th,td{padding:9px;border-bottom:1px solid #edf0f2;text-align:left;vertical-align:top}th{color:#667085;font-weight:600;font-size:12px}.path{font:12px ui-monospace,SFMono-Regular,Menlo,monospace;word-break:break-all}.muted{color:#667085}.badge{display:inline-block;padding:3px 7px;border-radius:999px;background:#eef2f6;font-size:12px}.success{color:#166534}.error{color:#991b1b}.running{color:#1d4ed8}.empty{padding:22px;text-align:center;color:#667085;background:#fafbfc;border-radius:8px}
.jobline{display:flex;align-items:center;gap:8px}.spinner{width:12px;height:12px;border:2px solid #cbd5e1;border-top-color:#2563eb;border-radius:50%;animation:spin .8s linear infinite}@keyframes spin{to{transform:rotate(360deg)}}
details{margin-top:8px}summary{cursor:pointer;color:#475467}.toolbar{display:flex;justify-content:space-between;gap:8px;align-items:center;margin-bottom:8px}.small{font-size:12px}
@media(max-width:900px){.grid{grid-template-columns:repeat(2,1fr)}.statusbar{align-items:flex-start;flex-direction:column}table{display:block;overflow:auto;white-space:nowrap}}
</style>
<header><div><b>AM Deduper</b> · NAS Music Library</div><div class="sub">自动监听 · SHA-256 精确去重 · Hard Link · 人工审核</div></header>
<div class="wrap">
<div class="actions"><button class="primary" data-action="scan" onclick="run('scan')">扫描并更新索引</button><button data-action="dry" onclick="run('dry')">分析重复(Dry-run)</button><button class="warn" data-action="dedupe" onclick="run('dedupe')">执行安全去重</button><button onclick="load(true)">刷新</button></div>
<div id="status" class="statusbar"><div><div class="status-main">正在加载…</div><div class="status-detail">读取任务和索引状态</div></div><span class="badge">自动刷新 5 秒</span></div>
<div class="grid" id="stats"></div>
<section><div class="toolbar"><h2>重复组</h2><span class="small muted" id="dupeCount"></span></div><div id="dupes"></div></section>
<section><div class="toolbar"><h2>待人工审核</h2><span class="small muted" id="reviewCount"></span></div><div id="review"></div></section>
<section><div class="toolbar"><h2>任务执行记录</h2><span class="small muted">包含自动去重任务</span></div><div id="jobsBox"></div></section>
<section><div class="toolbar"><h2>最近操作</h2><span class="small muted">可回滚仍保持同 inode 的 Hard Link</span></div><div id="historyBox"></div></section>
<section><h2>运行设置</h2><div id="settingsBox" class="muted">加载中...</div></section>
</div><div id="toast"></div>
<script>
const esc=s=>String(s??'').replace(/[&<>\"]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c]));
const labels={scan:'扫描索引',dry:'Dry-run 分析',dedupe:'执行去重'};
let busy=false;
function toast(msg,error=false){const e=document.getElementById('toast');e.textContent=msg;e.className=error?'error':'';e.style.display='block';clearTimeout(window._toast);window._toast=setTimeout(()=>e.style.display='none',4500)}
async function api(u,o={}){o.headers=Object.assign({},o.headers||{});const t=localStorage.getItem('am_token');if(t)o.headers.Authorization='Bearer '+t;let r=await fetch(u,o);let j={};try{j=await r.json()}catch{}if(r.status===401){const nt=prompt('请输入 AM Deduper 管理 Token');if(nt){localStorage.setItem('am_token',nt);return api(u,o)}throw Error('需要管理 Token')}if(!r.ok)throw Error(j.error||('HTTP '+r.status));return j}
function setButtons(v){document.querySelectorAll('[data-action]').forEach(b=>b.disabled=v)}
function renderStatus(j){const e=document.getElementById('status');if(!j||!j.items?.length){e.innerHTML='<div><div class="status-main">系统空闲</div><div class="status-detail">监听服务会在新音乐稳定写入后自动进行 SHA-256 精确去重。</div></div><span class="badge">运行正常</span>';return}const x=j.items[0],running=x.status==='running';let detail=x.result||x.error||'等待结果';e.innerHTML='<div><div class="status-main '+(running?'running':x.status==='failed'?'error':'')+'">'+(running?'<span class="jobline"><span class="spinner"></span>':'')+esc(labels[x.kind]||x.kind)+(running?'…</span>':'')+'</div><div class="status-detail">'+esc(detail)+'</div></div><span class="badge">'+esc(x.status)+'</span>';}
function formatResult(r){if(!r)return '';try{const x=JSON.parse(r);const parts=[];if(x.indexed!=null)parts.push('索引 '+x.indexed);if(x.exact_hardlinked!=null)parts.push('精确去重 '+x.exact_hardlinked+' 个');if(x.exact!=null)parts.push('Exact '+x.exact+' 个');if(x.acoustic)parts.push('L2 '+(x.acoustic.equivalent??0)+' 个');return parts.join(' · ')||r}catch{return r}}
async function load(manual=false){try{
const [s,d,q,j,h,cfg]=await Promise.all([api('/api/stats'),api('/api/duplicates'),api('/api/review'),api('/api/jobs'),api('/api/history'),api('/api/settings')]);
document.getElementById('stats').innerHTML=s.cards.map(x=>'<div class="card"><div class="label">'+esc(x.name)+'</div><div class="num">'+esc(x.value)+'</div><div class="hint">'+esc(x.hint||'')+'</div></div>').join('');
document.getElementById('dupeCount').textContent=d.groups.length+' 组';document.getElementById('dupes').innerHTML=d.groups.length?d.groups.map(g=>'<div class="card"><b>'+esc(g.type)+'</b> · 可节省约 '+esc(g.savings)+' MB · '+esc(g.items.length)+' 个逻辑路径<table><tr><th>路径</th><th>状态</th><th>inode</th></tr>'+g.items.map(x=>'<tr><td class="path">'+esc(x.file_path)+'</td><td><span class="badge">'+esc(x.status)+'</span>'+(x.id===g.winner_id?' <span class="badge">保留路径</span>':'')+'</td><td>'+esc(x.inode)+'</td></tr>').join('')+'</table></div>').join(''):'<div class="empty">没有发现 SHA-256 完全重复。新文件会由监听服务自动检查。</div>';
document.getElementById('reviewCount').textContent=q.items.length+' 项';document.getElementById('review').innerHTML=q.items.length?'<table><tr><th>相似度</th><th>文件</th><th>候选目标</th><th>操作</th></tr>'+q.items.map(x=>'<tr><td>'+esc(x.similarity||'-')+'</td><td class="path">'+esc(x.file_path)+'<br>'+esc(x.title||'')+' '+esc(x.artist||'')+'</td><td class="path">'+esc(x.candidate_path||'-')+'</td><td><button onclick="reviewAction('+x.id+',\'reject\')">保留原文件</button> '+(x.candidate_id?'<button class="primary" onclick="reviewAction('+x.id+',\'approve\','+x.candidate_id+')">确认硬链接</button>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无人工审核项目。</div>';
document.getElementById('jobsBox').innerHTML=j.items.length?'<table><tr><th>ID</th><th>任务</th><th>状态</th><th>时间</th><th>结果</th></tr>'+j.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(labels[x.kind]||x.kind)+'</td><td class="'+(x.status==='failed'?'error':x.status==='running'?'running':'success')+'">'+esc(x.status)+'</td><td>'+esc(x.started_at)+'<br>'+esc(x.finished_at||'进行中')+'</td><td>'+esc(formatResult(x.result)||x.error||'-')+(x.error?'<details><summary>错误详情</summary>'+esc(x.error)+'</details>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无任务。启动服务后会自动记录 startup-reconcile。</div>';
document.getElementById('historyBox').innerHTML=h.items.length?'<table><tr><th>ID</th><th>时间</th><th>操作</th><th>文件</th><th>目标</th><th></th></tr>'+h.items.map(x=>'<tr><td>'+x.id+'</td><td>'+esc(x.created_at)+'</td><td>'+esc(x.action)+'</td><td class="path">'+esc(x.file_path)+'</td><td class="path">'+esc(x.winner_path||'')+'</td><td>'+(x.reversible?'<button onclick="rollback('+x.id+')">回滚</button>':'')+'</td></tr>').join('')+'</table>':'<div class="empty">暂无去重操作记录。</div>';
document.getElementById('settingsBox').innerHTML='<div>音乐目录:<span class="path">'+esc(cfg.root)+'</span></div><div>数据库:<span class="path">'+esc(cfg.db)+'</span></div><div>认证:'+esc(cfg.auth)+'</div><div>自动精确去重:<b>已启用</b>(watch 启动时会先扫描一次,之后监听新文件)</div><div>L2:相似度 ≥ '+esc(cfg.l2_min)+' 自动判定,'+esc(cfg.l2_review)+' 起进入审核;时长窗口 ±'+esc(cfg.duration_delta)+' 秒</div>';
renderStatus(j);if(manual)toast('已刷新');
}catch(e){renderStatus(null);toast(e.message,true)}}
async function run(kind){if(busy)return;if(kind==='dedupe'&&!confirm('确认执行安全去重?\n\nSHA-256 完全相同的文件会使用 Hard Link 合并物理存储;逻辑路径会保留。'))return;busy=true;setButtons(true);try{const r=await api('/api/jobs',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({kind})});toast((labels[kind]||kind)+' 已启动,结果会显示在“任务执行记录”中。');await load();const timer=setInterval(async()=>{try{const j=await api('/api/jobs');renderStatus(j);if(!j.items.length||j.items[0].id!==r.job_id||j.items[0].status==='running')return;clearInterval(timer);busy=false;setButtons(false);toast((labels[kind]||kind)+'完成:'+(formatResult(j.items[0].result)||j.items[0].error||j.items[0].status),j.items[0].status==='failed');await load()}catch(e){clearInterval(timer);busy=false;setButtons(false);toast(e.message,true)}},1000)}catch(e){busy=false;setButtons(false);toast(e.message,true)}}
async function reviewAction(id,action,candidate_id){try{await api('/api/review',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id,action,candidate_id})});toast(action==='approve'?'已确认并建立 Hard Link':'已保留原文件');await load()}catch(e){toast(e.message,true)}}
async function rollback(id){if(!confirm('确认回滚此 Hard Link?回滚会恢复 loser 的独立文件内容,并保留两个逻辑路径。'))return;try{await api('/api/rollback',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({id})});toast('回滚完成');await load()}catch(e){toast(e.message,true)}}
load();setInterval(()=>load(),5000);
</script></html>'''
def stats():
with connect(DB) as c:
total = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
hard = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='hardlinked'").fetchone()["n"]
review = c.execute("SELECT COUNT(*) n FROM audio_tracks WHERE status='review'").fetchone()["n"]
size = c.execute("SELECT COALESCE(SUM(file_size),0) n FROM audio_tracks WHERE status IN ('active','hardlinked','review')").fetchone()["n"]
ops = c.execute("SELECT COUNT(*) n FROM operations").fetchone()["n"]
return {"cards":[
{"name":"逻辑音乐文件","value":total,"hint":"每个用户路径各算 1 个"},
{"name":"Hard Link 文件","value":hard,"hint":"物理数据已共享"},
{"name":"待人工审核","value":review,"hint":"L2 相似或版本冲突"},
{"name":"索引逻辑大小","value":f"{size/1024/1024/1024:.2f} GB","hint":"按路径统计,非实际占用"},
{"name":"操作记录","value":ops,"hint":"可追溯/可回滚"}]}
def duplicates():
with connect(DB) as c:
rows = c.execute("SELECT * FROM audio_tracks WHERE status IN ('active','hardlinked') AND sha256 IS NOT NULL ORDER BY sha256,id").fetchall()
groups = {}
for r in rows: groups.setdefault(r["sha256"], []).append(dict(r))
out = []
for _, items in groups.items():
if len(items) > 1:
winner = choose_winner(items)
ordered = [winner] + [x for x in items if x["id"] != winner["id"]]
savings = round(sum(x["file_size"] for x in ordered[1:]) / 1024 / 1024, 2)
out.append({"type":"SHA256 完全重复", "savings":savings, "winner_id":winner["id"], "items":ordered})
return {"groups":out[:100]}
def review_items():
with connect(DB) as c: rows = c.execute("SELECT * FROM audio_tracks WHERE status='review' ORDER BY updated_at DESC").fetchall()
out = []
for r in rows:
d = dict(r); d["similarity"] = (r["review_status"] or "").replace("similarity=", "")
with connect(DB) as c2:
cand = c2.execute("SELECT id,file_path FROM audio_tracks WHERE id!=? AND status IN ('active','hardlinked') AND duration BETWEEN ? AND ? ORDER BY ABS(duration-?) LIMIT 1", (r["id"], (r["duration"] or 0)-3, (r["duration"] or 0)+3, r["duration"] or 0)).fetchone()
d["candidate_id"] = cand["id"] if cand else None; d["candidate_path"] = cand["file_path"] if cand else None; out.append(d)
return {"items":out}
def history():
with connect(DB) as c: rows = c.execute("SELECT id,created_at,action,file_path,winner_path,details FROM operations ORDER BY id DESC LIMIT 100").fetchall()
return {"items":[dict(r, reversible=r["action"] in ("exact-hardlink","acoustic-hardlink","manual-hardlink")) for r in rows]}
def jobs():
with connect(DB) as c: rows = c.execute("SELECT * FROM jobs ORDER BY id DESC LIMIT 30").fetchall()
return {"items":[dict(r) for r in rows]}
def settings():
from .l2 import MIN_SIMILARITY, REVIEW_SIMILARITY, DURATION_DELTA
return {"root":str(ROOT),"db":str(DB),"auth":"Token 已启用" if TOKEN else "未启用(仅建议内网)","l2_min":MIN_SIMILARITY,"l2_review":REVIEW_SIMILARITY,"duration_delta":DURATION_DELTA}
def run_job(kind):
with connect(DB) as c:
jid = c.execute("INSERT INTO jobs(kind,status) VALUES(?,?)", (kind,"running")).lastrowid
def work():
try:
with LOCK:
indexed = scan(ROOT, DB)
if kind == "scan": result = {"indexed": indexed}
elif kind == "dry":
result = {"indexed": indexed, "exact": len(exact_dedupe(DB, ROOT, True)), "acoustic": acoustic_dedupe(DB, ROOT, True)}
else:
result = {"indexed": indexed, "exact": len(exact_dedupe(DB, ROOT)), "acoustic": acoustic_dedupe(DB, ROOT)}
with connect(DB) as c: c.execute("UPDATE jobs SET status='success',finished_at=CURRENT_TIMESTAMP,result=? WHERE id=?", (json.dumps(result,ensure_ascii=False),jid))
except Exception as e:
with connect(DB) as c: c.execute("UPDATE jobs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?", (str(e),jid))
threading.Thread(target=work,daemon=True).start()
return jid
def rollback(opid):
with connect(DB) as c: row = c.execute("SELECT * FROM operations WHERE id=?", (opid,)).fetchone()
if not row or row["action"] not in ("exact-hardlink","acoustic-hardlink","manual-hardlink"): raise ValueError("operation not reversible")
loser,winner = Path(row["file_path"]),Path(row["winner_path"])
if not loser.exists() or not winner.exists() or loser.stat().st_ino != winner.stat().st_ino: raise ValueError("current file state changed; refusing rollback")
tmp = loser.with_name(loser.name+".dedupe-rollback-"+uuid.uuid4().hex)
shutil.copy2(winner,tmp); os.replace(tmp,loser)
with connect(DB) as c:
c.execute("UPDATE audio_tracks SET status='active',review_status='rolled-back',updated_at=CURRENT_TIMESTAMP WHERE file_path=?", (str(loser),))
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)", ("rollback",str(loser),str(winner),f"rollback of operation {opid}"))
def approve(rid,cid):
with connect(DB) as c:
r=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status='review'",(rid,)).fetchone(); w=c.execute("SELECT * FROM audio_tracks WHERE id=? AND status IN ('active','hardlinked')",(cid,)).fetchone()
if not r or not w: raise ValueError("review item or candidate not found")
lp,wp=Path(r["file_path"]),Path(w["file_path"])
if not lp.exists() or not wp.exists() or not lp.is_relative_to(ROOT) or not wp.is_relative_to(ROOT): raise ValueError("invalid library path")
if lp.stat().st_dev != wp.stat().st_dev: raise ValueError("different filesystem")
if lp.stat().st_ino != wp.stat().st_ino: hardlink_duplicate(lp,wp)
with connect(DB) as c:
c.execute("UPDATE audio_tracks SET status='hardlinked',review_status='manual-approved',updated_at=CURRENT_TIMESTAMP WHERE id=?",(rid,))
c.execute("INSERT INTO operations(action,file_path,winner_path,details) VALUES(?,?,?,?)",("manual-hardlink",str(lp),str(wp),"manual review approval"))
class Handler(BaseHTTPRequestHandler):
def send(self,status,data,ctype="application/json"):
raw=data.encode() if isinstance(data,str) else json.dumps(data,ensure_ascii=False).encode(); self.send_response(status); self.send_header("Content-Type",ctype+"; charset=utf-8"); self.send_header("Content-Length",str(len(raw))); self.end_headers(); self.wfile.write(raw)
def authorized(self): return not TOKEN or self.headers.get("Authorization","") == "Bearer "+TOKEN
def do_GET(self):
p=urlparse(self.path).path
try:
if p=="/": return self.send(200,HTML,"text/html")
if not self.authorized(): return self.send(401,{"error":"unauthorized"})
routes={"/api/stats":stats,"/api/duplicates":duplicates,"/api/review":review_items,"/api/history":history,"/api/jobs":jobs,"/api/settings":settings}
if p in routes: return self.send(200,routes[p]())
return self.send(404,{"error":"not found"})
except Exception as e: return self.send(500,{"error":str(e)})
def do_POST(self):
p=urlparse(self.path).path
if not self.authorized(): return self.send(401,{"error":"unauthorized"})
try: body=json.loads(self.rfile.read(int(self.headers.get("content-length","0"))) or b"{}")
except Exception: return self.send(400,{"error":"invalid json"})
try:
if p=="/api/jobs" and body.get("kind") in {"scan","dry","dedupe"}: return self.send(202,{"ok":True,"job_id":run_job(body["kind"])})
if p=="/api/review":
if body.get("action")=="reject":
with connect(DB) as c: c.execute("UPDATE audio_tracks SET status='active',review_status='rejected',updated_at=CURRENT_TIMESTAMP WHERE id=? AND status='review'",(int(body["id"]),))
return self.send(200,{"ok":True})
if body.get("action")=="approve": approve(int(body["id"]),int(body["candidate_id"])); return self.send(200,{"ok":True})
if p=="/api/rollback": rollback(int(body["id"])); return self.send(200,{"ok":True})
return self.send(400,{"error":"invalid request"})
except Exception as e: return self.send(409,{"error":str(e)})
def log_message(self,*args): pass
def serve(host="0.0.0.0",port=8080):
print(f"AM Deduper UI listening on {host}:{port}",flush=True); ThreadingHTTPServer((host,port),Handler).serve_forever()