From 107f48d118dffc2dce4798646a9800f2e132fcd8 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 18 Oct 2026 18:51:45 -0400 Subject: [PATCH] find_dupes: triple-gate closeness (dHash<=6, aHash<=8, color-hist L1<=0.10) + strict linkage Measured on verified same/different pairs: dHash alone has a false- positive floor for same-style anime frames (distinct shots at distance <=10 and even <=6 with different color content). Same-shot broadcast replays are pixel-twins: near-zero color-hist distance and low aHash distance too. Requires ALL three gates, no transitive chaining. --- find_dupes.py | 67 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/find_dupes.py b/find_dupes.py index fe80932..d2b9ba2 100644 --- a/find_dupes.py +++ b/find_dupes.py @@ -27,10 +27,22 @@ def parse_args(): p = argparse.ArgumentParser( description=(__doc__ or "").split("\n")[0]) p.add_argument("-I", "--input-dir", required=True) - p.add_argument("--max-dist", type=int, default=10, + p.add_argument("--max-dist", type=int, default=6, help="Max differing dHash bits (of 64) to cluster. " - "0-6 near-identical, 8-12 replay-same-shot, " - ">14 starts eating costume/pose changes.") + "Broadcast replays of the identical shot differ " + "by only a few bits; verified at 10 that " + "genuinely-different frames can still be " + "within threshold (dHash floor), so default 6.") + p.add_argument("--max-adist", type=int, default=None, + help="Optional second gate: max differing aHash bits. " + "Different hashes rarely agree twice; same frames " + "always do.") + p.add_argument("--max-cdist", type=float, default=0.10, + help="Max L1 distance between normalized RGB color " + "histograms (0.128-bin/bh). Broadcast replays " + "are the SAME frames (near-zero color distance); " + "different shots differ >>0.10. This is the " + "signal that dHash lacks; set high to disable.") p.add_argument("--report", default=None, help="CSV report path.") p.add_argument("--move", action="store_true", help="Move non-keepsakes to --dupe-dir.") @@ -44,7 +56,16 @@ def parse_args(): def hashes(img): import cv2 + import numpy as np g = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + # color histogram signature (normalized L1 distance separates + # same-shot replays (~0) from different shots (>>0.1) — the signal + # dHash lacks: same gradient structure, different content) + hist = np.zeros(3 * 64, dtype=np.float32) + for c in range(3): + h = cv2.calcHist([img], [c], None, [64], [0, 256]).flatten() + hist[c*64:(c+1)*64] = h + hist /= hist.sum() # dHash 8x8 gradient small = cv2.resize(g, (9, 8), interpolation=cv2.INTER_AREA) diff = small[:, 1:] > small[:, :-1] @@ -61,7 +82,7 @@ def hashes(img): a |= 1 << bit # sharpness proxy for --keep-sharp sharp = cv2.Laplacian(g, cv2.CV_64F).var() - return d, a, sharp + return d, a, sharp, hist def main(): @@ -84,8 +105,8 @@ def main(): img = cv2.imread(os.path.join(args.input_dir, f)) if img is None: continue - d, a, sharp = hashes(img) - data[f] = (d, a, sharp) + d, a, sharp, hist = hashes(img) + data[f] = (d, a, sharp, hist) if (i + 1) % 500 == 0: print(f" {i+1}/{len(files)}", flush=True) @@ -106,19 +127,27 @@ def main(): for b in range(64): ham += ((x >> b) & 1).astype(np.int16) - # union-find clustering on distance threshold - parent = list(range(n)) + # combined close matrix: dHash AND aHash AND color-histogram gates. + # dHash alone has a false-positive floor (verified: distinct frames + # at distance <=10); a second, differently-computed hash plus a + # color signature is what separates 'same frame replayed' from + # 'same show, different shot'. + close = ham <= args.max_dist - def find(i): - while parent[i] != i: - parent[i] = parent[parent[i]] - i = parent[i] - return i + amat = np.array([data[f][1] for f in names], dtype=np.int64) + xa = (amat[:, None] ^ amat[None, :]) + aham = np.zeros(xa.shape, dtype=np.int16) + for b in range(64): + aham += ((xa >> b) & 1).astype(np.int16) + close &= aham <= (args.max_adist + if args.max_adist is not None else 8) - def union(i, j): - ri, rj = find(i), find(j) - if ri != rj: - parent[ri] = rj + hmat = np.stack([data[f][3] for f in names]) # N x 192 hist rows + cdist = np.zeros((n, n), dtype=np.float32) + for i in range(n): # row-wise keeps it ~seconds and RAM-light + cdist[i] = np.abs(hmat - hmat[i]).sum(axis=1) + close &= cdist <= args.max_cdist + np.fill_diagonal(close, False) # strict (complete-linkage) clustering — NO transitive chaining: # seed with the ungrouped image having most neighbours, take its @@ -126,8 +155,6 @@ def main(): # within threshold. Union-find chaining merges unrelated frames # after enough hops (verified failure: 90-frame 'group' of six # different scenes). - close = ham <= args.max_dist - np.fill_diagonal(close, False) order = sorted(range(n), key=lambda i: -close[i].sum()) used = np.zeros(n, dtype=bool) clusters = [] @@ -138,7 +165,7 @@ def main(): if close[seed][j] and not used[j]] group = [] for j in cand: # keep only pairwise-close members - if all(bool(ham[j][k] <= args.max_dist) for k in group): + if all(bool(close[j][k]) for k in group): group.append(j) if len(group) > 1: for j in group: