find_dupes: triple-gate closeness (dHash<=6, aHash<=8, color-hist L1<=0.10) + strict linkage

Measured on verified same/different pairs: dHash alone has a false-
positive floor for same-style anime frames (distinct shots at distance
<=10 and even <=6 with different color content). Same-shot broadcast
replays are pixel-twins: near-zero color-hist distance and low aHash
distance too. Requires ALL three gates, no transitive chaining.

find_dupes.py

@@ -27,10 +27,22 @@ def parse_args():
     p = argparse.ArgumentParser(
         description=(__doc__ or "").split("\n")[0])
     p.add_argument("-I", "--input-dir", required=True)
-    p.add_argument("--max-dist", type=int, default=10,
+    p.add_argument("--max-dist", type=int, default=6,
                    help="Max differing dHash bits (of 64) to cluster. "
-                        "0-6 near-identical, 8-12 replay-same-shot, "
-                        ">14 starts eating costume/pose changes.")
+                        "Broadcast replays of the identical shot differ "
+                        "by only a few bits; verified at 10 that "
+                        "genuinely-different frames can still be "
+                        "within threshold (dHash floor), so default 6.")
+    p.add_argument("--max-adist", type=int, default=None,
+                   help="Optional second gate: max differing aHash bits. "
+                        "Different hashes rarely agree twice; same frames "
+                        "always do.")
+    p.add_argument("--max-cdist", type=float, default=0.10,
+                   help="Max L1 distance between normalized RGB color "
+                        "histograms (0.128-bin/bh). Broadcast replays "
+                        "are the SAME frames (near-zero color distance); "
+                        "different shots differ >>0.10. This is the "
+                        "signal that dHash lacks; set high to disable.")
     p.add_argument("--report", default=None, help="CSV report path.")
     p.add_argument("--move", action="store_true",
                    help="Move non-keepsakes to --dupe-dir.")
@@ -44,7 +56,16 @@ def parse_args():
 
 def hashes(img):
     import cv2
+    import numpy as np
     g = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    # color histogram signature (normalized L1 distance separates
+    # same-shot replays (~0) from different shots (>>0.1) — the signal
+    # dHash lacks: same gradient structure, different content)
+    hist = np.zeros(3 * 64, dtype=np.float32)
+    for c in range(3):
+        h = cv2.calcHist([img], [c], None, [64], [0, 256]).flatten()
+        hist[c*64:(c+1)*64] = h
+    hist /= hist.sum()
     # dHash 8x8 gradient
     small = cv2.resize(g, (9, 8), interpolation=cv2.INTER_AREA)
     diff = small[:, 1:] > small[:, :-1]
@@ -61,7 +82,7 @@ def hashes(img):
             a |= 1 << bit
     # sharpness proxy for --keep-sharp
     sharp = cv2.Laplacian(g, cv2.CV_64F).var()
-    return d, a, sharp
+    return d, a, sharp, hist
 
 
 def main():
@@ -84,8 +105,8 @@ def main():
         img = cv2.imread(os.path.join(args.input_dir, f))
         if img is None:
             continue
-        d, a, sharp = hashes(img)
-        data[f] = (d, a, sharp)
+        d, a, sharp, hist = hashes(img)
+        data[f] = (d, a, sharp, hist)
         if (i + 1) % 500 == 0:
             print(f"  {i+1}/{len(files)}", flush=True)
 
@@ -106,19 +127,27 @@ def main():
     for b in range(64):
         ham += ((x >> b) & 1).astype(np.int16)
 
-    # union-find clustering on distance threshold
-    parent = list(range(n))
+    # combined close matrix: dHash AND aHash AND color-histogram gates.
+    # dHash alone has a false-positive floor (verified: distinct frames
+    # at distance <=10); a second, differently-computed hash plus a
+    # color signature is what separates 'same frame replayed' from
+    # 'same show, different shot'.
+    close = ham <= args.max_dist
 
-    def find(i):
-        while parent[i] != i:
-            parent[i] = parent[parent[i]]
-            i = parent[i]
-        return i
+    amat = np.array([data[f][1] for f in names], dtype=np.int64)
+    xa = (amat[:, None] ^ amat[None, :])
+    aham = np.zeros(xa.shape, dtype=np.int16)
+    for b in range(64):
+        aham += ((xa >> b) & 1).astype(np.int16)
+    close &= aham <= (args.max_adist
+                      if args.max_adist is not None else 8)
 
-    def union(i, j):
-        ri, rj = find(i), find(j)
-        if ri != rj:
-            parent[ri] = rj
+    hmat = np.stack([data[f][3] for f in names])  # N x 192 hist rows
+    cdist = np.zeros((n, n), dtype=np.float32)
+    for i in range(n):  # row-wise keeps it ~seconds and RAM-light
+        cdist[i] = np.abs(hmat - hmat[i]).sum(axis=1)
+    close &= cdist <= args.max_cdist
+    np.fill_diagonal(close, False)
 
     # strict (complete-linkage) clustering — NO transitive chaining:
     # seed with the ungrouped image having most neighbours, take its
@@ -126,8 +155,6 @@ def main():
     # within threshold. Union-find chaining merges unrelated frames
     # after enough hops (verified failure: 90-frame 'group' of six
     # different scenes).
-    close = ham <= args.max_dist
-    np.fill_diagonal(close, False)
     order = sorted(range(n), key=lambda i: -close[i].sum())
     used = np.zeros(n, dtype=bool)
     clusters = []
@@ -138,7 +165,7 @@ def main():
                          if close[seed][j] and not used[j]]
         group = []
         for j in cand:  # keep only pairwise-close members
-            if all(bool(ham[j][k] <= args.max_dist) for k in group):
+            if all(bool(close[j][k]) for k in group):
                 group.append(j)
         if len(group) > 1:
             for j in group: