From 7e128c4c3256089a02b481e20eb947819185ad94 Mon Sep 17 00:00:00 2001
From: Alex <alex@matrixwide.com>
Date: Sun, 18 Oct 2026 18:47:08 -0400
Subject: [PATCH] find_dupes: replace union-find with strict complete-linkage
 clustering
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Verified failure with 10/64 threshold + transitive chaining: a 90-image
'group' contained six different scenes (blonde outdoors, storm school,
night bedroom, manic grin, cat street, dining room) — chains of
similar-but-distinct frames glue unrelated shots together. Strict
linkage requires every member within threshold of every other.
Default quarantine dir renamed duplicates/.
---
 find_dupes.py | 43 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 11 deletions(-)

diff --git a/find_dupes.py b/find_dupes.py
index 19bc894..fe80932 100644
--- a/find_dupes.py
+++ b/find_dupes.py
@@ -35,7 +35,7 @@ def parse_args():
     p.add_argument("--move", action="store_true",
                    help="Move non-keepsakes to --dupe-dir.")
     p.add_argument("--dupe-dir", default=None,
-                   help="Default: <input-dir>/dupes")
+                   help="Default: <input-dir>/duplicates")
     p.add_argument("--keep-sharp", action="store_true",
                    help="Keepsake = sharpest (Laplacian var) instead of "
                         "lowest filename.")
@@ -90,6 +90,12 @@ def main():
             print(f"  {i+1}/{len(files)}", flush=True)
 
     names = list(data)
+    # strict (complete-linkage) clustering — NO transitive chaining:
+    # seed with the ungrouped image having most neighbours, take its
+    # within-threshold neighbourhood, then keep only members pairwise
+    # within threshold. Union-find chaining merges unrelated frames
+    # after enough hops (verified failure: 90-frame 'group' of six
+    # different scenes).
     n = len(names)
     print(f"Pairwise compare {n}x{n} (numpy hamming) ...")
     dmat = np.array([data[f][0] for f in names], dtype=np.uint64)
@@ -114,15 +120,30 @@ def main():
         if ri != rj:
             parent[ri] = rj
 
-    idx = np.argwhere(ham <= args.max_dist)
-    for i, j in idx:
-        if i < j:
-            union(int(i), int(j))
-
-    groups = defaultdict(list)
-    for i in range(n):
-        groups[find(i)].append(i)
-    clusters = [g for g in groups.values() if len(g) > 1]
+    # strict (complete-linkage) clustering — NO transitive chaining:
+    # seed with the ungrouped image having most neighbours, take its
+    # within-threshold neighbourhood, then keep only members pairwise
+    # within threshold. Union-find chaining merges unrelated frames
+    # after enough hops (verified failure: 90-frame 'group' of six
+    # different scenes).
+    close = ham <= args.max_dist
+    np.fill_diagonal(close, False)
+    order = sorted(range(n), key=lambda i: -close[i].sum())
+    used = np.zeros(n, dtype=bool)
+    clusters = []
+    for seed in order:
+        if used[seed] or not close[seed].any():
+            continue
+        cand = [seed] + [j for j in range(n)
+                         if close[seed][j] and not used[j]]
+        group = []
+        for j in cand:  # keep only pairwise-close members
+            if all(bool(ham[j][k] <= args.max_dist) for k in group):
+                group.append(j)
+        if len(group) > 1:
+            for j in group:
+                used[j] = True
+            clusters.append(group)
     clusters.sort(key=len, reverse=True)
 
     dupes_total = sum(len(g) - 1 for g in clusters)
@@ -151,7 +172,7 @@ def main():
         print(f"\nReport: {args.report}")
 
     if args.move:
-        dest = args.dupe_dir or os.path.join(args.input_dir, "dupes")
+        dest = args.dupe_dir or os.path.join(args.input_dir, "duplicates")
         os.makedirs(dest, exist_ok=True)
         moved = 0
         for r in rows: