find_dupes: replace union-find with strict complete-linkage clustering

Verified failure with 10/64 threshold + transitive chaining: a 90-image
'group' contained six different scenes (blonde outdoors, storm school,
night bedroom, manic grin, cat street, dining room) — chains of
similar-but-distinct frames glue unrelated shots together. Strict
linkage requires every member within threshold of every other.
Default quarantine dir renamed duplicates/.
This commit is contained in:
2026-10-18 18:47:08 -04:00
parent 4816b007c0
commit 7e128c4c32
+32 -11
View File
@@ -35,7 +35,7 @@ def parse_args():
p.add_argument("--move", action="store_true", p.add_argument("--move", action="store_true",
help="Move non-keepsakes to --dupe-dir.") help="Move non-keepsakes to --dupe-dir.")
p.add_argument("--dupe-dir", default=None, p.add_argument("--dupe-dir", default=None,
help="Default: <input-dir>/dupes") help="Default: <input-dir>/duplicates")
p.add_argument("--keep-sharp", action="store_true", p.add_argument("--keep-sharp", action="store_true",
help="Keepsake = sharpest (Laplacian var) instead of " help="Keepsake = sharpest (Laplacian var) instead of "
"lowest filename.") "lowest filename.")
@@ -90,6 +90,12 @@ def main():
print(f" {i+1}/{len(files)}", flush=True) print(f" {i+1}/{len(files)}", flush=True)
names = list(data) names = list(data)
# strict (complete-linkage) clustering — NO transitive chaining:
# seed with the ungrouped image having most neighbours, take its
# within-threshold neighbourhood, then keep only members pairwise
# within threshold. Union-find chaining merges unrelated frames
# after enough hops (verified failure: 90-frame 'group' of six
# different scenes).
n = len(names) n = len(names)
print(f"Pairwise compare {n}x{n} (numpy hamming) ...") print(f"Pairwise compare {n}x{n} (numpy hamming) ...")
dmat = np.array([data[f][0] for f in names], dtype=np.uint64) dmat = np.array([data[f][0] for f in names], dtype=np.uint64)
@@ -114,15 +120,30 @@ def main():
if ri != rj: if ri != rj:
parent[ri] = rj parent[ri] = rj
idx = np.argwhere(ham <= args.max_dist) # strict (complete-linkage) clustering — NO transitive chaining:
for i, j in idx: # seed with the ungrouped image having most neighbours, take its
if i < j: # within-threshold neighbourhood, then keep only members pairwise
union(int(i), int(j)) # within threshold. Union-find chaining merges unrelated frames
# after enough hops (verified failure: 90-frame 'group' of six
groups = defaultdict(list) # different scenes).
for i in range(n): close = ham <= args.max_dist
groups[find(i)].append(i) np.fill_diagonal(close, False)
clusters = [g for g in groups.values() if len(g) > 1] order = sorted(range(n), key=lambda i: -close[i].sum())
used = np.zeros(n, dtype=bool)
clusters = []
for seed in order:
if used[seed] or not close[seed].any():
continue
cand = [seed] + [j for j in range(n)
if close[seed][j] and not used[j]]
group = []
for j in cand: # keep only pairwise-close members
if all(bool(ham[j][k] <= args.max_dist) for k in group):
group.append(j)
if len(group) > 1:
for j in group:
used[j] = True
clusters.append(group)
clusters.sort(key=len, reverse=True) clusters.sort(key=len, reverse=True)
dupes_total = sum(len(g) - 1 for g in clusters) dupes_total = sum(len(g) - 1 for g in clusters)
@@ -151,7 +172,7 @@ def main():
print(f"\nReport: {args.report}") print(f"\nReport: {args.report}")
if args.move: if args.move:
dest = args.dupe_dir or os.path.join(args.input_dir, "dupes") dest = args.dupe_dir or os.path.join(args.input_dir, "duplicates")
os.makedirs(dest, exist_ok=True) os.makedirs(dest, exist_ok=True)
moved = 0 moved = 0
for r in rows: for r in rows: