find_dupes: replace union-find with strict complete-linkage clustering
Verified failure with 10/64 threshold + transitive chaining: a 90-image 'group' contained six different scenes (blonde outdoors, storm school, night bedroom, manic grin, cat street, dining room) — chains of similar-but-distinct frames glue unrelated shots together. Strict linkage requires every member within threshold of every other. Default quarantine dir renamed duplicates/.
This commit is contained in:
+32
-11
@@ -35,7 +35,7 @@ def parse_args():
|
||||
p.add_argument("--move", action="store_true",
|
||||
help="Move non-keepsakes to --dupe-dir.")
|
||||
p.add_argument("--dupe-dir", default=None,
|
||||
help="Default: <input-dir>/dupes")
|
||||
help="Default: <input-dir>/duplicates")
|
||||
p.add_argument("--keep-sharp", action="store_true",
|
||||
help="Keepsake = sharpest (Laplacian var) instead of "
|
||||
"lowest filename.")
|
||||
@@ -90,6 +90,12 @@ def main():
|
||||
print(f" {i+1}/{len(files)}", flush=True)
|
||||
|
||||
names = list(data)
|
||||
# strict (complete-linkage) clustering — NO transitive chaining:
|
||||
# seed with the ungrouped image having most neighbours, take its
|
||||
# within-threshold neighbourhood, then keep only members pairwise
|
||||
# within threshold. Union-find chaining merges unrelated frames
|
||||
# after enough hops (verified failure: 90-frame 'group' of six
|
||||
# different scenes).
|
||||
n = len(names)
|
||||
print(f"Pairwise compare {n}x{n} (numpy hamming) ...")
|
||||
dmat = np.array([data[f][0] for f in names], dtype=np.uint64)
|
||||
@@ -114,15 +120,30 @@ def main():
|
||||
if ri != rj:
|
||||
parent[ri] = rj
|
||||
|
||||
idx = np.argwhere(ham <= args.max_dist)
|
||||
for i, j in idx:
|
||||
if i < j:
|
||||
union(int(i), int(j))
|
||||
|
||||
groups = defaultdict(list)
|
||||
for i in range(n):
|
||||
groups[find(i)].append(i)
|
||||
clusters = [g for g in groups.values() if len(g) > 1]
|
||||
# strict (complete-linkage) clustering — NO transitive chaining:
|
||||
# seed with the ungrouped image having most neighbours, take its
|
||||
# within-threshold neighbourhood, then keep only members pairwise
|
||||
# within threshold. Union-find chaining merges unrelated frames
|
||||
# after enough hops (verified failure: 90-frame 'group' of six
|
||||
# different scenes).
|
||||
close = ham <= args.max_dist
|
||||
np.fill_diagonal(close, False)
|
||||
order = sorted(range(n), key=lambda i: -close[i].sum())
|
||||
used = np.zeros(n, dtype=bool)
|
||||
clusters = []
|
||||
for seed in order:
|
||||
if used[seed] or not close[seed].any():
|
||||
continue
|
||||
cand = [seed] + [j for j in range(n)
|
||||
if close[seed][j] and not used[j]]
|
||||
group = []
|
||||
for j in cand: # keep only pairwise-close members
|
||||
if all(bool(ham[j][k] <= args.max_dist) for k in group):
|
||||
group.append(j)
|
||||
if len(group) > 1:
|
||||
for j in group:
|
||||
used[j] = True
|
||||
clusters.append(group)
|
||||
clusters.sort(key=len, reverse=True)
|
||||
|
||||
dupes_total = sum(len(g) - 1 for g in clusters)
|
||||
@@ -151,7 +172,7 @@ def main():
|
||||
print(f"\nReport: {args.report}")
|
||||
|
||||
if args.move:
|
||||
dest = args.dupe_dir or os.path.join(args.input_dir, "dupes")
|
||||
dest = args.dupe_dir or os.path.join(args.input_dir, "duplicates")
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
moved = 0
|
||||
for r in rows:
|
||||
|
||||
Reference in New Issue
Block a user