find_dupes: replace union-find with strict complete-linkage clustering
Verified failure with 10/64 threshold + transitive chaining: a 90-image 'group' contained six different scenes (blonde outdoors, storm school, night bedroom, manic grin, cat street, dining room) — chains of similar-but-distinct frames glue unrelated shots together. Strict linkage requires every member within threshold of every other. Default quarantine dir renamed duplicates/.
This commit is contained in:
+32
-11
@@ -35,7 +35,7 @@ def parse_args():
|
|||||||
p.add_argument("--move", action="store_true",
|
p.add_argument("--move", action="store_true",
|
||||||
help="Move non-keepsakes to --dupe-dir.")
|
help="Move non-keepsakes to --dupe-dir.")
|
||||||
p.add_argument("--dupe-dir", default=None,
|
p.add_argument("--dupe-dir", default=None,
|
||||||
help="Default: <input-dir>/dupes")
|
help="Default: <input-dir>/duplicates")
|
||||||
p.add_argument("--keep-sharp", action="store_true",
|
p.add_argument("--keep-sharp", action="store_true",
|
||||||
help="Keepsake = sharpest (Laplacian var) instead of "
|
help="Keepsake = sharpest (Laplacian var) instead of "
|
||||||
"lowest filename.")
|
"lowest filename.")
|
||||||
@@ -90,6 +90,12 @@ def main():
|
|||||||
print(f" {i+1}/{len(files)}", flush=True)
|
print(f" {i+1}/{len(files)}", flush=True)
|
||||||
|
|
||||||
names = list(data)
|
names = list(data)
|
||||||
|
# strict (complete-linkage) clustering — NO transitive chaining:
|
||||||
|
# seed with the ungrouped image having most neighbours, take its
|
||||||
|
# within-threshold neighbourhood, then keep only members pairwise
|
||||||
|
# within threshold. Union-find chaining merges unrelated frames
|
||||||
|
# after enough hops (verified failure: 90-frame 'group' of six
|
||||||
|
# different scenes).
|
||||||
n = len(names)
|
n = len(names)
|
||||||
print(f"Pairwise compare {n}x{n} (numpy hamming) ...")
|
print(f"Pairwise compare {n}x{n} (numpy hamming) ...")
|
||||||
dmat = np.array([data[f][0] for f in names], dtype=np.uint64)
|
dmat = np.array([data[f][0] for f in names], dtype=np.uint64)
|
||||||
@@ -114,15 +120,30 @@ def main():
|
|||||||
if ri != rj:
|
if ri != rj:
|
||||||
parent[ri] = rj
|
parent[ri] = rj
|
||||||
|
|
||||||
idx = np.argwhere(ham <= args.max_dist)
|
# strict (complete-linkage) clustering — NO transitive chaining:
|
||||||
for i, j in idx:
|
# seed with the ungrouped image having most neighbours, take its
|
||||||
if i < j:
|
# within-threshold neighbourhood, then keep only members pairwise
|
||||||
union(int(i), int(j))
|
# within threshold. Union-find chaining merges unrelated frames
|
||||||
|
# after enough hops (verified failure: 90-frame 'group' of six
|
||||||
groups = defaultdict(list)
|
# different scenes).
|
||||||
for i in range(n):
|
close = ham <= args.max_dist
|
||||||
groups[find(i)].append(i)
|
np.fill_diagonal(close, False)
|
||||||
clusters = [g for g in groups.values() if len(g) > 1]
|
order = sorted(range(n), key=lambda i: -close[i].sum())
|
||||||
|
used = np.zeros(n, dtype=bool)
|
||||||
|
clusters = []
|
||||||
|
for seed in order:
|
||||||
|
if used[seed] or not close[seed].any():
|
||||||
|
continue
|
||||||
|
cand = [seed] + [j for j in range(n)
|
||||||
|
if close[seed][j] and not used[j]]
|
||||||
|
group = []
|
||||||
|
for j in cand: # keep only pairwise-close members
|
||||||
|
if all(bool(ham[j][k] <= args.max_dist) for k in group):
|
||||||
|
group.append(j)
|
||||||
|
if len(group) > 1:
|
||||||
|
for j in group:
|
||||||
|
used[j] = True
|
||||||
|
clusters.append(group)
|
||||||
clusters.sort(key=len, reverse=True)
|
clusters.sort(key=len, reverse=True)
|
||||||
|
|
||||||
dupes_total = sum(len(g) - 1 for g in clusters)
|
dupes_total = sum(len(g) - 1 for g in clusters)
|
||||||
@@ -151,7 +172,7 @@ def main():
|
|||||||
print(f"\nReport: {args.report}")
|
print(f"\nReport: {args.report}")
|
||||||
|
|
||||||
if args.move:
|
if args.move:
|
||||||
dest = args.dupe_dir or os.path.join(args.input_dir, "dupes")
|
dest = args.dupe_dir or os.path.join(args.input_dir, "duplicates")
|
||||||
os.makedirs(dest, exist_ok=True)
|
os.makedirs(dest, exist_ok=True)
|
||||||
moved = 0
|
moved = 0
|
||||||
for r in rows:
|
for r in rows:
|
||||||
|
|||||||
Reference in New Issue
Block a user