diff --git a/find_dupes.py b/find_dupes.py index f9c346a..8ac1c2c 100644 --- a/find_dupes.py +++ b/find_dupes.py @@ -111,21 +111,18 @@ def main(): print(f" {i+1}/{len(files)}", flush=True) names = list(data) - # strict (complete-linkage) clustering — NO transitive chaining: - # seed with the ungrouped image having most neighbours, take its - # within-threshold neighbourhood, then keep only members pairwise - # within threshold. Union-find chaining merges unrelated frames - # after enough hops (verified failure: 90-frame 'group' of six - # different scenes). n = len(names) - print(f"Pairwise compare {n}x{n} (numpy hamming) ...") + print(f"Pairwise compare {n}x{n} (streaming hamming, chunked) ...") - def hamming64(mat_u64): - """Full pairwise Hamming, memory-safe at 30k+ scale. The old - 64-pass int64 shift-add peaked at ~27 GB on 32k frames (OOM); - bitwise_count (numpy>=2.3) or a per-byte popcount LUT keeps the - peak at one uint64 xor matrix + a uint8 result (max dist 64).""" - x = mat_u64[:, None] ^ mat_u64[None, :] + # Memory-safe at 30k+ scale under ~4 GB cgroup caps (verified: the + # full n^2 uint64 xor matrix — 8 GB at n=32k — got a background run + # oom-killed in a hermes-worker systemd scope on 2026-10-29). Row + # chunks of 1024 keep peak temps at a few hundred MB; only the final + # bool close-matrix (n^2 bytes) stays resident. uint8 distances + # suffice (max 64) via np.bitwise_count (numpy>=2.3) or a per-byte + # popcount LUT. + def hamming_chunk(mat, rows): + x = mat[rows, None] ^ mat[None, :] if hasattr(np, "bitwise_count"): ham = np.bitwise_count(x) else: @@ -139,22 +136,28 @@ def main(): return ham dmat = np.array([data[f][0] for f in names], dtype=np.uint64) - close = hamming64(dmat) <= args.max_dist + amat = np.array([data[f][1] for f in names], dtype=np.uint64) + max_a = args.max_adist if args.max_adist is not None else 8 + CH = 1024 + close = np.zeros((n, n), dtype=bool) + for s in range(0, n, CH): + rows = slice(s, min(s + CH, n)) + close[rows] = ((hamming_chunk(dmat, rows) <= args.max_dist) + & (hamming_chunk(amat, rows) <= max_a)) + if (s // CH) % 8 == 0: + print(f" hashes {min(s + CH, n)}/{n}", flush=True) + del dmat, amat # combined close matrix: dHash AND aHash AND color-histogram gates. # dHash alone has a false-positive floor (verified: distinct frames # at distance <=10); a second, differently-computed hash plus a # color signature is what separates 'same frame replayed' from - # 'same show, different shot'. - amat = np.array([data[f][1] for f in names], dtype=np.uint64) - close &= hamming64(amat) <= (args.max_adist - if args.max_adist is not None else 8) - + # 'same show, different shot'. Color L1 is computed per row only for + # rows that survived both hashes — few rows at real corpus scale, + # and it replaces the old n^2 float32 matrix (4 GB at n=32k). hmat = np.stack([data[f][3] for f in names]) # N x 192 hist rows - cdist = np.zeros((n, n), dtype=np.float32) - for i in range(n): # row-wise keeps it ~seconds and RAM-light - cdist[i] = np.abs(hmat - hmat[i]).sum(axis=1) - close &= cdist <= args.max_cdist + for i in np.flatnonzero(close.any(axis=1)): + close[i] &= np.abs(hmat - hmat[i]).sum(axis=1) <= args.max_cdist np.fill_diagonal(close, False) # strict (complete-linkage) clustering — NO transitive chaining: