diff --git a/find_dupes.py b/find_dupes.py index 097598f..f9c346a 100644 --- a/find_dupes.py +++ b/find_dupes.py @@ -119,28 +119,36 @@ def main(): # different scenes). n = len(names) print(f"Pairwise compare {n}x{n} (numpy hamming) ...") + + def hamming64(mat_u64): + """Full pairwise Hamming, memory-safe at 30k+ scale. The old + 64-pass int64 shift-add peaked at ~27 GB on 32k frames (OOM); + bitwise_count (numpy>=2.3) or a per-byte popcount LUT keeps the + peak at one uint64 xor matrix + a uint8 result (max dist 64).""" + x = mat_u64[:, None] ^ mat_u64[None, :] + if hasattr(np, "bitwise_count"): + ham = np.bitwise_count(x) + else: + bc = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8) + x8 = x.view(np.uint8).reshape(x.shape + (8,)) + ham = np.zeros(x.shape, dtype=np.uint8) + for b in range(8): + ham += bc[x8[..., b]] + del x8 + del x + return ham + dmat = np.array([data[f][0] for f in names], dtype=np.uint64) - # hamming distances via xor + popcount (work in int64; numpy has no - # sane uint64 shift-add promotion) - x = (dmat[:, None] ^ dmat[None, :]).astype(np.int64) - ham = np.zeros(x.shape, dtype=np.int16) - for b in range(64): - ham += ((x >> b) & 1).astype(np.int16) + close = hamming64(dmat) <= args.max_dist # combined close matrix: dHash AND aHash AND color-histogram gates. # dHash alone has a false-positive floor (verified: distinct frames # at distance <=10); a second, differently-computed hash plus a # color signature is what separates 'same frame replayed' from # 'same show, different shot'. - close = ham <= args.max_dist - amat = np.array([data[f][1] for f in names], dtype=np.uint64) - xa = (amat[:, None] ^ amat[None, :]).astype(np.int64) - aham = np.zeros(xa.shape, dtype=np.int16) - for b in range(64): - aham += ((xa >> b) & 1).astype(np.int16) - close &= aham <= (args.max_adist - if args.max_adist is not None else 8) + close &= hamming64(amat) <= (args.max_adist + if args.max_adist is not None else 8) hmat = np.stack([data[f][3] for f in names]) # N x 192 hist rows cdist = np.zeros((n, n), dtype=np.float32)