find_dupes: memory-safe pairwise Hamming at 30k+ scale

The 64-pass int64 shift-add materialized ~8GB temporaries per pass
(~27GB peak at n=31,937) — OOM on a 31GB box. Replace with
np.bitwise_count when available (numpy>=2.3) and a per-byte popcount
LUT fallback; uint8 results suffice (max distance 64). Verified on a
smoke corpus (exact-duplicate groups found, move semantics intact)
before the Beyblade X 31,937-still run.

find_dupes.py

@@ -119,27 +119,35 @@ def main():
     # different scenes).
     n = len(names)
     print(f"Pairwise compare {n}x{n} (numpy hamming) ...")
+
+    def hamming64(mat_u64):
+        """Full pairwise Hamming, memory-safe at 30k+ scale. The old
+        64-pass int64 shift-add peaked at ~27 GB on 32k frames (OOM);
+        bitwise_count (numpy>=2.3) or a per-byte popcount LUT keeps the
+        peak at one uint64 xor matrix + a uint8 result (max dist 64)."""
+        x = mat_u64[:, None] ^ mat_u64[None, :]
+        if hasattr(np, "bitwise_count"):
+            ham = np.bitwise_count(x)
+        else:
+            bc = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
+            x8 = x.view(np.uint8).reshape(x.shape + (8,))
+            ham = np.zeros(x.shape, dtype=np.uint8)
+            for b in range(8):
+                ham += bc[x8[..., b]]
+            del x8
+        del x
+        return ham
+
     dmat = np.array([data[f][0] for f in names], dtype=np.uint64)
-    # hamming distances via xor + popcount (work in int64; numpy has no
+    close = hamming64(dmat) <= args.max_dist
-    # sane uint64 shift-add promotion)
-    x = (dmat[:, None] ^ dmat[None, :]).astype(np.int64)
-    ham = np.zeros(x.shape, dtype=np.int16)
-    for b in range(64):
-        ham += ((x >> b) & 1).astype(np.int16)
 
     # combined close matrix: dHash AND aHash AND color-histogram gates.
     # dHash alone has a false-positive floor (verified: distinct frames
     # at distance <=10); a second, differently-computed hash plus a
     # color signature is what separates 'same frame replayed' from
     # 'same show, different shot'.
-    close = ham <= args.max_dist
-
     amat = np.array([data[f][1] for f in names], dtype=np.uint64)
-    xa = (amat[:, None] ^ amat[None, :]).astype(np.int64)
+    close &= hamming64(amat) <= (args.max_adist
-    aham = np.zeros(xa.shape, dtype=np.int16)
-    for b in range(64):
-        aham += ((xa >> b) & 1).astype(np.int16)
-    close &= aham <= (args.max_adist
                                  if args.max_adist is not None else 8)
 
     hmat = np.stack([data[f][3] for f in names])  # N x 192 hist rows