find_dupes: streaming row-chunked compare (fits ~4GB cgroup caps)

The single-matrix fix still oom-killed at n=31,937: Hermes background
jobs run in a systemd scope capped near 4GB, and the n^2 uint64 xor
matrix alone is 8GB. Now row chunks of 1024 stream through both hash
gates (peak a few hundred MB temps), the only resident n^2 structure
is the bool close-matrix (1GB), and the color-histogram L1 is computed
only for rows surviving both hashes (replaces the 4GB float32 matrix).
Verified on a 2,502-file corpus incl planted cross-chunk duplicate
pair and an orphan copy (correctly untouched).

.gitignore

@@ -6,7 +6,3 @@ output/
 preview/
 *.pt
 *.part
-.venv-ov/
-*.png
-!models/*.png
-kernel.errors.txt

README.md

@@ -63,10 +63,31 @@ context leaks onto the wrong card:
 | `--min-aspect` / `--max-aspect` | 0.45 / 1.6 | face box W/H ratio window — effect blobs rarely keep face proportions |
 | `--edge-margin` | 0 | require faces this far inside frame edges; `0` accepts edge-touching faces (original hard-rejected them) |
 | `--scene-threshold` | 30.0 | PySceneDetect ContentDetector threshold (same as original) |
+| `--probe-stride` | 1 | within a scene still needing a face, run inference every Nth frame instead of every frame (1 = original behavior). Decode stays frame-perfect; only inference is thinned |
+| `--probe-max-per-scene` | 4 | with `--probe-stride > 1`, probes per scene before giving up and waiting for the next scene start |
 | `--imgsz` | 1280 | inference size; model was trained at 1280, `640` is faster at some recall cost |
 | `--all-files` | off | also scan OP/ED files. Off by default: music videos are full of stylized graphics (feather storms, transformation emblems) that false-positive as face-shaped; episodes carry the real character close-ups |
 | `--preview-dir` | off | also write annotated copies (green boxes) of every saved frame for eyeball QC |
 
+### Probe scheduling (thinned inference)
+
+The original loop predicts on *every* frame until a scene yields a face —
+free on a 3090 (~0.2 s/predict), hopeless on an iGPU (~5 s/img). With
+`--probe-stride N` the decode stays frame-perfect but inference runs only
+at scene starts and then every Nth frame, giving up after
+`--probe-max-per-scene` probes and waiting for the next scene. Ported from
+the `feat/igpu-ov` branch to main (Oct 28) so both platforms share one
+schedule.
+
+| mode (same 43k-frame special, 3090 GPU1) | wall time | stills |
+|-------------------------------------------|-----------|--------|
+| default (every frame)                     | 6m11s     | 475    |
+| `--probe-stride 25 --probe-max-per-scene 4` | 2m34s   | 459 (96.6%) |
+
+Use `25`/`4` for real runs; stride 1 keeps byte-identical behavior, so
+older extracted corpora are unaffected. Accepted trade-off: a face visible
+for less than ~stride frames between probes can slip through.
+
 ## Notes
 
 - Runs every frame of every scene until a face qualifies, then waits

anime_facesaver.py

@@ -177,10 +177,6 @@ def resolve_gpu(gpu_arg):
     if gpu_arg.lower() == "cpu":
         os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
         return "cpu"
-    if gpu_arg.lower() == "igpu":
-        # Intel iGPU via OpenVINO (Level Zero) — no CUDA, no torch pinning.
-        # OVFaceDetector takes over model loading in main().
-        return "igpu"
     if gpu_arg.lower() == "auto":
         return None  # let ultralytics/torch decide
     os.environ["CUDA_VISIBLE_DEVICES"] = gpu_arg
@@ -252,61 +248,6 @@ def face_qualifies(box, frame_shape, min_width, min_height,
     return True
 
 
-class AvFrameReader:
-    """Forward-only frame reader with a cv2.VideoCapture-compatible
-    surface (read/release). PyAV ships its own ffmpeg build with libdav1d:
-    the ffmpeg bundled inside opencv-python has no software AV1 decoder,
-    and Alder Lake-N has no AV1 hw decode, so BD AV1 rips (e.g. SavI0r)
-    decode 0 frames through cv2. scenedetect likewise prefers its pyav
-    backend once av is installed, fixing scene detection on the same files.
-    """
-
-    def __init__(self, path):
-        import av
-        self._c = av.open(path)
-        self._vs = self._c.streams.video[0]
-        self._vs.thread_type = "AUTO"
-        self._iter = self._c.decode(self._vs)
-
-    @property
-    def frame_count(self):
-        vs = self._vs
-        if vs.frames:
-            return int(vs.frames)
-        # MKV headers often leave stream.frames unset; estimate from the
-        # container duration (it is this file's total decode work anyway).
-        dur = self._c.duration  # microseconds, or None
-        if dur and vs.average_rate:
-            return int(float(dur) / 1e6 * float(vs.average_rate))
-        return 0
-
-    def read(self):
-        try:
-            frame = next(self._iter)
-        except StopIteration:
-            return False, None
-        return True, frame.to_ndarray(format="bgr24")
-
-    def release(self):
-        try:
-            self._c.close()
-        except Exception:
-            pass
-
-
-def open_reader(video_path):
-    """PyAV when available (AV1-capable), else plain cv2."""
-    import cv2
-    try:
-        return AvFrameReader(video_path)
-    except ImportError:
-        pass
-    cap = cv2.VideoCapture(video_path)
-    if not cap.isOpened():
-        return None
-    return cap
-
-
 def process_video(video_path, output_dir, args, model, device, draw,
                   output_start=0):
     from scenedetect import open_video, SceneManager, ContentDetector
@@ -315,12 +256,6 @@ def process_video(video_path, output_dir, args, model, device, draw,
 
     try:
         import cv2
-        try:
-            import av  # noqa: F401
-            # scenedetect defaults to backend="opencv", which cannot decode
-            # AV1 in this env; pyav ships its own ffmpeg with libdav1d.
-            video = open_video(video_path, backend="pyav")
-        except ImportError:
         video = open_video(video_path)
         scene_manager = SceneManager()
         scene_manager.add_detector(
@@ -330,18 +265,14 @@ def process_video(video_path, output_dir, args, model, device, draw,
               f"{video_path}: {e}")
         return output_start
 
-    cap = open_reader(video_path)
-    if cap is None:
+    cap = cv2.VideoCapture(video_path)
+    if not cap.isOpened():
         print(f"Error opening video file {video_path}")
         return output_start
 
-    frame_count = getattr(cap, "frame_count", 0)
-    if not frame_count and hasattr(cap, "get"):
     frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
 
     try:
-        print(f"{video_name}: detecting scenes "
-              "(silent for several minutes on CPU decode)...", flush=True)
         scene_manager.detect_scenes(video=video)
         scene_list = scene_manager.get_scene_list()
         scene_starts = {scene[0].get_frames() for scene in scene_list}
@@ -459,22 +390,6 @@ def main():
     from ultralytics import YOLO
     import cv2
 
-    if device == "igpu":
-        # Intel iGPU path: OpenVINO replaces ultralytics' torch predict loop.
-        # Ultralytics import stays (shared pre/post code paths), but weights
-        # load as ONNX through OVFaceDetector instead of torch.
-        from ov_detector import OVFaceDetector
-        onnx_path = os.path.splitext(args.model)[0] + ".onnx"
-        if not os.path.exists(onnx_path):
-            print(f"Error: --gpu igpu needs an exported ONNX at {onnx_path}\n"
-                  "  (one-time: YOLO('<pt>').export(format='onnx', "
-                  "dynamic=True, simplify=True, opset=17))")
-            sys.exit(1)
-        model = OVFaceDetector(onnx_path, device="GPU", imgsz=args.imgsz)
-        names = model.names
-        print("Device: igpu (OpenVINO, Intel iGPU via Level Zero)")
-        print(f"Model loaded (OpenVINO GPU). Classes: {names}")
-    else:
     if device == "cuda:0" and not torch.cuda.is_available():
         print("CUDA requested but not visible after pinning "
               f"(--gpu {args.gpu}). Falling back to CPU.")

find_dupes.py

@@ -111,42 +111,53 @@ def main():
             print(f"  {i+1}/{len(files)}", flush=True)
 
     names = list(data)
-    # strict (complete-linkage) clustering — NO transitive chaining:
-    # seed with the ungrouped image having most neighbours, take its
-    # within-threshold neighbourhood, then keep only members pairwise
-    # within threshold. Union-find chaining merges unrelated frames
-    # after enough hops (verified failure: 90-frame 'group' of six
-    # different scenes).
     n = len(names)
-    print(f"Pairwise compare {n}x{n} (numpy hamming) ...")
+    print(f"Pairwise compare {n}x{n} (streaming hamming, chunked) ...")
+
+    # Memory-safe at 30k+ scale under ~4 GB cgroup caps (verified: the
+    # full n^2 uint64 xor matrix — 8 GB at n=32k — got a background run
+    # oom-killed in a hermes-worker systemd scope on 2026-10-29). Row
+    # chunks of 1024 keep peak temps at a few hundred MB; only the final
+    # bool close-matrix (n^2 bytes) stays resident. uint8 distances
+    # suffice (max 64) via np.bitwise_count (numpy>=2.3) or a per-byte
+    # popcount LUT.
+    def hamming_chunk(mat, rows):
+        x = mat[rows, None] ^ mat[None, :]
+        if hasattr(np, "bitwise_count"):
+            ham = np.bitwise_count(x)
+        else:
+            bc = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint8)
+            x8 = x.view(np.uint8).reshape(x.shape + (8,))
+            ham = np.zeros(x.shape, dtype=np.uint8)
+            for b in range(8):
+                ham += bc[x8[..., b]]
+            del x8
+        del x
+        return ham
+
     dmat = np.array([data[f][0] for f in names], dtype=np.uint64)
-    # hamming distances via xor + popcount (work in int64; numpy has no
-    # sane uint64 shift-add promotion)
-    x = (dmat[:, None] ^ dmat[None, :]).astype(np.int64)
-    ham = np.zeros(x.shape, dtype=np.int16)
-    for b in range(64):
-        ham += ((x >> b) & 1).astype(np.int16)
+    amat = np.array([data[f][1] for f in names], dtype=np.uint64)
+    max_a = args.max_adist if args.max_adist is not None else 8
+    CH = 1024
+    close = np.zeros((n, n), dtype=bool)
+    for s in range(0, n, CH):
+        rows = slice(s, min(s + CH, n))
+        close[rows] = ((hamming_chunk(dmat, rows) <= args.max_dist)
+                       & (hamming_chunk(amat, rows) <= max_a))
+        if (s // CH) % 8 == 0:
+            print(f"  hashes {min(s + CH, n)}/{n}", flush=True)
+    del dmat, amat
 
     # combined close matrix: dHash AND aHash AND color-histogram gates.
     # dHash alone has a false-positive floor (verified: distinct frames
     # at distance <=10); a second, differently-computed hash plus a
     # color signature is what separates 'same frame replayed' from
-    # 'same show, different shot'.
-    close = ham <= args.max_dist
-
-    amat = np.array([data[f][1] for f in names], dtype=np.uint64)
-    xa = (amat[:, None] ^ amat[None, :]).astype(np.int64)
-    aham = np.zeros(xa.shape, dtype=np.int16)
-    for b in range(64):
-        aham += ((xa >> b) & 1).astype(np.int16)
-    close &= aham <= (args.max_adist
-                      if args.max_adist is not None else 8)
-
+    # 'same show, different shot'. Color L1 is computed per row only for
+    # rows that survived both hashes — few rows at real corpus scale,
+    # and it replaces the old n^2 float32 matrix (4 GB at n=32k).
     hmat = np.stack([data[f][3] for f in names])  # N x 192 hist rows
-    cdist = np.zeros((n, n), dtype=np.float32)
-    for i in range(n):  # row-wise keeps it ~seconds and RAM-light
-        cdist[i] = np.abs(hmat - hmat[i]).sum(axis=1)
-    close &= cdist <= args.max_cdist
+    for i in np.flatnonzero(close.any(axis=1)):
+        close[i] &= np.abs(hmat - hmat[i]).sum(axis=1) <= args.max_cdist
     np.fill_diagonal(close, False)
 
     # strict (complete-linkage) clustering — NO transitive chaining:

ov_detector.py

@@ -1,136 +0,0 @@
-"""OpenVINO-backed anime-face detector for anime_facesaver (iGPU path).
-
-Used when --gpu igpu: replaces ultralytics' torch predict loop with a direct
-OpenVINO inference request on Intel GPUs (Level Zero / ANV). Pre/post-processing
-emulates ultralytics YOLO defaults: 1280 letterbox (stride 32), conf/iou gates,
-class-agnostic-free single-class NMS.
-
-On chibi (i3-N305): GPU fp32 measured 5.2 s/img vs CPU 14.5 s/img
-(2026-10-20, models/yolov8x6_animeface.onnx, 25 real stills).
-"""
-import os
-
-import numpy as np
-
-
-def letterbox(img, new_shape=1280, stride=32):
-    """Square ultralytics letterbox (auto=False): fit, then center-pad to
-    new_shape x new_shape. The GPU plugin compiles one static shape this way
-    and sidesteps dynamic-shape concat failures on non-square inputs
-    (measured: (1,3,736,1280) fails inside the graph; 1280x1280 always works).
-
-    Returns (blob CHW float32 /255, scale, (pad_left, pad_top)).
-    Inverse map for boxes: x_orig = (x_lb - pad_left) / scale.
-    """
-    import cv2
-
-    h, w = img.shape[:2]
-    r = min(new_shape / h, new_shape / w)
-    nw, nh = int(round(w * r)), int(round(h * r))
-    interp = cv2.INTER_AREA if r < 1 else cv2.INTER_LINEAR
-    resized = cv2.resize(img, (nw, nh), interpolation=interp)
-    canvas = np.full((new_shape, new_shape, 3), 114, dtype=np.uint8)
-    top, left = (new_shape - nh) // 2, (new_shape - nw) // 2
-    canvas[top:top + nh, left:left + nw] = resized
-    blob = canvas[..., ::-1].transpose(2, 0, 1)[None].astype(np.float32) / 255.0
-    return blob, r, (left, top)
-
-
-def numpy_ceil_div(a, b):
-    return -(-a // b)
-
-
-def nms_numpy(boxes, scores, iou_thres):
-    """Plain greedy NMS; returns kept indices."""
-    if len(boxes) == 0:
-        return []
-    x1, y1, x2, y2 = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
-    areas = (x2 - x1) * (y2 - y1)
-    order = scores.argsort()[::-1]
-    keep = []
-    while order.size:
-        i = order[0]
-        keep.append(int(i))
-        xx1 = np.maximum(x1[i], x1[order[1:]])
-        yy1 = np.maximum(y1[i], y1[order[1:]])
-        xx2 = np.minimum(x2[i], x2[order[1:]])
-        yy2 = np.minimum(y2[i], y2[order[1:]])
-        inter = np.clip(xx2 - xx1, 0, None) * np.clip(yy2 - yy1, 0, None)
-        iou = inter / (areas[i] + areas[order[1:]] - inter + 1e-9)
-        order = order[1:][iou <= iou_thres]
-    return keep
-
-
-class OVFaceDetector:
-    """model.predict(frame_bgr, conf=, iou=, imgsz=, device=) compatible shim.
-
-    Returns a one-element list with .boxes being a Boxes-like object exposing
-    .xyxy (numpy, already denormalized to original-frame pixels) and .conf.
-    """
-
-    class _Boxes:
-        """numpy-backed stand-in for ultralytics Boxes (.xyxy.cpu().numpy())."""
-        class _Lazy:
-            def __init__(self, arr):
-                self._arr = arr
-
-            def cpu(self):
-                return self
-
-            def numpy(self):
-                return self._arr
-
-            def int(self):
-                return OVFaceDetector._Boxes._Lazy(self._arr.astype(np.int32))
-
-            def tolist(self):
-                return self._arr.tolist()
-
-        def __init__(self, xyxy, conf):
-            self.xyxy = self._Lazy(xyxy)
-            self.conf = conf
-
-    class _Result:
-        def __init__(self, boxes):
-            self.boxes = boxes
-
-    def __init__(self, model_path, device="GPU", imgsz=1280):
-        import openvino as ov
-
-        core = ov.Core()
-        if device != "CPU" and device not in core.available_devices:
-            raise RuntimeError(
-                f"OpenVINO device {device!r} not available "
-                f"(have {core.available_devices}). Is the render-group fix "
-                f"active (sg render / re-login) and intel-level-zero-gpu "
-                f"installed?")
-        self._ov = ov
-        self._req = core.compile_model(model_path, device).create_infer_request()
-        self.imgsz = imgsz
-        self.names = {0: "face"}
-
-    def predict(self, frame, conf=0.25, iou=0.45, imgsz=None, device=None,
-                verbose=False):
-        imgsz = imgsz or self.imgsz
-        blob, scale, (pad_l, pad_t) = letterbox(frame, imgsz)
-        out = self._req.infer([self._ov.Tensor(blob)])
-        pred = list(out.values())[0]  # [1, 5, N]: cx cy w h conf cls
-        pred = pred[0].T  # [N, 5]
-        scores = pred[:, 4]
-        cand = pred[scores >= conf]
-        if len(cand) == 0:
-            return [self._Result(self._Boxes(
-                np.zeros((0, 4), np.float32), np.zeros((0,), np.float32)))]
-        cx, cy, w, h = cand[:, 0], cand[:, 1], cand[:, 2], cand[:, 3]
-        boxes = np.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], 1)
-        keep = nms_numpy(boxes, cand[:, 4], iou)
-        boxes, scores = boxes[keep], cand[keep, 4]
-        # map letterbox coords back to original frame pixels
-        boxes[:, [0, 2]] -= pad_l
-        boxes[:, [1, 3]] -= pad_t
-        boxes = boxes / scale
-        h0, w0 = frame.shape[:2]
-        boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, w0)
-        boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, h0)
-        return [self._Result(self._Boxes(boxes.astype(np.float32),
-                                         scores.astype(np.float32)))]

validate_ov.py

@@ -1,41 +0,0 @@
-"""Cross-validate OVFaceDetector vs ultralytics CPU baseline on real stills."""
-import glob, sys, time
-import numpy as np, cv2
-
-imgs = sorted(glob.glob('/home/alex/Documents/ov-bench/imgs/*.png'))
-sys.path.insert(0, '.')
-from ov_detector import OVFaceDetector
-
-# baseline: ultralytics torch CPU (deterministic on CPU)
-from ultralytics import YOLO
-base = YOLO('models/yolov8x6_animeface.pt')
-
-ovd = OVFaceDetector('models/yolov8x6_animeface.onnx', device='GPU', imgsz=1280)
-
-def iou(a, b):
-    xx1, yy1 = max(a[0], b[0]), max(a[1], b[1])
-    xx2, yy2 = min(a[2], b[2]), min(a[3], b[3])
-    inter = max(0, xx2-xx1) * max(0, yy2-yy1)
-    ua = (a[2]-a[0])*(a[3]-a[1]) + (b[2]-b[0])*(b[3]-b[1]) - inter
-    return inter / max(ua, 1e-9)
-
-ious, n_match, n_only_base, n_only_ov = [], 0, 0, 0
-t0=time.time()
-for p in imgs:
-    fr = cv2.imread(p)
-    rb = base.predict(fr, conf=0.25, imgsz=1280, device='cpu', verbose=False)[0]
-    ro = ovd.predict(fr, conf=0.25, imgsz=1280, verbose=False)[0]
-    bb = rb.boxes.xyxy.cpu().numpy() if rb.boxes is not None else np.zeros((0,4))
-    bo = ro.boxes.xyxy.numpy()
-    used = set()
-    for box in bb:
-        best, bi = 0, -1
-        for j, ob in enumerate(bo):
-            v = iou(box, ob)
-            if v > best: best, bi = v, j
-        if best > 0.5:
-            ious.append(best); n_match += 1; used.add(bi)
-        else: n_only_base += 1
-    n_only_ov += len(bo) - len(used)
-dt = time.time()-t0
-print(f"imgs={len(imgs)} matched={n_match} (mean IoU {np.mean(ious):.3f}) only_base={n_only_base} only_ov={n_only_ov} [{dt:.0f}s total]")