what are these

README.md

@@ -34,6 +34,8 @@ The pipeline operates in a "Fail-Fast" cascade with **early exit** optimization:
   - **Active Mode (Max FPS)**: Switches to full speed when activity is detected.
   - **Cooldown**: Stays active for 2 seconds after people leave frame.
 - **Non-Maximum Suppression (NMS)**: Cleans up overlapping boxes per-class.
+- **Weighted Boxes Fusion (WBF)**: Alternative to NMS that merges overlapping boxes for better localization.
+- **Inference Budget Caps**: Limits expensive model calls (e.g., max 3 person crops, max 5 verifications).
 - **Smart Event Recording**: Captures 2s before + 3s after each smoking event as separate clips.
 - **Live Display**: Real-time visualization window with 'q' to quit.
 - **Metrics Tracking**: FPS, latency, detection counts with summary on exit.
@@ -179,7 +181,8 @@ gp/
 │   │   └── api.py           # APIPersonDetector, VLMVerifier (cloud stages)
 │   │
 │   └── ops/                 # Algorithms & utilities
-│       ├── nms.py           # Non-Maximum Suppression
+│       ├── nms.py           # NMS + Weighted Boxes Fusion (WBF)
+│       ├── selection.py     # Budget-aware box selection (top-K, merge)
 │       ├── geometry.py      # Box expansion, zone filtering
 │       └── drawing.py       # Visualization helpers
 │
@@ -207,6 +210,12 @@ class PipelineConfig:
     idle_interval: float = 0.5   # 2 FPS when idle
     active_interval: float = 0.0 # Max FPS when active
     cooldown_frames: int = 60    # Stay active after detection
+
+    # Inference budget caps
+    max_person_crops: int = 3       # Max cig model runs (0 = unlimited)
+    person_priority: str = "area"   # "area", "confidence", "area_conf"
+    merge_people: bool = False      # Merge overlapping people into regions
+    max_cig_verifications: int = 5  # Max verifier runs (0 = unlimited)
 ```
 
 Pass a custom config to the pipeline:
@@ -232,6 +241,26 @@ Key tuning parameters:
 - `active_interval`: Set to 0.0 for max detection rate
 - `cig_high_confidence`: Lower for faster alerts, higher for fewer false positives
 
+### Inference Budget
+
+When using lightweight (nano) models that produce many false positives, cap expensive downstream calls:
+
+```python
+config = PipelineConfig(
+    max_person_crops=3,        # Only process 3 people max per frame
+    person_priority="area",    # Prioritize largest (closest to camera)
+    merge_people=True,         # Merge overlapping people into regions
+    merge_people_iou=0.3,      # IoU threshold for merging
+    max_cig_verifications=5,   # Only verify top 5 cigarette candidates
+)
+```
+
+| Priority | Strategy |
+|----------|----------|
+| `area` | Largest boxes first (closer to camera, more detail) |
+| `confidence` | Highest scoring boxes first |
+| `area_conf` | Balanced: area × confidence |
+
 ## 🔌 Pluggable Stage System
 
 The `--staged` flag enables the modular stage pipeline, allowing you to mix local models with cloud APIs:

pipeline/config.py

@@ -2,6 +2,8 @@
 
 from dataclasses import dataclass
 
+from pipeline.ops.selection import SelectPriority
+
 
 @dataclass
 class PipelineConfig:
@@ -32,6 +34,20 @@ class PipelineConfig:
     active_interval: float = 0.0  # Seconds between inference when active
     cooldown_frames: int = 60  # Frames to stay active after last detection
 
+    # === Inference Budget Caps ===
+    # Limits expensive model calls when there are too many detections
+
+    # Max person crops to run cig detection on (0 = unlimited)
+    max_person_crops: int = 3
+    # How to prioritize which people to process: "area", "confidence", "area_conf"
+    person_priority: SelectPriority = "area"
+    # Merge overlapping people into regions instead of processing separately
+    merge_people: bool = False
+    merge_people_iou: float = 0.3
+
+    # Max cigarette candidates to run verifier on (0 = unlimited)
+    max_cig_verifications: int = 5
+
 
 # Default config instance
 DEFAULT_CONFIG = PipelineConfig()

pipeline/ops/__init__.py

@@ -18,6 +18,11 @@ from pipeline.ops.geometry import (
     point_in_polygon,
 )
 from pipeline.ops.nms import nms, nms_by_class, wbf, wbf_by_class
+from pipeline.ops.selection import (
+    merge_overlapping,
+    select_for_verification,
+    select_top_k,
+)
 
 __all__ = [
     # NMS / Box Fusion
@@ -25,6 +30,10 @@ __all__ = [
     "nms_by_class",
     "wbf",
     "wbf_by_class",
+    # Selection / Budget
+    "select_top_k",
+    "select_for_verification",
+    "merge_overlapping",
     # Geometry
     "expand_box",
     "point_in_box",

pipeline/ops/selection.py

@@ -0,0 +1,174 @@
+"""Budget-aware box selection for capping expensive inference calls."""
+
+from typing import List, Literal
+
+from pipeline.types import BoundingBox, Detection
+
+type SelectPriority = Literal["area", "confidence", "area_conf"]
+
+
+def select_top_k(
+    detections: List[Detection],
+    k: int,
+    diversity_iou: float = 0.5,
+    priority: SelectPriority = "area",
+) -> List[Detection]:
+    """
+    Select top K detections with spatial diversity.
+
+    Picks the K "best" boxes while ensuring they don't overlap too much.
+    This caps expensive downstream inference (e.g., max 3 person crops → cig model).
+
+    Args:
+        detections: Candidate detections
+        k: Maximum number to select
+        diversity_iou: Reject candidates overlapping selected boxes above this IoU
+        priority: How to rank candidates:
+            - "area": Larger boxes first (closer to camera, more detail)
+            - "confidence": Higher confidence first
+            - "area_conf": area * confidence (balanced)
+
+    Returns:
+        Up to K diverse, high-priority detections
+    """
+    if not detections or k <= 0:
+        return []
+
+    # Score each detection
+    def score(det: Detection) -> float:
+        box = det.box
+        # area = (box.x2 - box.x1) * (box.y2 - box.y1)
+        area = box.area
+        if priority == "area":
+            return area
+        elif priority == "confidence":
+            return det.score
+        else:  # area_conf
+            return area * det.score
+
+    # Sort by priority (highest first)
+    ranked = sorted(detections, key=score, reverse=True)
+
+    selected = []
+    for det in ranked:
+        if len(selected) >= k:
+            break
+
+        # Check diversity: don't select if overlaps too much with already selected
+        dominated = False
+        for sel in selected:
+            if det.box.iou(sel.box) > diversity_iou:
+                dominated = True
+                break
+
+        if not dominated:
+            selected.append(det)
+
+    return selected
+
+
+def merge_overlapping(
+    detections: List[Detection],
+    iou_threshold: float = 0.3,
+) -> List[Detection]:
+    """
+    Merge overlapping boxes into larger regions.
+
+    Instead of processing each overlapping person separately, merge them
+    into one bigger crop. Good when people are clustered together.
+
+    Args:
+        detections: List of detections (e.g., people)
+        iou_threshold: Merge boxes with IoU above this
+
+    Returns:
+        Merged detections (fewer, larger boxes)
+    """
+    if not detections:
+        return []
+
+    # Sort by confidence
+    dets = sorted(detections, key=lambda d: d.score, reverse=True)
+
+    clusters: List[List[Detection]] = []
+
+    for det in dets:
+        merged = False
+        for cluster in clusters:
+            # Check if this det overlaps with any box in cluster
+            for member in cluster:
+                if det.box.iou(member.box) >= iou_threshold:
+                    cluster.append(det)
+                    merged = True
+                    break
+            if merged:
+                break
+
+        if not merged:
+            clusters.append([det])
+
+    # Create merged boxes from clusters
+    result = []
+    for cluster in clusters:
+        merged_det = _merge_cluster(cluster)
+        result.append(merged_det)
+
+    return result
+
+
+def _merge_cluster(cluster: List[Detection]) -> Detection:
+    """Merge a cluster of detections into one encompassing box."""
+    if len(cluster) == 1:
+        return cluster[0]
+
+    # Bounding box that contains all
+    x1 = min(d.box.x1 for d in cluster)
+    y1 = min(d.box.y1 for d in cluster)
+    x2 = max(d.box.x2 for d in cluster)
+    y2 = max(d.box.y2 for d in cluster)
+
+    # Confidence: max of cluster (could also use average)
+    conf = max(d.score for d in cluster)
+
+    # Use label from highest confidence detection
+    best = max(cluster, key=lambda d: d.score)
+
+    return Detection(
+        box=BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, conf=conf),
+        score=conf,
+        label=best.label,
+        metadata={
+            "merged_count": len(cluster),
+            "original_boxes": [
+                (d.box.x1, d.box.y1, d.box.x2, d.box.y2) for d in cluster
+            ],
+        },
+    )
+
+
+def select_for_verification(
+    detections: List[Detection],
+    k: int,
+    min_confidence: float = 0.0,
+) -> List[Detection]:
+    """
+    Select top K cigarette candidates for expensive verification.
+
+    Simple confidence-based selection for the verifier stage.
+    Higher confidence cigs are more likely real → verify those first.
+
+    Args:
+        detections: Cigarette candidates
+        k: Max verifications to run
+        min_confidence: Skip candidates below this
+
+    Returns:
+        Top K candidates by confidence
+    """
+    if not detections or k <= 0:
+        return []
+
+    filtered = [d for d in detections if d.score >= min_confidence]
+    ranked = sorted(filtered, key=lambda d: d.score, reverse=True)
+
+    return ranked[:k]

pipeline/stages/api.py

@@ -148,6 +148,7 @@ class APIPersonDetector(APIStageBase):
         resp.raise_for_status()
         return resp.json()
 
+
 # haiku 4.5 is multimodal but i fear its going to ping 65 million images per second (we need cache control? who are we? why cant we upload video?)
 class VLMVerifier(APIStageBase):
     """

pipeline/stages/local.py

@@ -5,7 +5,12 @@ import time
 from typing import List, Optional
 
 from pipeline.config import DEFAULT_CONFIG, PipelineConfig
-from pipeline.ops import nms_by_class
+from pipeline.ops import (
+    merge_overlapping,
+    nms_by_class,
+    select_for_verification,
+    select_top_k,
+)
 from pipeline.protocols import Model, Verifier
 from pipeline.stages.protocol import StageContext, StageResult
 from pipeline.types import BoundingBox, Detection, Ticket
@@ -66,10 +71,39 @@ class LocalCigaretteDetector:
         cfg = self.cfg
         h_img, w_img = ctx.frame_data.shape[:2]
 
+        # === Budget: Limit how many people we process ===
+        people_to_process = ctx.people
+
+        if cfg.merge_people:
+            # Merge overlapping people into regions
+            people_to_process = merge_overlapping(
+                people_to_process, cfg.merge_people_iou
+            )
+            if len(people_to_process) < len(ctx.people):
+                logger.debug(
+                    "Merged %d people into %d regions",
+                    len(ctx.people),
+                    len(people_to_process),
+                )
+
+        if cfg.max_person_crops > 0 and len(people_to_process) > cfg.max_person_crops:
+            # Select top K diverse people by priority
+            people_to_process = select_top_k(
+                people_to_process,
+                k=cfg.max_person_crops,
+                priority=cfg.person_priority,
+            )
+            logger.debug(
+                "Budget: Processing %d/%d people (priority=%s)",
+                len(people_to_process),
+                len(ctx.people),
+                cfg.person_priority,
+            )
+
         all_cigs: List[Detection] = []
         early_hit = None
 
-        for person in ctx.people:
+        for person in people_to_process:
             # Expand person box for crop
             crop_box = self._expand_box(person.box, cfg.person_margin, w_img, h_img)
             ex1, ey1, ex2, ey2 = crop_box
@@ -195,12 +229,28 @@ class LocalVerifier:
         cfg = self.cfg
         h_img, w_img = ctx.frame_data.shape[:2]
 
+        # === Budget: Limit how many cigs we verify ===
+        cigs_to_verify = ctx.cigarettes
+
+        if (
+            cfg.max_cig_verifications > 0
+            and len(cigs_to_verify) > cfg.max_cig_verifications
+        ):
+            # Select top K by confidence (most likely real → verify first)
+            cigs_to_verify = select_for_verification(
+                cigs_to_verify,
+                k=cfg.max_cig_verifications,
+                min_confidence=cfg.cig_min_confidence,
+            )
+            logger.debug(
+                "Budget: Verifying %d/%d cigarettes",
+                len(cigs_to_verify),
+                len(ctx.cigarettes),
+            )
+
         confirmed: list[tuple[Detection, float]] = []
 
-        for cig_det in ctx.cigarettes:
-            offset = cig_det.metadata.get("crop_offset", (0, 0))
-            ex1, ey1 = offset
-
+        for cig_det in cigs_to_verify:
             # Extract padded crop for classifier
             crop = self._extract_cig_crop(ctx.frame_data, cig_det, w_img, h_img)
             if crop.size == 0:

pipeline/stages/runner.py

@@ -60,7 +60,7 @@ class StagePipeline:
                     stage.name,
                     ctx.stop_reason,
                 )
-                continue # break?
+                continue  # break?
 
             try:
                 result = stage.process(ctx)

pipeline/types.py

@@ -10,7 +10,7 @@ class BoundingBox:
     y1: int
     x2: int
     y2: int
-    conf: float = 0.0
+    conf: float = 0.0  # legacy
 
     @property
     def area(self):