ON DIH CHEESE ts is just so food
This commit is contained in:
lda committed
1 parent
013c082447
commit
02e00cbaa2
15 files changed
+1228
-86
No files matched your search
+250
@@ -0,0 +1,250 @@
|
||||
"""
|
||||
Smoking Detection Pipeline Refinement
|
||||
-------------------------------------
|
||||
This module implements a cascaded detection pipeline for smoking detection.
|
||||
It relies on a "fail-fast" strategy:
|
||||
1. Detect Person (High recall, fast) -> If no person, skip.
|
||||
2. Detect Cigarette (Specialized, medium speed) -> If no cigarette candidate, skip.
|
||||
3. Verify Cigarette (High precision, slow) -> Crop and verify.
|
||||
4. (Optional) VLM Review -> Final arbiter for edge cases.
|
||||
|
||||
Design Principles:
|
||||
- Modularity: Each stage is independent.
|
||||
- Scalability: Can be distributed (Steps can run on workers).
|
||||
- Type Safety: Uses Python typing for clarity.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterator, List, Optional, Protocol
|
||||
|
||||
# --- Data Structures ---
|
||||
|
||||
|
||||
@dataclass
|
||||
class BoundingBox:
|
||||
x1: int
|
||||
y1: int
|
||||
x2: int
|
||||
y2: int
|
||||
|
||||
@property
|
||||
def area(self):
|
||||
return (self.x2 - self.x1) * (self.y2 - self.y1)
|
||||
|
||||
def crop(self, image: Any) -> Any:
|
||||
# Pseudo-implementation for cropping an image array
|
||||
# return image[self.y1:self.y2, self.x1:self.x2]
|
||||
return image
|
||||
|
||||
|
||||
@dataclass
|
||||
class Detection:
|
||||
box: BoundingBox
|
||||
score: float
|
||||
label: str
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Frame:
|
||||
"""Represents a single frame in the video stream."""
|
||||
|
||||
timestamp: float
|
||||
data: Any # numpy array or bytes
|
||||
frame_id: int
|
||||
detections: List[Detection] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Ticket:
|
||||
"""Represents a potential alert event."""
|
||||
|
||||
frame_id: int
|
||||
confidence: float
|
||||
evidence: List[Any] # List of images or crops
|
||||
stage_confirmed: str
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
# --- Protocols ---
|
||||
|
||||
|
||||
class Model(Protocol):
|
||||
def predict(self, input_data: Any) -> List[Detection]:
|
||||
"""Run inference and return detections."""
|
||||
...
|
||||
|
||||
|
||||
class Verifier(Protocol):
|
||||
def verify(self, input_data: Any) -> float:
|
||||
"""Return a confidence score (0.0 to 1.0) for the specific target."""
|
||||
...
|
||||
|
||||
|
||||
class StreamSource(Protocol):
|
||||
def stream(self) -> Iterator[Frame]:
|
||||
"""Yield frames from source."""
|
||||
...
|
||||
|
||||
|
||||
class AlertStore(Protocol):
|
||||
def store(self, ticket: Ticket):
|
||||
"""Persist the ticket."""
|
||||
...
|
||||
|
||||
|
||||
# --- Implementations (Mocks) ---
|
||||
|
||||
|
||||
class MockPersonDetector:
|
||||
def predict(self, input_data: Any) -> List[Detection]:
|
||||
# Simulate: 50% chance to see a person
|
||||
import random
|
||||
|
||||
if random.random() > 0.5:
|
||||
return [Detection(BoundingBox(0, 0, 100, 200), 0.85, "person")]
|
||||
return []
|
||||
|
||||
|
||||
class MockCigaretteDetector:
|
||||
def predict(self, input_data: Any) -> List[Detection]:
|
||||
# Simulate: 30% chance to see a cigarette-like object on a person
|
||||
import random
|
||||
|
||||
if random.random() > 0.7:
|
||||
return [Detection(BoundingBox(10, 10, 20, 20), 0.6, "cigarette_candidate")]
|
||||
return []
|
||||
|
||||
|
||||
class MockClassifier:
|
||||
def verify(self, input_data: Any) -> float:
|
||||
# High precision check
|
||||
return 0.95
|
||||
|
||||
|
||||
class MockVLM:
|
||||
def verify(self, input_data: Any) -> float:
|
||||
# "Reasoning" about the image
|
||||
return 0.1 # "I see a pen, not a cigarette"
|
||||
|
||||
|
||||
class ConsoleStore:
|
||||
def store(self, ticket: Ticket):
|
||||
print(
|
||||
f"[STORE] Persisting Alert! Conf: {ticket.confidence:.2f} @ Stage: {ticket.stage_confirmed}"
|
||||
)
|
||||
|
||||
|
||||
# --- The Pipeline ---
|
||||
|
||||
|
||||
class SmokingDetectionPipeline:
|
||||
def __init__(
|
||||
self,
|
||||
person_model: Model,
|
||||
cigarette_model: Model,
|
||||
verifier_model: Verifier,
|
||||
vlm_model: Optional[Verifier] = None,
|
||||
store: Optional[AlertStore] = None,
|
||||
):
|
||||
self.person_model = person_model
|
||||
self.cigarette_model = cigarette_model
|
||||
self.verifier_model = verifier_model
|
||||
self.vlm_model = vlm_model
|
||||
self.store = store or ConsoleStore()
|
||||
|
||||
# Buffer for temporal context (optional, for VLM or sequence logic)
|
||||
self.frame_buffer = deque(maxlen=30)
|
||||
|
||||
def process_stream(self, source: StreamSource):
|
||||
"""Main loop processing the video stream."""
|
||||
|
||||
# Batch processing strategy:
|
||||
# Instead of strict batches, we process frame-by-frame but can look back
|
||||
for frame in source.stream():
|
||||
self.frame_buffer.append(frame)
|
||||
self._process_frame(frame)
|
||||
|
||||
def _process_frame(self, frame: Frame):
|
||||
# --- Step 1: Broad Phase (Person Detection) ---
|
||||
# "Is there a human here?"
|
||||
# Optimization: Resize or skip frames if inference is slow
|
||||
people = self.person_model.predict(frame.data)
|
||||
|
||||
if not people:
|
||||
return # FAST EXIT
|
||||
|
||||
# --- Step 2: Narrow Phase (Cigarette Candidate) ---
|
||||
# "Is the human holding something small?"
|
||||
# We could optimize by only running on crops of the 'people'
|
||||
candidates = []
|
||||
for person in people:
|
||||
# ROI: Region of Interest (e.g., upper body or hands)
|
||||
person_crop = person.box.crop(frame.data)
|
||||
|
||||
cig_candidates = self.cigarette_model.predict(person_crop)
|
||||
if cig_candidates:
|
||||
candidates.append((person, cig_candidates))
|
||||
|
||||
if not candidates:
|
||||
return # EXIT
|
||||
|
||||
# --- Step 3: Verification (Classifier) ---
|
||||
# "Is that small thing actually a cigarette?"
|
||||
confirmed_hits = []
|
||||
for person, cigs in candidates:
|
||||
for cig in cigs:
|
||||
# Crop just the object
|
||||
obj_crop = cig.box.crop(frame.data)
|
||||
|
||||
score = self.verifier_model.verify(obj_crop)
|
||||
|
||||
if score > 0.8: # High confidence threshold
|
||||
confirmed_hits.append((person, cig, score))
|
||||
elif score > 0.4 and self.vlm_model:
|
||||
# Ambiguous case: Send to VLM?
|
||||
# "Why bring a gatling to a sword fight" -> Only when the sword gets stuck.
|
||||
vlm_score = self.vlm_model.verify(
|
||||
obj_crop
|
||||
) # Or pass seq of frames?
|
||||
if vlm_score > 0.8:
|
||||
confirmed_hits.append((person, cig, vlm_score))
|
||||
# --- Step 4: Action ---
|
||||
if confirmed_hits:
|
||||
best_hit = max(confirmed_hits, key=lambda x: x[2])
|
||||
ticket = Ticket(
|
||||
frame_id=frame.frame_id,
|
||||
confidence=best_hit[2],
|
||||
evidence=[frame.data], # Store frame reference
|
||||
stage_confirmed="VLM"
|
||||
if self.vlm_model and best_hit[2] == 0.1
|
||||
else "Verifier",
|
||||
)
|
||||
self.store.store(ticket)
|
||||
|
||||
|
||||
# --- Usage Example ---
|
||||
|
||||
|
||||
def run_demo():
|
||||
# Mock stream source
|
||||
class CameraSource:
|
||||
def stream(self):
|
||||
for i in range(100):
|
||||
yield Frame(timestamp=time.time(), data=f"Frame_{i}", frame_id=i)
|
||||
time.sleep(0.05) # simulate 20fps
|
||||
|
||||
pipeline = SmokingDetectionPipeline(
|
||||
person_model=MockPersonDetector(),
|
||||
cigarette_model=MockCigaretteDetector(),
|
||||
verifier_model=MockClassifier(),
|
||||
vlm_model=None, # Disabled for now as per "why bring gatling..."
|
||||
)
|
||||
|
||||
pipeline.process_stream(CameraSource())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_demo()
|
||||
Reference in new issue
Block a user