running version, only with bit delay

app.py

@@ -12,7 +12,7 @@ from streamlit_toggle import st_toggle_switch
 import pandas as pd
 from tools.nametypes import Stats, Detection
 from pathlib import Path
-from tools.utils import get_ice_servers, download_file, display_match, rgb
+from tools.utils import get_ice_servers, download_file, display_match, rgb, format_list
 from tools.face_recognition import (
     detect_faces,
     align_faces,
@@ -54,12 +54,19 @@ with st.sidebar:
         track_color=rgb(50, 50, 50),
     )
 
-    st.markdown("## Webcam")
+    st.markdown("## Webcam & Stream")
     resolution = st.selectbox(
         "Webcam Resolution",
         [(1920, 1080), (1280, 720), (640, 360)],
         index=2,
     )
+    st.markdown("Note: To change the resolution, you have to restart the stream.")
+
+    ice_server = st.selectbox("ICE Server", ["twilio", "metered"], index=0)
+    st.markdown(
+        "Note: metered is a free server with limited bandwidth, and can take a while to connect. Twilio is a paid service and is payed by me, so please don't abuse it."
+    )
+
     st.markdown("## Face Detection")
     max_faces = st.number_input("Maximum Number of Faces", value=2, min_value=1)
     detection_confidence = st.slider(
@@ -68,17 +75,13 @@ with st.sidebar:
     tracking_confidence = st.slider(
         "Min Tracking Confidence", min_value=0.0, max_value=1.0, value=0.9
     )
-    on_draw = st_toggle_switch(
-        "Show Drawings",
-        key="show_drawings",
-        default_value=True,
-        active_color=rgb(255, 75, 75),
-        track_color=rgb(100, 100, 100),
-    )
     st.markdown("## Face Recognition")
     similarity_threshold = st.slider(
         "Similarity Threshold", min_value=0.0, max_value=2.0, value=0.67
     )
+    st.markdown(
+        "This sets a maximum distance for the cosine similarity between the embeddings of the detected face and the gallery images. If the distance is below the threshold, the face is recognized as the gallery image with the lowest distance. If the distance is above the threshold, the face is not recognized."
+    )
 
 download_file(
     MODEL_URL,
@@ -94,6 +97,16 @@ else:
     face_recognition_model = tflite.Interpreter(model_path=MODEL_LOCAL_PATH.as_posix())
     st.session_state[cache_key] = face_recognition_model
 
+# Session-specific caching of the face recognition model
+cache_key = "face_id_model_gal"
+if cache_key in st.session_state:
+    face_recognition_model_gal = st.session_state[cache_key]
+else:
+    face_recognition_model_gal = tflite.Interpreter(
+        model_path=MODEL_LOCAL_PATH.as_posix()
+    )
+    st.session_state[cache_key] = face_recognition_model_gal
+
 # Session-specific caching of the face detection model
 cache_key = "face_detection_model"
 if cache_key in st.session_state:
@@ -112,58 +125,58 @@ detections_queue: "queue.Queue[List[Detection]]" = queue.Queue()
 
 
 def video_frame_callback(frame: av.VideoFrame) -> av.VideoFrame:
-    detections = None
+    # Initialize detections
+    detections = []
+
+    # Initialize stats
+    stats = Stats()
+
+    # Start timer for FPS calculation
     frame_start = time.time()
 
     # Convert frame to numpy array
     frame = frame.to_ndarray(format="rgb24")
 
-    # Get frame resolution
+    # Get frame resolution and add to stats
     resolution = frame.shape
+    stats = stats._replace(resolution=resolution)
 
-    start = time.time()
     if face_rec_on:
+        # Run face detection
+        start = time.time()
         detections = detect_faces(frame, face_detection_model)
-    time_detection = (time.time() - start) * 1000
+        stats = stats._replace(num_faces=len(detections) if detections else 0)
+        stats = stats._replace(detection=(time.time() - start) * 1000)
 
-    start = time.time()
-    if face_rec_on:
+        # Run face alignment
+        start = time.time()
         detections = align_faces(frame, detections)
-    time_normalization = (time.time() - start) * 1000
+        stats = stats._replace(alignment=(time.time() - start) * 1000)
 
-    start = time.time()
-    if face_rec_on:
+        # Run inference
+        start = time.time()
         detections = inference(detections, face_recognition_model)
-    time_inference = (time.time() - start) * 1000
+        stats = stats._replace(inference=(time.time() - start) * 1000)
 
-    start = time.time()
-    if face_rec_on:
+        # Run face recognition
+        start = time.time()
         detections = recognize_faces(detections, gallery, similarity_threshold)
-    time_recognition = (time.time() - start) * 1000
+        stats = stats._replace(recognition=(time.time() - start) * 1000)
 
-    start = time.time()
-    if face_rec_on and on_draw:
+        # Draw detections
+        start = time.time()
         frame = draw_detections(frame, detections)
-    time_drawing = (time.time() - start) * 1000
+        stats = stats._replace(drawing=(time.time() - start) * 1000)
 
     # Convert frame back to av.VideoFrame
     frame = av.VideoFrame.from_ndarray(frame, format="rgb24")
 
-    # Put detections, stats and timings into queues (to be accessible by other thread)
-    if face_rec_on:
-        detections_queue.put(detections)
-    stats_queue.put(
-        Stats(
-            fps=1 / (time.time() - frame_start),
-            resolution=resolution,
-            num_faces=len(detections) if detections else 0,
-            detection=time_detection,
-            normalization=time_normalization,
-            inference=time_inference,
-            recognition=time_recognition,
-            drawing=time_drawing,
-        )
-    )
+    # Calculate FPS and add to stats
+    stats = stats._replace(fps=1 / (time.time() - frame_start))
+
+    # Send data to other thread
+    detections_queue.put(detections)
+    stats_queue.put(stats)
 
     return frame
 
@@ -176,7 +189,7 @@ gallery = st.sidebar.file_uploader(
     "Upload images to gallery", type=["png", "jpg", "jpeg"], accept_multiple_files=True
 )
 if gallery:
-    gallery = process_gallery(gallery, face_detection_model, face_recognition_model)
+    gallery = process_gallery(gallery, face_detection_model, face_recognition_model_gal)
     st.sidebar.markdown("**Gallery Images**")
     st.sidebar.image(
         [identity.image for identity in gallery],
@@ -190,7 +203,7 @@ stats = st.empty()
 ctx = webrtc_streamer(
     key="FaceIDAppDemo",
     mode=WebRtcMode.SENDRECV,
-    rtc_configuration={"iceServers": get_ice_servers("twilio")},
+    rtc_configuration={"iceServers": get_ice_servers(name=ice_server)},
     video_frame_callback=video_frame_callback,
     media_stream_constraints={
         "video": {
@@ -198,16 +211,18 @@ ctx = webrtc_streamer(
                 "min": resolution[0],
                 "ideal": resolution[0],
                 "max": resolution[0],
-            }
+            },
+            "height": {
+                "min": resolution[1],
+                "ideal": resolution[1],
+                "max": resolution[1],
+            },
         },
         "audio": False,
     },
-    async_processing=False,  # WHAT IS THIS?
+    async_processing=True,
 )
 
-st.markdown("**Timings [ms]**")
-timings = st.empty()
-
 st.markdown("**Identified Faces**")
 identified_faces = st.empty()
 
@@ -217,19 +232,24 @@ detections = st.empty()
 # Display Live Stats
 if ctx.state.playing:
     while True:
+        # Get stats
         stats_dataframe = pd.DataFrame([stats_queue.get()])
+
+        # Write stats to streamlit
         stats.dataframe(stats_dataframe.style.format(thousands=" ", precision=2))
 
+        # Get detections
         detections_data = detections_queue.get()
-        detections_dataframe = pd.DataFrame(detections_data).drop(
-            columns=["face", "face_match"], errors="ignore"
+        detections_dataframe = (
+            pd.DataFrame(detections_data)
+            .drop(columns=["face", "face_match"], errors="ignore")
+            .applymap(lambda x: (format_list(x)))
         )
-        # Apply formatting to DataFrame
-        # print(detections_dataframe.columns)
-        # detections_dataframe["embedding"] = detections_dataframe["embedding"].embedding.applymap(format_floats)
 
+        # Write detections to streamlit
         detections.dataframe(detections_dataframe)
 
+        # Write identified faces to streamlit
         identified_faces.image(
             [display_match(d) for d in detections_data if d.name is not None],
             caption=[
@@ -238,6 +258,4 @@ if ctx.state.playing:
                 if d.name is not None
             ],
             width=112,
-        )  # TODO formatting
-
-        # time.sleep(1)
+        )

tools/face_recognition.py

@@ -62,6 +62,7 @@ def align_faces(img, detections):
         )
     return updated_detections
 
+
 # TODO Error when uploading image while running!
 def inference(detections, model):
     updated_detections = []
@@ -78,17 +79,16 @@ def inference(detections, model):
         ]
 
         for idx, detection in enumerate(detections):
-            updated_detections.append(detection._replace(emdedding=embs[idx]))
+            updated_detections.append(detection._replace(embedding=embs[idx]))
     return updated_detections
 
 
 def recognize_faces(detections, gallery, thresh=0.67):
-
     if len(gallery) == 0 or len(detections) == 0:
         return detections
 
     gallery_embs = np.asarray([identity.embedding for identity in gallery])
-    detection_embs = np.asarray([detection.emdedding for detection in detections])
+    detection_embs = np.asarray([detection.embedding for detection in detections])
 
     cos_distances = cosine_distances(detection_embs, gallery_embs)
 
@@ -103,8 +103,13 @@ def recognize_faces(detections, gallery, thresh=0.67):
             pred = idx_min
         updated_detections.append(
             detection._replace(
-                name=gallery[pred].name.split(".jpg")[0].split(".png")[0].split(".jpeg")[0] if pred is not None else None,
-                emdedding_match=gallery[pred].embedding if pred is not None else None,
+                name=gallery[pred]
+                .name.split(".jpg")[0]
+                .split(".png")[0]
+                .split(".jpeg")[0]
+                if pred is not None
+                else None,
+                embedding_match=gallery[pred].embedding if pred is not None else None,
                 face_match=gallery[pred].image if pred is not None else None,
                 distance=dist,
             )
@@ -135,7 +140,7 @@ def process_gallery(files, face_detection_model, face_recognition_model):
         gallery.append(
             Identity(
                 name=file.name,
-                embedding=detections[0].emdedding,
+                embedding=detections[0].embedding,
                 image=detections[0].face,
             )
         )

tools/nametypes.py

@@ -7,22 +7,22 @@ class Detection(NamedTuple):
     landmarks: List[List[int]]
     name: str = None
     face: np.ndarray = None
-    emdedding: np.ndarray = None
-    emdedding_match: np.ndarray = None
+    embedding: np.ndarray = None
+    embedding_match: np.ndarray = None
     face_match: np.ndarray = None
     distance: float = None
 
 
 class Stats(NamedTuple):
-    fps: float
-    resolution: List[int]
-    num_faces: int
-    detection: float
-    normalization: float
-    inference: float
-    recognition: float
-    drawing: float
-    
+    fps: float = 0
+    resolution: List[int] = [None, None, None]
+    num_faces: int = 0
+    detection: float = None
+    alignment: float = None
+    inference: float = None
+    recognition: float = None
+    drawing: float = None
+
 
 class Identity(NamedTuple):
     name: str

tools/utils.py

@@ -110,7 +110,6 @@ def download_file(url, model_path: Path, file_hash=None):
         download = True
 
     if download:
-
         # These are handles to two visual elements to animate.
         weights_warning, progress_bar = None, None
         try:
@@ -144,14 +143,18 @@ def download_file(url, model_path: Path, file_hash=None):
 
 
 # Function to format floats within a list
-def format_floats(val):
-    if isinstance(val, list): 
-        return [f"{num:.2f}" for num in val]
+def format_list(val):
+    if isinstance(val, list):
+        return [format_list(num) for num in val]
     if isinstance(val, np.ndarray):
-        return np.asarray([f"{num:.2f}" for num in val])
+        return np.asarray([format_list(num) for num in val])
+    if isinstance(val, np.float32):
+        return f"{val:.2f}"
+    if isinstance(val, float):
+        return f"{val:.2f}"
     else:
         return val
-    
+
 
 def display_match(d):
     im = np.concatenate([d.face, d.face_match])
@@ -163,10 +166,10 @@ def display_match(d):
         left=border_size,
         right=border_size,
         borderType=cv2.BORDER_CONSTANT,
-        value=(255, 255, 120)
+        value=(255, 255, 120),
     )
     return border
 
 
 def rgb(r, g, b):
-    return '#{:02x}{:02x}{:02x}'.format(r, g, b)
+    return "#{:02x}{:02x}{:02x}".format(r, g, b)