Add contextual VAD turn-event trainer

README.md

@@ -12,6 +12,18 @@ license: mit
 
 Small CPU-friendly classifier for improving VAD events on top of an STT + LLM + TTS voice-agent pipeline.
 
+## Live Playground
+
+The Space includes a browser playground for quick event testing:
+
+- record from the browser microphone
+- upload an audio file
+- optionally type the STT text you want the model to see
+- set assistant/TTS state to test interruption behavior
+- inspect a 100ms frame-level event timeline
+
+The playground does not run a full STT model on Free CPU. Audio drives VAD/timing features, while the optional transcript box simulates the STT partial/stable text features used by the classifier.
+
 ## How It Works
 
 Raw acoustic VAD is fast, but it only answers "is there speech-like audio right now?" A voice agent needs richer decisions:

app.py

@@ -6,6 +6,8 @@ from pathlib import Path
 from typing import Any
 
 import gradio as gr
+import numpy as np
+import pandas as pd
 from huggingface_hub import hf_hub_download
 
 from turn_event_model import (
@@ -26,6 +28,36 @@ LOCAL_MODEL_PATH = LOCAL_MODEL_DIR / "turn_event_model.joblib"
 MODEL = None
 STARTUP_STATUS = "Model not loaded yet."
 
+BACKCHANNEL_WORDS = {
+    "yeah",
+    "yes",
+    "yep",
+    "yup",
+    "mhm",
+    "mmhm",
+    "mm-hm",
+    "uh huh",
+    "right",
+    "ok",
+    "okay",
+    "sure",
+}
+
+CONTINUATION_WORDS = {
+    "and",
+    "but",
+    "because",
+    "so",
+    "or",
+    "for",
+    "with",
+    "to",
+    "um",
+    "uh",
+    "like",
+    "then",
+}
+
 
 def try_download_model() -> bool:
     global MODEL
@@ -160,12 +192,214 @@ def train_from_upload(csv_file: str | None, n_samples: float, model_repo_id: str
     )
 
 
+def clean_words(text: str) -> list[str]:
+    return [word.strip(".,!?;:()[]{}\"'").lower() for word in text.split() if word.strip()]
+
+
+def transcript_features(text: str, progress: float) -> dict[str, Any]:
+    text = (text or "").strip()
+    words = clean_words(text)
+    shown_chars = int(round(len(text) * max(0.0, min(1.0, progress))))
+    shown_words = max(0, min(len(words), int(round(len(words) * max(0.0, min(1.0, progress))))))
+    last_word = words[-1] if words else ""
+    compact = " ".join(words)
+    return {
+        "stable_chars": shown_chars,
+        "partial_chars": max(shown_chars, int(round(len(text) * min(1.0, progress + 0.18)))),
+        "stable_word_count": shown_words,
+        "words_since_pause": shown_words,
+        "stt_confidence": 0.86 if text else 0.08,
+        "ends_with_punctuation": int(text.endswith((".", "!", "?"))),
+        "ends_with_continuation": int(bool(last_word and last_word in CONTINUATION_WORDS and not text.endswith((".", "!", "?")))),
+        "backchannel_like": int(compact in BACKCHANNEL_WORDS or (len(words) <= 2 and compact in BACKCHANNEL_WORDS)),
+    }
+
+
+def audio_to_float_mono(audio: tuple[int, np.ndarray] | None) -> tuple[int, np.ndarray] | None:
+    if audio is None:
+        return None
+    sample_rate, samples = audio
+    if samples is None:
+        return None
+    arr = np.asarray(samples)
+    if arr.ndim > 1:
+        arr = arr.mean(axis=1)
+    arr = arr.astype(np.float32)
+    peak = float(np.max(np.abs(arr))) if arr.size else 0.0
+    if peak > 1.5:
+        arr = arr / max(peak, 1.0)
+    return int(sample_rate), arr
+
+
+def frame_rms(samples: np.ndarray, sample_rate: int, frame_ms: int) -> tuple[np.ndarray, int]:
+    frame_len = max(1, int(sample_rate * frame_ms / 1000))
+    n_frames = max(1, int(np.ceil(len(samples) / frame_len)))
+    padded = np.pad(samples, (0, max(0, n_frames * frame_len - len(samples))))
+    frames = padded.reshape(n_frames, frame_len)
+    rms = np.sqrt(np.mean(np.square(frames), axis=1))
+    return rms, frame_len
+
+
+def acoustic_vad_probs(rms: np.ndarray) -> np.ndarray:
+    if len(rms) == 0:
+        return np.asarray([], dtype=np.float32)
+    noise_floor = float(np.percentile(rms, 25))
+    threshold = max(0.012, noise_floor * 2.8)
+    scale = max(0.006, threshold * 0.55)
+    logits = np.clip((rms - threshold) / scale, -20, 20)
+    probs = 1.0 / (1.0 + np.exp(-logits))
+    return probs.astype(np.float32)
+
+
+def analyze_playground_audio(
+    audio: tuple[int, np.ndarray] | None,
+    transcript: str,
+    assistant_speaking: bool,
+    expected_answer_type: str,
+    required_slots_filled: bool,
+    tts_echo_risk: float,
+    recent_endpoint_candidate: bool,
+) -> tuple[pd.DataFrame, dict[str, Any]]:
+    if MODEL is None:
+        return pd.DataFrame(), {"error": "Model is not loaded yet."}
+
+    converted = audio_to_float_mono(audio)
+    if converted is None:
+        return pd.DataFrame(), {"error": "Record from the microphone or upload an audio file first."}
+
+    sample_rate, samples = converted
+    if samples.size == 0:
+        return pd.DataFrame(), {"error": "Audio was empty."}
+
+    frame_ms = 100
+    rms, _ = frame_rms(samples, sample_rate, frame_ms)
+    probs = acoustic_vad_probs(rms)
+    duration_s = len(samples) / sample_rate
+    speech_ms = 0.0
+    silence_ms = 0.0
+    endpoint_seen = bool(recent_endpoint_candidate)
+    rows: list[dict[str, Any]] = []
+
+    for idx, (energy, vad_prob) in enumerate(zip(rms, probs)):
+        t_ms = idx * frame_ms
+        progress = (t_ms + frame_ms) / max(duration_s * 1000, frame_ms)
+        vad_active = float(vad_prob) >= 0.55
+        if vad_active:
+            speech_ms += frame_ms
+            silence_ms = 0.0
+        else:
+            silence_ms += frame_ms
+
+        text_features = transcript_features(transcript, progress)
+        payload = {
+            **DEFAULT_FEATURES,
+            **text_features,
+            "vad_prob": float(vad_prob),
+            "vad_active": int(vad_active),
+            "speech_ms": speech_ms,
+            "silence_ms": silence_ms,
+            "energy": float(min(1.0, energy * 18.0)),
+            "assistant_speaking": int(assistant_speaking),
+            "tts_playback_ms": float(t_ms if assistant_speaking else 0.0),
+            "tts_echo_risk": float(tts_echo_risk if assistant_speaking else 0.0),
+            "time_since_user_started_ms": speech_ms if speech_ms > 0 else 0.0,
+            "time_since_assistant_started_ms": float(t_ms if assistant_speaking else 0.0),
+            "recent_endpoint_candidate": int(endpoint_seen),
+            "required_slots_filled": int(required_slots_filled),
+            "expected_answer_type": expected_answer_type,
+        }
+        pred = predict_event(MODEL, payload)
+        if pred["event"] == "endpoint_candidate":
+            endpoint_seen = True
+        top3 = ", ".join(
+            f"{item['event']}:{item['probability']:.2f}"
+            for item in pred["probabilities"][:3]
+        )
+        rows.append(
+            {
+                "time_s": round(t_ms / 1000, 2),
+                "event": pred["event"],
+                "confidence": round(float(pred["confidence"]), 4),
+                "vad_prob": round(float(vad_prob), 4),
+                "energy": round(float(min(1.0, energy * 18.0)), 4),
+                "speech_ms": int(speech_ms),
+                "silence_ms": int(silence_ms),
+                "top3": top3,
+            }
+        )
+
+    timeline = pd.DataFrame(rows)
+    event_counts = timeline["event"].value_counts().to_dict() if not timeline.empty else {}
+    changes = timeline[timeline["event"].ne(timeline["event"].shift())] if not timeline.empty else timeline
+    summary = {
+        "duration_s": round(duration_s, 2),
+        "sample_rate": sample_rate,
+        "frames": int(len(timeline)),
+        "final_event": rows[-1]["event"] if rows else None,
+        "final_confidence": rows[-1]["confidence"] if rows else None,
+        "dominant_event": max(event_counts, key=event_counts.get) if event_counts else None,
+        "event_counts": event_counts,
+        "event_changes": changes[["time_s", "event", "confidence"]].head(40).to_dict(orient="records"),
+        "note": "Audio drives VAD/timing. The optional transcript/context fields simulate the STT and dialogue-state features used by the model.",
+    }
+    return timeline, summary
+
+
 startup()
 
 with gr.Blocks() as demo:
     gr.Markdown("# Contextual VAD")
     status = gr.Textbox(value=STARTUP_STATUS, label="Status", interactive=False)
 
+    with gr.Tab("Live Playground"):
+        gr.Markdown("Record from the browser microphone or upload an audio file, then inspect the frame-level event timeline.")
+        with gr.Row():
+            with gr.Column():
+                playground_audio = gr.Audio(
+                    sources=["microphone", "upload"],
+                    type="numpy",
+                    label="Microphone or audio file",
+                    format="wav",
+                )
+                playground_transcript = gr.Textbox(
+                    label="Optional STT text",
+                    placeholder="Type what the user said, e.g. actually wait I need to change that",
+                    lines=3,
+                )
+            with gr.Column():
+                playground_assistant_speaking = gr.Checkbox(value=False, label="Assistant/TTS is speaking")
+                playground_expected_answer_type = gr.Dropdown(
+                    ["yes_no", "slot_fill", "open_ended", "confirmation"],
+                    value="open_ended",
+                    label="Expected answer type",
+                )
+                playground_required_slots_filled = gr.Checkbox(value=False, label="Required slots filled")
+                playground_recent_endpoint = gr.Checkbox(value=False, label="Recent endpoint candidate")
+                playground_echo_risk = gr.Slider(0, 1, value=0.12, step=0.01, label="Echo risk while TTS plays")
+
+        analyze_button = gr.Button("Analyze Audio", variant="primary")
+        playground_timeline = gr.Dataframe(label="Event timeline", interactive=False)
+        playground_summary = gr.JSON(label="Summary")
+        playground_inputs = [
+            playground_audio,
+            playground_transcript,
+            playground_assistant_speaking,
+            playground_expected_answer_type,
+            playground_required_slots_filled,
+            playground_echo_risk,
+            playground_recent_endpoint,
+        ]
+        analyze_button.click(
+            fn=analyze_playground_audio,
+            inputs=playground_inputs,
+            outputs=[playground_timeline, playground_summary],
+        )
+        playground_audio.change(
+            fn=analyze_playground_audio,
+            inputs=playground_inputs,
+            outputs=[playground_timeline, playground_summary],
+        )
+
     with gr.Tab("Predict"):
         with gr.Row():
             with gr.Column():