somukandula commited on
Commit
07dfb0c
·
verified ·
1 Parent(s): fa2c673

Add contextual VAD turn-event trainer

Browse files
README.md CHANGED
@@ -1,13 +1,15 @@
1
  ---
2
- title: Contextual Vad Turn Event Trainer
3
- emoji: 🐢
4
- colorFrom: yellow
5
- colorTo: purple
6
  sdk: gradio
7
- sdk_version: 6.14.0
8
- python_version: '3.13'
9
  app_file: app.py
10
  pinned: false
 
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
1
  ---
2
+ title: Contextual VAD Turn Event Trainer
3
+ colorFrom: blue
4
+ colorTo: green
 
5
  sdk: gradio
 
 
6
  app_file: app.py
7
  pinned: false
8
+ license: mit
9
  ---
10
 
11
+ # Contextual VAD Turn Event Trainer
12
+
13
+ Small CPU-friendly classifier for improving VAD events on top of an STT + LLM + TTS voice-agent pipeline.
14
+
15
+ The Space trains a bootstrap model from synthetic event rows by default. Upload a CSV with the same feature schema and an `event_label` column to train on real call logs.
__pycache__/app.cpython-310.pyc ADDED
Binary file (6.64 kB). View file
 
__pycache__/turn_event_model.cpython-310.pyc ADDED
Binary file (14 kB). View file
 
app.py ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import os
5
+ import tempfile
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ import gradio as gr
10
+ from huggingface_hub import hf_hub_download
11
+
12
+ from turn_event_model import (
13
+ DEFAULT_FEATURES,
14
+ EVENT_LABELS,
15
+ example_payload,
16
+ load_model,
17
+ predict_event,
18
+ push_model_to_hub,
19
+ train_turn_event_model,
20
+ )
21
+
22
+
23
+ MODEL_REPO_ID = os.environ.get("MODEL_REPO_ID", "somukandula/contextual-vad-turn-event-model")
24
+ AUTO_TRAIN_ON_STARTUP = os.environ.get("AUTO_TRAIN_ON_STARTUP", "1") == "1"
25
+ LOCAL_MODEL_DIR = Path("trained_model")
26
+ LOCAL_MODEL_PATH = LOCAL_MODEL_DIR / "turn_event_model.joblib"
27
+
28
+ MODEL = None
29
+ STARTUP_STATUS = "Model not loaded yet."
30
+
31
+
32
+ def try_download_model() -> bool:
33
+ global MODEL
34
+ try:
35
+ path = hf_hub_download(
36
+ repo_id=MODEL_REPO_ID,
37
+ filename="turn_event_model.joblib",
38
+ repo_type="model",
39
+ )
40
+ MODEL = load_model(path)
41
+ return True
42
+ except Exception:
43
+ return False
44
+
45
+
46
+ def train_and_optionally_push(
47
+ csv_file: str | None = None,
48
+ n_samples: int = 12000,
49
+ model_repo_id: str = MODEL_REPO_ID,
50
+ ) -> tuple[str, dict[str, Any]]:
51
+ global MODEL
52
+ result = train_turn_event_model(
53
+ output_dir=LOCAL_MODEL_DIR,
54
+ csv_path=csv_file,
55
+ n_samples=int(n_samples),
56
+ )
57
+ MODEL = load_model(result.model_path)
58
+
59
+ hub_url = None
60
+ if os.environ.get("HF_TOKEN"):
61
+ hub_url = push_model_to_hub(result.output_dir, model_repo_id)
62
+
63
+ summary = {
64
+ "accuracy": result.metrics["accuracy"],
65
+ "n_rows": result.metrics["n_rows"],
66
+ "source": result.metrics["source"],
67
+ "model_repo": model_repo_id,
68
+ "hub_url": hub_url,
69
+ }
70
+ message = "Training complete."
71
+ if hub_url:
72
+ message += f" Pushed model to {hub_url}."
73
+ else:
74
+ message += " HF_TOKEN was not available, so the model stayed inside the Space runtime."
75
+ return message, summary
76
+
77
+
78
+ def startup() -> None:
79
+ global MODEL, STARTUP_STATUS
80
+ if try_download_model():
81
+ STARTUP_STATUS = f"Loaded model from {MODEL_REPO_ID}."
82
+ return
83
+
84
+ if AUTO_TRAIN_ON_STARTUP:
85
+ message, summary = train_and_optionally_push(model_repo_id=MODEL_REPO_ID)
86
+ STARTUP_STATUS = f"{message} {json.dumps(summary)}"
87
+ return
88
+
89
+ STARTUP_STATUS = "No model found. Use Train to create one."
90
+
91
+
92
+ def coerce_bool(value: bool) -> int:
93
+ return int(bool(value))
94
+
95
+
96
+ def predict_from_controls(
97
+ vad_prob: float,
98
+ vad_active: bool,
99
+ speech_ms: float,
100
+ silence_ms: float,
101
+ energy: float,
102
+ stt_confidence: float,
103
+ stable_chars: float,
104
+ partial_chars: float,
105
+ stable_word_count: float,
106
+ words_since_pause: float,
107
+ ends_with_punctuation: bool,
108
+ ends_with_continuation: bool,
109
+ required_slots_filled: bool,
110
+ assistant_speaking: bool,
111
+ backchannel_like: bool,
112
+ tts_playback_ms: float,
113
+ tts_echo_risk: float,
114
+ time_since_user_started_ms: float,
115
+ time_since_assistant_started_ms: float,
116
+ recent_endpoint_candidate: bool,
117
+ expected_answer_type: str,
118
+ ) -> dict[str, Any]:
119
+ if MODEL is None:
120
+ return {"error": "Model is not loaded yet."}
121
+ payload = {
122
+ "vad_prob": vad_prob,
123
+ "vad_active": coerce_bool(vad_active),
124
+ "speech_ms": speech_ms,
125
+ "silence_ms": silence_ms,
126
+ "energy": energy,
127
+ "stt_confidence": stt_confidence,
128
+ "stable_chars": stable_chars,
129
+ "partial_chars": partial_chars,
130
+ "stable_word_count": stable_word_count,
131
+ "words_since_pause": words_since_pause,
132
+ "ends_with_punctuation": coerce_bool(ends_with_punctuation),
133
+ "ends_with_continuation": coerce_bool(ends_with_continuation),
134
+ "required_slots_filled": coerce_bool(required_slots_filled),
135
+ "assistant_speaking": coerce_bool(assistant_speaking),
136
+ "backchannel_like": coerce_bool(backchannel_like),
137
+ "tts_playback_ms": tts_playback_ms,
138
+ "tts_echo_risk": tts_echo_risk,
139
+ "time_since_user_started_ms": time_since_user_started_ms,
140
+ "time_since_assistant_started_ms": time_since_assistant_started_ms,
141
+ "recent_endpoint_candidate": coerce_bool(recent_endpoint_candidate),
142
+ "expected_answer_type": expected_answer_type,
143
+ }
144
+ return predict_event(MODEL, payload)
145
+
146
+
147
+ def predict_from_json(payload_text: str) -> dict[str, Any]:
148
+ if MODEL is None:
149
+ return {"error": "Model is not loaded yet."}
150
+ try:
151
+ payload = json.loads(payload_text)
152
+ except json.JSONDecodeError as exc:
153
+ return {"error": f"Invalid JSON: {exc}"}
154
+ return predict_event(MODEL, payload)
155
+
156
+
157
+ def train_from_upload(csv_file: str | None, n_samples: float, model_repo_id: str) -> tuple[str, dict[str, Any]]:
158
+ return train_and_optionally_push(
159
+ csv_file=csv_file,
160
+ n_samples=int(n_samples),
161
+ model_repo_id=model_repo_id.strip() or MODEL_REPO_ID,
162
+ )
163
+
164
+
165
+ startup()
166
+
167
+ with gr.Blocks() as demo:
168
+ gr.Markdown("# Contextual VAD Turn Event Model")
169
+ status = gr.Textbox(value=STARTUP_STATUS, label="Status", interactive=False)
170
+
171
+ with gr.Tab("Predict"):
172
+ with gr.Row():
173
+ with gr.Column():
174
+ vad_prob = gr.Slider(0, 1, value=0.91, step=0.01, label="VAD probability")
175
+ vad_active = gr.Checkbox(value=True, label="VAD active")
176
+ speech_ms = gr.Number(value=900, label="Speech ms")
177
+ silence_ms = gr.Number(value=0, label="Silence ms")
178
+ energy = gr.Slider(0, 1, value=0.82, step=0.01, label="Energy")
179
+ stt_confidence = gr.Slider(0, 1, value=0.84, step=0.01, label="STT confidence")
180
+ stable_chars = gr.Number(value=42, label="Stable chars")
181
+ partial_chars = gr.Number(value=52, label="Partial chars")
182
+ stable_word_count = gr.Number(value=8, label="Stable word count")
183
+ words_since_pause = gr.Number(value=6, label="Words since pause")
184
+ with gr.Column():
185
+ ends_with_punctuation = gr.Checkbox(value=False, label="Ends with punctuation")
186
+ ends_with_continuation = gr.Checkbox(value=False, label="Ends with continuation")
187
+ required_slots_filled = gr.Checkbox(value=False, label="Required slots filled")
188
+ assistant_speaking = gr.Checkbox(value=True, label="Assistant speaking")
189
+ backchannel_like = gr.Checkbox(value=False, label="Backchannel-like text")
190
+ tts_playback_ms = gr.Number(value=2300, label="TTS playback ms")
191
+ tts_echo_risk = gr.Slider(0, 1, value=0.12, step=0.01, label="TTS echo risk")
192
+ time_since_user_started_ms = gr.Number(value=900, label="User started ms ago")
193
+ time_since_assistant_started_ms = gr.Number(value=2300, label="Assistant started ms ago")
194
+ recent_endpoint_candidate = gr.Checkbox(value=False, label="Recent endpoint candidate")
195
+ expected_answer_type = gr.Dropdown(
196
+ ["yes_no", "slot_fill", "open_ended", "confirmation"],
197
+ value="open_ended",
198
+ label="Expected answer type",
199
+ )
200
+
201
+ predict_button = gr.Button("Predict", variant="primary")
202
+ prediction = gr.JSON(label="Prediction")
203
+ predict_button.click(
204
+ fn=predict_from_controls,
205
+ inputs=[
206
+ vad_prob,
207
+ vad_active,
208
+ speech_ms,
209
+ silence_ms,
210
+ energy,
211
+ stt_confidence,
212
+ stable_chars,
213
+ partial_chars,
214
+ stable_word_count,
215
+ words_since_pause,
216
+ ends_with_punctuation,
217
+ ends_with_continuation,
218
+ required_slots_filled,
219
+ assistant_speaking,
220
+ backchannel_like,
221
+ tts_playback_ms,
222
+ tts_echo_risk,
223
+ time_since_user_started_ms,
224
+ time_since_assistant_started_ms,
225
+ recent_endpoint_candidate,
226
+ expected_answer_type,
227
+ ],
228
+ outputs=prediction,
229
+ )
230
+
231
+ with gr.Tab("JSON"):
232
+ payload = gr.Textbox(
233
+ value=json.dumps(example_payload(), indent=2),
234
+ lines=18,
235
+ label="Payload",
236
+ )
237
+ json_button = gr.Button("Predict JSON", variant="primary")
238
+ json_prediction = gr.JSON(label="Prediction")
239
+ json_button.click(fn=predict_from_json, inputs=payload, outputs=json_prediction)
240
+
241
+ with gr.Tab("Train"):
242
+ csv_upload = gr.File(label="Training CSV", file_types=[".csv"], type="filepath")
243
+ sample_count = gr.Number(value=12000, label="Synthetic rows")
244
+ repo_id = gr.Textbox(value=MODEL_REPO_ID, label="Model repo")
245
+ train_button = gr.Button("Train", variant="primary")
246
+ train_message = gr.Textbox(label="Training result", interactive=False)
247
+ train_metrics = gr.JSON(label="Metrics")
248
+ train_button.click(
249
+ fn=train_from_upload,
250
+ inputs=[csv_upload, sample_count, repo_id],
251
+ outputs=[train_message, train_metrics],
252
+ )
253
+
254
+
255
+ if __name__ == "__main__":
256
+ demo.launch()
257
+
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ gradio>=4.44.0
2
+ huggingface_hub>=0.24.0
3
+ joblib>=1.3.0
4
+ numpy>=1.24.0
5
+ pandas>=2.0.0
6
+ scikit-learn>=1.3.0
7
+
sample_training_schema.csv ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ vad_prob,speech_ms,silence_ms,energy,stt_confidence,stable_chars,partial_chars,stable_word_count,words_since_pause,tts_playback_ms,tts_echo_risk,time_since_user_started_ms,time_since_assistant_started_ms,recent_endpoint_candidate,vad_active,ends_with_punctuation,ends_with_continuation,required_slots_filled,assistant_speaking,backchannel_like,expected_answer_type,event_label
2
+ 0.91,900,0,0.82,0.84,42,52,8,6,2300,0.12,900,2300,0,1,0,0,0,1,0,open_ended,interruption_confirmed
3
+ 0.08,2600,950,0.10,0.88,90,0,16,4,0,0,2600,0,0,0,1,0,1,0,0,slot_fill,turn_committed
4
+ 0.78,310,0,0.58,0.68,6,8,1,1,2200,0.20,310,2200,0,1,0,0,0,1,1,open_ended,backchannel_detected
5
+
turn_event_model.py ADDED
@@ -0,0 +1,524 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import math
5
+ import os
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ import joblib
11
+ import numpy as np
12
+ import pandas as pd
13
+ from huggingface_hub import HfApi
14
+ from sklearn.compose import ColumnTransformer
15
+ from sklearn.impute import SimpleImputer
16
+ from sklearn.linear_model import LogisticRegression
17
+ from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
18
+ from sklearn.model_selection import train_test_split
19
+ from sklearn.pipeline import Pipeline
20
+ from sklearn.preprocessing import OneHotEncoder, StandardScaler
21
+
22
+
23
+ EVENT_LABELS = [
24
+ "listening",
25
+ "speech_started",
26
+ "endpoint_candidate",
27
+ "turn_committed",
28
+ "user_resumed",
29
+ "interruption_started",
30
+ "interruption_confirmed",
31
+ "backchannel_detected",
32
+ "false_alarm",
33
+ ]
34
+
35
+ NUMERIC_FEATURES = [
36
+ "vad_prob",
37
+ "speech_ms",
38
+ "silence_ms",
39
+ "energy",
40
+ "stt_confidence",
41
+ "stable_chars",
42
+ "partial_chars",
43
+ "stable_word_count",
44
+ "words_since_pause",
45
+ "tts_playback_ms",
46
+ "tts_echo_risk",
47
+ "time_since_user_started_ms",
48
+ "time_since_assistant_started_ms",
49
+ "recent_endpoint_candidate",
50
+ "vad_active",
51
+ "ends_with_punctuation",
52
+ "ends_with_continuation",
53
+ "required_slots_filled",
54
+ "assistant_speaking",
55
+ "backchannel_like",
56
+ ]
57
+
58
+ CATEGORICAL_FEATURES = ["expected_answer_type"]
59
+ FEATURE_COLUMNS = NUMERIC_FEATURES + CATEGORICAL_FEATURES
60
+
61
+ DEFAULT_FEATURES: dict[str, Any] = {
62
+ "vad_prob": 0.0,
63
+ "speech_ms": 0.0,
64
+ "silence_ms": 0.0,
65
+ "energy": 0.0,
66
+ "stt_confidence": 0.0,
67
+ "stable_chars": 0.0,
68
+ "partial_chars": 0.0,
69
+ "stable_word_count": 0.0,
70
+ "words_since_pause": 0.0,
71
+ "tts_playback_ms": 0.0,
72
+ "tts_echo_risk": 0.0,
73
+ "time_since_user_started_ms": 0.0,
74
+ "time_since_assistant_started_ms": 0.0,
75
+ "recent_endpoint_candidate": 0,
76
+ "vad_active": 0,
77
+ "ends_with_punctuation": 0,
78
+ "ends_with_continuation": 0,
79
+ "required_slots_filled": 0,
80
+ "assistant_speaking": 0,
81
+ "backchannel_like": 0,
82
+ "expected_answer_type": "open_ended",
83
+ }
84
+
85
+
86
+ @dataclass
87
+ class TrainResult:
88
+ model_path: str
89
+ output_dir: str
90
+ metrics: dict[str, Any]
91
+
92
+
93
+ def clamp(value: float, low: float, high: float) -> float:
94
+ return float(max(low, min(high, value)))
95
+
96
+
97
+ def jitter(rng: np.random.Generator, center: float, spread: float, low: float, high: float) -> float:
98
+ return clamp(rng.normal(center, spread), low, high)
99
+
100
+
101
+ def randint(rng: np.random.Generator, low: int, high: int) -> int:
102
+ return int(rng.integers(low, high + 1))
103
+
104
+
105
+ def chance(rng: np.random.Generator, p: float) -> int:
106
+ return int(rng.random() < p)
107
+
108
+
109
+ def choice(rng: np.random.Generator, values: list[str]) -> str:
110
+ return str(rng.choice(values))
111
+
112
+
113
+ def base_row(rng: np.random.Generator) -> dict[str, Any]:
114
+ return {
115
+ **DEFAULT_FEATURES,
116
+ "expected_answer_type": choice(
117
+ rng,
118
+ ["yes_no", "slot_fill", "open_ended", "confirmation"],
119
+ ),
120
+ }
121
+
122
+
123
+ def row_for_label(label: str, rng: np.random.Generator) -> dict[str, Any]:
124
+ row = base_row(rng)
125
+
126
+ if label == "listening":
127
+ row.update(
128
+ vad_prob=jitter(rng, 0.12, 0.08, 0.0, 0.35),
129
+ vad_active=0,
130
+ silence_ms=jitter(rng, 120, 120, 0, 500),
131
+ energy=jitter(rng, 0.12, 0.08, 0, 0.35),
132
+ stt_confidence=jitter(rng, 0.05, 0.08, 0, 0.25),
133
+ assistant_speaking=chance(rng, 0.25),
134
+ )
135
+
136
+ elif label == "speech_started":
137
+ row.update(
138
+ vad_prob=jitter(rng, 0.82, 0.1, 0.55, 1.0),
139
+ vad_active=1,
140
+ speech_ms=jitter(rng, 95, 45, 20, 190),
141
+ silence_ms=jitter(rng, 0, 20, 0, 80),
142
+ energy=jitter(rng, 0.72, 0.15, 0.35, 1.0),
143
+ partial_chars=randint(rng, 0, 10),
144
+ stt_confidence=jitter(rng, 0.25, 0.15, 0, 0.55),
145
+ assistant_speaking=0,
146
+ time_since_user_started_ms=jitter(rng, 100, 45, 20, 200),
147
+ )
148
+
149
+ elif label == "endpoint_candidate":
150
+ continuation = chance(rng, 0.35)
151
+ row.update(
152
+ vad_prob=jitter(rng, 0.18, 0.1, 0.0, 0.45),
153
+ vad_active=0,
154
+ speech_ms=jitter(rng, 1800, 900, 350, 6000),
155
+ silence_ms=jitter(rng, 450, 180, 180, 900),
156
+ energy=jitter(rng, 0.18, 0.1, 0.0, 0.45),
157
+ stt_confidence=jitter(rng, 0.74, 0.12, 0.35, 0.97),
158
+ stable_chars=randint(rng, 12, 120),
159
+ partial_chars=randint(rng, 0, 40),
160
+ stable_word_count=randint(rng, 3, 24),
161
+ words_since_pause=randint(rng, 2, 12),
162
+ ends_with_punctuation=chance(rng, 0.35),
163
+ ends_with_continuation=continuation,
164
+ required_slots_filled=chance(rng, 0.45),
165
+ assistant_speaking=0,
166
+ )
167
+
168
+ elif label == "turn_committed":
169
+ answer_type = choice(rng, ["yes_no", "slot_fill", "confirmation", "open_ended"])
170
+ row.update(
171
+ expected_answer_type=answer_type,
172
+ vad_prob=jitter(rng, 0.08, 0.07, 0.0, 0.28),
173
+ vad_active=0,
174
+ speech_ms=jitter(rng, 2600, 1400, 250, 9000),
175
+ silence_ms=jitter(rng, 950, 330, 420, 2200),
176
+ energy=jitter(rng, 0.1, 0.07, 0.0, 0.28),
177
+ stt_confidence=jitter(rng, 0.88, 0.08, 0.6, 0.99),
178
+ stable_chars=randint(rng, 6 if answer_type == "yes_no" else 30, 180),
179
+ partial_chars=randint(rng, 0, 8),
180
+ stable_word_count=randint(rng, 1 if answer_type == "yes_no" else 6, 36),
181
+ words_since_pause=randint(rng, 1, 8),
182
+ ends_with_punctuation=chance(rng, 0.8),
183
+ ends_with_continuation=chance(rng, 0.04),
184
+ required_slots_filled=chance(rng, 0.85),
185
+ assistant_speaking=0,
186
+ )
187
+
188
+ elif label == "user_resumed":
189
+ row.update(
190
+ vad_prob=jitter(rng, 0.86, 0.09, 0.6, 1.0),
191
+ vad_active=1,
192
+ speech_ms=jitter(rng, 250, 150, 60, 850),
193
+ silence_ms=jitter(rng, 20, 25, 0, 100),
194
+ energy=jitter(rng, 0.76, 0.14, 0.4, 1.0),
195
+ stt_confidence=jitter(rng, 0.55, 0.2, 0.12, 0.9),
196
+ partial_chars=randint(rng, 4, 45),
197
+ stable_chars=randint(rng, 0, 35),
198
+ stable_word_count=randint(rng, 0, 8),
199
+ recent_endpoint_candidate=1,
200
+ assistant_speaking=0,
201
+ )
202
+
203
+ elif label == "interruption_started":
204
+ row.update(
205
+ vad_prob=jitter(rng, 0.82, 0.11, 0.55, 1.0),
206
+ vad_active=1,
207
+ speech_ms=jitter(rng, 170, 80, 60, 380),
208
+ silence_ms=jitter(rng, 0, 15, 0, 60),
209
+ energy=jitter(rng, 0.72, 0.16, 0.3, 1.0),
210
+ stt_confidence=jitter(rng, 0.32, 0.18, 0.0, 0.7),
211
+ partial_chars=randint(rng, 0, 25),
212
+ stable_chars=randint(rng, 0, 10),
213
+ tts_playback_ms=jitter(rng, 1800, 1000, 200, 8000),
214
+ tts_echo_risk=jitter(rng, 0.22, 0.15, 0, 0.55),
215
+ assistant_speaking=1,
216
+ time_since_assistant_started_ms=jitter(rng, 1800, 1000, 200, 8000),
217
+ )
218
+
219
+ elif label == "interruption_confirmed":
220
+ row.update(
221
+ vad_prob=jitter(rng, 0.91, 0.07, 0.68, 1.0),
222
+ vad_active=1,
223
+ speech_ms=jitter(rng, 1050, 550, 420, 3600),
224
+ silence_ms=jitter(rng, 0, 20, 0, 80),
225
+ energy=jitter(rng, 0.82, 0.12, 0.45, 1.0),
226
+ stt_confidence=jitter(rng, 0.8, 0.12, 0.45, 0.99),
227
+ partial_chars=randint(rng, 12, 140),
228
+ stable_chars=randint(rng, 10, 120),
229
+ stable_word_count=randint(rng, 2, 24),
230
+ words_since_pause=randint(rng, 1, 12),
231
+ tts_playback_ms=jitter(rng, 2400, 1400, 250, 10000),
232
+ tts_echo_risk=jitter(rng, 0.16, 0.12, 0, 0.45),
233
+ assistant_speaking=1,
234
+ backchannel_like=chance(rng, 0.05),
235
+ time_since_assistant_started_ms=jitter(rng, 2400, 1400, 250, 10000),
236
+ )
237
+
238
+ elif label == "backchannel_detected":
239
+ row.update(
240
+ vad_prob=jitter(rng, 0.78, 0.11, 0.5, 1.0),
241
+ vad_active=1,
242
+ speech_ms=jitter(rng, 310, 160, 80, 900),
243
+ silence_ms=jitter(rng, 0, 20, 0, 80),
244
+ energy=jitter(rng, 0.58, 0.17, 0.25, 1.0),
245
+ stt_confidence=jitter(rng, 0.68, 0.17, 0.25, 0.96),
246
+ stable_chars=randint(rng, 2, 12),
247
+ partial_chars=randint(rng, 0, 18),
248
+ stable_word_count=randint(rng, 1, 3),
249
+ words_since_pause=randint(rng, 1, 3),
250
+ tts_playback_ms=jitter(rng, 2200, 1400, 250, 9000),
251
+ tts_echo_risk=jitter(rng, 0.2, 0.15, 0, 0.55),
252
+ assistant_speaking=1,
253
+ backchannel_like=1,
254
+ )
255
+
256
+ elif label == "false_alarm":
257
+ row.update(
258
+ vad_prob=jitter(rng, 0.62, 0.18, 0.25, 0.95),
259
+ vad_active=chance(rng, 0.75),
260
+ speech_ms=jitter(rng, 90, 80, 0, 260),
261
+ silence_ms=jitter(rng, 20, 40, 0, 150),
262
+ energy=jitter(rng, 0.48, 0.22, 0.05, 0.95),
263
+ stt_confidence=jitter(rng, 0.1, 0.1, 0, 0.35),
264
+ stable_chars=randint(rng, 0, 4),
265
+ partial_chars=randint(rng, 0, 8),
266
+ stable_word_count=0,
267
+ tts_playback_ms=jitter(rng, 1600, 1400, 0, 8000),
268
+ tts_echo_risk=jitter(rng, 0.82, 0.14, 0.45, 1.0),
269
+ assistant_speaking=chance(rng, 0.8),
270
+ )
271
+
272
+ row["event_label"] = label
273
+ return row
274
+
275
+
276
+ def generate_synthetic_dataset(n_samples: int = 12000, seed: int = 7) -> pd.DataFrame:
277
+ rng = np.random.default_rng(seed)
278
+ weights = np.array([0.22, 0.08, 0.14, 0.16, 0.08, 0.08, 0.09, 0.08, 0.07])
279
+ labels = rng.choice(EVENT_LABELS, size=n_samples, p=weights / weights.sum())
280
+ rows = [row_for_label(str(label), rng) for label in labels]
281
+ return pd.DataFrame(rows)
282
+
283
+
284
+ def coerce_schema(df: pd.DataFrame) -> pd.DataFrame:
285
+ out = df.copy()
286
+ for column, default in DEFAULT_FEATURES.items():
287
+ if column not in out.columns:
288
+ out[column] = default
289
+ for column in NUMERIC_FEATURES:
290
+ out[column] = pd.to_numeric(out[column], errors="coerce").fillna(DEFAULT_FEATURES[column])
291
+ out["expected_answer_type"] = out["expected_answer_type"].fillna("open_ended").astype(str)
292
+ return out
293
+
294
+
295
+ def build_pipeline() -> Pipeline:
296
+ numeric_pipeline = Pipeline(
297
+ steps=[
298
+ ("imputer", SimpleImputer(strategy="median")),
299
+ ("scaler", StandardScaler()),
300
+ ]
301
+ )
302
+ categorical_pipeline = Pipeline(
303
+ steps=[
304
+ ("imputer", SimpleImputer(strategy="most_frequent")),
305
+ ("onehot", OneHotEncoder(handle_unknown="ignore")),
306
+ ]
307
+ )
308
+ preprocessor = ColumnTransformer(
309
+ transformers=[
310
+ ("num", numeric_pipeline, NUMERIC_FEATURES),
311
+ ("cat", categorical_pipeline, CATEGORICAL_FEATURES),
312
+ ]
313
+ )
314
+ classifier = LogisticRegression(
315
+ max_iter=1200,
316
+ class_weight="balanced",
317
+ n_jobs=1,
318
+ )
319
+ return Pipeline(steps=[("preprocess", preprocessor), ("classifier", classifier)])
320
+
321
+
322
+ def split_train_test(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series]:
323
+ x = df[FEATURE_COLUMNS]
324
+ y = df["event_label"].astype(str)
325
+ try:
326
+ return train_test_split(x, y, test_size=0.2, random_state=42, stratify=y)
327
+ except ValueError:
328
+ return train_test_split(x, y, test_size=0.2, random_state=42)
329
+
330
+
331
+ def train_turn_event_model(
332
+ output_dir: str | Path = "trained_model",
333
+ csv_path: str | Path | None = None,
334
+ n_samples: int = 12000,
335
+ seed: int = 7,
336
+ ) -> TrainResult:
337
+ output = Path(output_dir)
338
+ output.mkdir(parents=True, exist_ok=True)
339
+
340
+ if csv_path:
341
+ df = pd.read_csv(csv_path)
342
+ if "event_label" not in df.columns:
343
+ raise ValueError("Training CSV must contain an event_label column.")
344
+ source = f"uploaded_csv:{Path(csv_path).name}"
345
+ else:
346
+ df = generate_synthetic_dataset(n_samples=n_samples, seed=seed)
347
+ source = f"synthetic_bootstrap:n={n_samples}:seed={seed}"
348
+
349
+ df = coerce_schema(df)
350
+ df = df[df["event_label"].isin(EVENT_LABELS)].copy()
351
+ if df.empty:
352
+ raise ValueError("No usable training rows after schema coercion.")
353
+
354
+ x_train, x_test, y_train, y_test = split_train_test(df)
355
+ model = build_pipeline()
356
+ model.fit(x_train, y_train)
357
+
358
+ y_pred = model.predict(x_test)
359
+ labels_in_test = sorted(set(y_test) | set(y_pred))
360
+ report = classification_report(
361
+ y_test,
362
+ y_pred,
363
+ labels=labels_in_test,
364
+ output_dict=True,
365
+ zero_division=0,
366
+ )
367
+ matrix = confusion_matrix(y_test, y_pred, labels=labels_in_test).tolist()
368
+ metrics = {
369
+ "accuracy": float(accuracy_score(y_test, y_pred)),
370
+ "n_rows": int(len(df)),
371
+ "n_train": int(len(x_train)),
372
+ "n_test": int(len(x_test)),
373
+ "source": source,
374
+ "labels": labels_in_test,
375
+ "classification_report": report,
376
+ "confusion_matrix": matrix,
377
+ }
378
+
379
+ model_path = output / "turn_event_model.joblib"
380
+ joblib.dump(model, model_path)
381
+ (output / "feature_schema.json").write_text(
382
+ json.dumps(
383
+ {
384
+ "feature_columns": FEATURE_COLUMNS,
385
+ "numeric_features": NUMERIC_FEATURES,
386
+ "categorical_features": CATEGORICAL_FEATURES,
387
+ "default_features": DEFAULT_FEATURES,
388
+ "event_labels": EVENT_LABELS,
389
+ },
390
+ indent=2,
391
+ )
392
+ )
393
+ (output / "metrics.json").write_text(json.dumps(metrics, indent=2))
394
+ (output / "example_payload.json").write_text(json.dumps(example_payload(), indent=2))
395
+ (output / "README.md").write_text(model_card(metrics))
396
+
397
+ return TrainResult(str(model_path), str(output), metrics)
398
+
399
+
400
+ def model_card(metrics: dict[str, Any]) -> str:
401
+ accuracy = metrics.get("accuracy", 0.0)
402
+ source = metrics.get("source", "unknown")
403
+ return f"""---
404
+ library_name: scikit-learn
405
+ tags:
406
+ - voice-agent
407
+ - vad
408
+ - turn-taking
409
+ - tabular-classification
410
+ - generated_from_trainer
411
+ license: mit
412
+ ---
413
+
414
+ # Contextual VAD Turn Event Model
415
+
416
+ This is a small scikit-learn classifier that predicts higher-level voice-agent events from derived VAD, STT, TTS, and dialogue-state features.
417
+
418
+ It is designed to sit on top of a streaming STT + LLM + TTS pipeline and produce probabilities for:
419
+
420
+ - `listening`
421
+ - `speech_started`
422
+ - `endpoint_candidate`
423
+ - `turn_committed`
424
+ - `user_resumed`
425
+ - `interruption_started`
426
+ - `interruption_confirmed`
427
+ - `backchannel_detected`
428
+ - `false_alarm`
429
+
430
+ Training source: `{source}`
431
+
432
+ Validation accuracy: `{accuracy:.4f}`
433
+
434
+ ## Intended use
435
+
436
+ Use this as a bootstrap policy model. Replace the synthetic bootstrap data with real call-frame logs before production use.
437
+
438
+ ## Files
439
+
440
+ - `turn_event_model.joblib`: scikit-learn pipeline.
441
+ - `feature_schema.json`: feature names and defaults.
442
+ - `metrics.json`: validation metrics.
443
+ - `example_payload.json`: one valid inference payload.
444
+ """
445
+
446
+
447
+ def example_payload() -> dict[str, Any]:
448
+ payload = DEFAULT_FEATURES.copy()
449
+ payload.update(
450
+ {
451
+ "vad_prob": 0.91,
452
+ "vad_active": 1,
453
+ "speech_ms": 900,
454
+ "silence_ms": 0,
455
+ "energy": 0.82,
456
+ "stt_confidence": 0.84,
457
+ "stable_chars": 42,
458
+ "partial_chars": 52,
459
+ "stable_word_count": 8,
460
+ "words_since_pause": 6,
461
+ "assistant_speaking": 1,
462
+ "tts_playback_ms": 2300,
463
+ "tts_echo_risk": 0.12,
464
+ "time_since_assistant_started_ms": 2300,
465
+ "expected_answer_type": "open_ended",
466
+ }
467
+ )
468
+ return payload
469
+
470
+
471
+ def load_model(model_path: str | Path) -> Pipeline:
472
+ return joblib.load(model_path)
473
+
474
+
475
+ def normalize_payload(payload: dict[str, Any]) -> pd.DataFrame:
476
+ row = DEFAULT_FEATURES.copy()
477
+ row.update({k: v for k, v in payload.items() if v is not None})
478
+ for key in NUMERIC_FEATURES:
479
+ value = row.get(key, DEFAULT_FEATURES[key])
480
+ if isinstance(value, bool):
481
+ row[key] = int(value)
482
+ else:
483
+ try:
484
+ number = float(value)
485
+ row[key] = 0.0 if math.isnan(number) else number
486
+ except (TypeError, ValueError):
487
+ row[key] = DEFAULT_FEATURES[key]
488
+ row["expected_answer_type"] = str(row.get("expected_answer_type", "open_ended"))
489
+ return pd.DataFrame([row])[FEATURE_COLUMNS]
490
+
491
+
492
+ def predict_event(model: Pipeline, payload: dict[str, Any]) -> dict[str, Any]:
493
+ x = normalize_payload(payload)
494
+ classes = list(model.classes_)
495
+ probabilities = model.predict_proba(x)[0]
496
+ ranked = sorted(
497
+ [
498
+ {"event": str(label), "probability": float(prob)}
499
+ for label, prob in zip(classes, probabilities)
500
+ ],
501
+ key=lambda item: item["probability"],
502
+ reverse=True,
503
+ )
504
+ return {
505
+ "event": ranked[0]["event"],
506
+ "confidence": ranked[0]["probability"],
507
+ "probabilities": ranked,
508
+ }
509
+
510
+
511
+ def push_model_to_hub(output_dir: str | Path, model_repo_id: str) -> str:
512
+ token = os.environ.get("HF_TOKEN")
513
+ if not token:
514
+ raise RuntimeError("HF_TOKEN is required to push the trained model to the Hub.")
515
+ api = HfApi(token=token)
516
+ api.create_repo(repo_id=model_repo_id, repo_type="model", exist_ok=True)
517
+ api.upload_folder(
518
+ folder_path=str(output_dir),
519
+ repo_id=model_repo_id,
520
+ repo_type="model",
521
+ commit_message="Train contextual VAD turn-event model",
522
+ )
523
+ return f"https://huggingface.co/{model_repo_id}"
524
+