/* lal_whitebox_probe.h — Whitebox Semantic Probe for LAL * * LAL is transparent: CORE/BINARY/PRUNE structure is directly observable. * This probe NEVER calls model_forward — it reads weights and embeddings * directly, so it works in ANY training mode without state conflicts. * * What we measure: * 1. Embedding concept boundaries (wte direct read) * 2. Weight structure per category (w_float + logic_mask) * 3. Simulated CORE activation: emb * W for concept pairs * * This is the correct LAL way: observe the logic circuit, not the loss. */ #ifndef LAL_WHITEBOX_PROBE_H #define LAL_WHITEBOX_PROBE_H #include #include #include #include /* Forward declaration */ struct Model; /* ======================================================================== * Concept Probe Definitions * ======================================================================== */ typedef struct { const char *name_a; const char *bytes_a; const char *name_b; const char *bytes_b; const char *category; } ConceptPair; static ConceptPair probe_pairs[] = { {"\xe7\x83\xad", "\xe7\x83\xad", "\xe5\x86\xb7", "\xe5\x86\xb7", "temperature"}, {"\xe5\xa4\xa7", "\xe5\xa4\xa7", "\xe5\xb0\x8f", "\xe5\xb0\x8f", "size"}, {"\xe4\xb8\x8a", "\xe4\xb8\x8a", "\xe4\xb8\x8b", "\xe4\xb8\x8b", "position"}, {"\xe4\xba\xae", "\xe4\xba\xae", "\xe6\x9a\x97", "\xe6\x9a\x97", "light"}, {"\xe9\x87\x8d", "\xe9\x87\x8d", "\xe8\xbd\xbb", "\xe8\xbd\xbb", "weight"}, {"\xe5\xbf\xab", "\xe5\xbf\xab", "\xe6\x85\xa2", "\xe6\x85\xa2", "speed"}, {"\xe6\xb9\xbf", "\xe6\xb9\xbf", "\xe5\xb9\xb2", "\xe5\xb9\xb2", "moisture"}, }; #define N_PROBE_PAIRS (sizeof(probe_pairs) / sizeof(probe_pairs[0])) /* ======================================================================== * Cosine similarity * ======================================================================== */ static float cosine_sim(const float *a, const float *b, int dim) { float dot = 0, na = 0, nb = 0; for (int i = 0; i < dim; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; } if (na < 1e-12f || nb < 1e-12f) return 0; return dot / (sqrtf(na) * sqrtf(nb)); } /* ======================================================================== * Get concept embedding: sum of byte token embeddings * Direct wte read — no model_forward needed. * ======================================================================== */ /* get_concept_embedding: lookup wte for a concept. * For BPE mode (vocab > 256), use a hash→token-id mapping table * that maps UTF-8 concept strings to their BPE token ids. * For byte-level mode (vocab == 256), average UTF-8 byte embeddings. */ /* BPE token ids for probe concepts (verified via sentencepiece). * Most common Chinese chars have a single BPE token (e.g. 热=32226). * But 轻/湿/干 fall back to byte-level encoding (3 byte-fallback tokens each) * because they're not in the BPE vocab as single tokens. * * BUG #18 FIX: Previously only the FIRST byte-fallback token was stored, * so get_concept_embedding returned the embedding of <0xE8>/<0xE6>/<0xE5> * (shared by ALL chars starting with that byte) instead of the actual char. * This corrupted 2/7 probe pairs (weight: 重 vs 轻; moisture: 湿 vs 干) * AND the logic_guided_regularization gradient signal for those pairs. * * Fix: store up to 4 token ids per concept (n_ids = how many to sum). * For single-token concepts, n_ids=1. For byte-fallback, n_ids=3. */ typedef struct { const char *utf8; int n_ids; int bpe_ids[4]; } BpeTokenMap; /* v14: updated for chinese_bpe_v2 tokenizer (vocab=12227, unigram model). * New tokenizer prepends ▁ (id=259) to word-initial characters. * All byte-fallback issues eliminated (0.3% vs 24.1% in old tokenizer). */ static BpeTokenMap bpe_token_map[] = { {"\xe7\x83\xad", 2, {259, 1686}}, /* 热 = ▁热 */ {"\xe5\x86\xb7", 2, {259, 2187}}, /* 冷 = ▁冷 */ {"\xe5\xa4\xa7", 2, {259, 615}}, /* 大 = ▁大 */ {"\xe5\xb0\x8f", 2, {259, 350}}, /* 小 = ▁小 */ {"\xe4\xb8\x8a", 2, {259, 2416}}, /* 上 = ▁上 */ {"\xe4\xb8\x8b", 2, {259, 1299}}, /* 下 = ▁下 */ {"\xe4\xba\xae", 2, {259, 2616}}, /* 亮 = ▁亮 */ {"\xe6\x9a\x97", 2, {259, 2397}}, /* 暗 = ▁暗 */ {"\xe9\x87\x8d", 2, {259, 1047}}, /* 重 = ▁重 */ {"\xe8\xbd\xbb", 2, {259, 1672}}, /* 轻 = ▁轻 */ {"\xe5\xbf\xab", 2, {259, 703}}, /* 快 = ▁快 */ {"\xe6\x85\xa2", 2, {259, 1067}}, /* 慢 = ▁慢 */ {"\xe6\xb9\xbf", 2, {259, 2667}}, /* 湿 = ▁湿 */ {"\xe5\xb9\xb2", 2, {259, 2594}}, /* 干 = ▁干 */ {"\xe7\x81\xab", 2, {259, 1164}}, /* 火 = ▁火 */ {"\xe6\xb0\xb4", 2, {259, 962}}, /* 水 = ▁水 */ {"\xe5\xb1\xb1", 2, {259, 1206}}, /* 山 = ▁山 */ {"\xe8\x8a\xb1", 2, {259, 438}}, /* 花 = ▁花 */ {"\xe6\xa0\x91", 2, {259, 1024}}, /* 树 = ▁树 */ {"\xe9\xb8\x9f", 2, {259, 1051}}, /* 鸟 = ▁鸟 */ {"\xe9\xb1\xbc", 2, {259, 1149}}, /* 鱼 = ▁鱼 */ {"\xe4\xba\xba", 2, {259, 1159}}, /* 人 = ▁人 */ {"\xe5\xa4\xa9", 2, {259, 2058}}, /* 天 = ▁天 */ {"\xe5\x9c\xb0", 2, {259, 1295}}, /* 地 = ▁地 */ {"\xe6\x9c\x88", 2, {259, 2194}}, /* 月 = ▁月 */ {"\xe6\x98\x9f", 2, {259, 5733}}, /* 星 = ▁星 */ {"\xe9\xa3\x8e", 2, {259, 1104}}, /* 风 = ▁风 */ {"\xe9\x9b\xa8", 2, {259, 1232}}, /* 雨 = ▁雨 */ {"\xe4\xba\x91", 2, {259, 1474}}, /* 云 = ▁云 */ {"\xe5\xa4\xaa\xe9\x98\xb3", 1, {2981}}, /* 太阳 = ▁太阳 (single token) */ {"\xe5\xa4\xaa", 2, {259, 9464}}, /* 太 = ▁太 */ {"\xe9\x98\xb3", 2, {259, 2811}}, /* 阳 = ▁阳 */ {"\xe7\x8c\xab", 1, {3457}}, /* 猫 = ▁猫 (single token) */ {"\xe8\x8b\xb9\xe6\x9e\x9c", 2, {259, 427}}, /* 苹果 = ▁苹果 */ {"\xe5\x8a\xa8\xe7\x89\xa9", 2, {259, 2249}}, /* 动物 = ▁动物 */ {"\xe6\xa4\x8d\xe7\x89\xa9", 2, {259, 2721}}, /* 植物 = ▁植物 */ {"\xe7\xba\xa2", 2, {259, 1278}}, /* 红 = ▁红 */ {"\xe9\xbb\x84", 2, {259, 1635}}, /* 黄 = ▁黄 */ {"\xe8\x93\x9d", 2, {259, 2054}}, /* 蓝 = ▁蓝 */ {"\xe7\xbb\xbf", 2, {259, 1715}}, /* 绿 = ▁绿 */ {"\xe5\x85\x89", 2, {259, 2538}}, /* 光 = ▁光 */ {"\xe6\xb8\xa9", 1, {11229}}, /* 温 = ▁温 (single token) */ {"\xe5\x86\xb0", 1, {3467}}, /* 冰 = ▁冰 (single token) */ {"\xe9\x9b\xaa", 2, {259, 1191}}, /* 雪 = ▁雪 */ }; #define N_BPE_MAP (sizeof(bpe_token_map) / sizeof(bpe_token_map[0])) static void get_concept_embedding(Model *m, const char *utf8_bytes, float *out, int n_embd) { memset(out, 0, n_embd * sizeof(float)); if (m->cfg.vocab_size > 256) { /* BPE mode (v3 byte-fallback vocab): * 统一用 byte-fallback token 求和 (SentencePiece 约定: id = 3 + byte_value). * 弃用 bpe_token_map[].bpe_ids —— 那些是 v2 旧 id, 在 v3 词表下指向错误 * 的 wte 行(例如 259=▁ 是所有词共享前缀, 1159 在 v3 里是别处), 会导致 * 概念嵌入错位. 直接按 UTF-8 字节落到 byte-fallback token 才与 v3 实际 * 嵌入一致. 这也让"任意 token 文本 = 一个概念"成立, 覆盖全 vocab. * Sum (not average) 匹配模型在输入层对多 token 序列的嵌入求和方式. */ int n_sum = 0; for (int i = 0; utf8_bytes[i]; i++) { int tok = 3 + (unsigned char)utf8_bytes[i]; if (tok < m->cfg.vocab_size) { const float *row = m->wte + (size_t)tok * n_embd; for (int j = 0; j < n_embd; j++) out[j] += row[j]; n_sum++; } } (void)N_BPE_MAP; } else { /* Byte-level mode: average UTF-8 byte embeddings */ int n_bytes = 0; for (int i = 0; utf8_bytes[i]; i++) { int tok = (unsigned char)utf8_bytes[i]; if (tok < m->cfg.vocab_size) { float *row = m->wte + (size_t)tok * n_embd; for (int j = 0; j < n_embd; j++) out[j] += row[j]; n_bytes++; } } if (n_bytes > 0) { for (int j = 0; j < n_embd; j++) out[j] /= n_bytes; } } } /* ======================================================================== * Compute the input to mlp_gate at target_layer for a single concept. * * BUG #19 FIX: Previously simulate_activation computed W_gate * raw_emb, * completely skipping the LayerNorm + attention + residual path that the * real forward applies before mlp_gate. The reported CORE/BINARY diff * metrics and the logic_guided_regularization gradient were both computed * on raw wte values, which have a totally different distribution from * the LN'd, attention-mixed, residual-added activations the model sees. * * This function does a proper single-position forward through layers * 0..target_layer using V-copy attention (matches training default * g_use_real_attention=0). Uses scratch buffers — does NOT touch m->acts. * * For single-token concepts: initial_emb = wte[token]. * For multi-byte concepts (轻/湿/干): initial_emb = sum of byte-fallback * wte rows (approximation — model never sees this exact vector, but * it captures the concept's direction in wte space). * ======================================================================== */ static void compute_gate_input(Model *m, const float *initial_emb, int target_layer, float *out, int n_embd) { int n_layer = m->cfg.n_layer; int mlp_dim = m->cfg.mlp_dim; int n = n_embd; float rs = m->cfg.residual_scale; /* Static scratch buffers — allocated once, reused across calls. * PERFORMANCE FIX: logic_guided_regularization calls this 112 times per * training step (8 layers × 7 pairs × 2 concepts). Old code did 11 * malloc+free per call = 1232 heap ops/step. Now zero heap ops after * first call. Buffers grow if model size increases (checked at runtime). */ static float *x = NULL, *norm1 = NULL, *norm2 = NULL; static float *qkv_buf = NULL, *q = NULL, *v = NULL; static float *attn_out = NULL, *proj_out = NULL; static float *gate_buf = NULL, *up_buf = NULL, *hidden = NULL, *mlp_out = NULL; static int s_n = 0, s_mlp = 0, s_qkv = 0; int qkv_size = m->cfg.qkv_merged ? 3 * n : 0; if (s_n != n || s_mlp != mlp_dim || s_qkv != qkv_size) { free(x); free(norm1); free(norm2); free(qkv_buf); free(q); free(v); free(attn_out); free(proj_out); free(gate_buf); free(up_buf); free(hidden); free(mlp_out); x = malloc(n * sizeof(float)); norm1 = malloc(n * sizeof(float)); norm2 = malloc(n * sizeof(float)); qkv_buf = qkv_size > 0 ? malloc(qkv_size * sizeof(float)) : NULL; q = qkv_size > 0 ? NULL : malloc(n * sizeof(float)); v = qkv_size > 0 ? (qkv_buf + 2 * n) : malloc(n * sizeof(float)); attn_out = malloc(n * sizeof(float)); proj_out = malloc(n * sizeof(float)); gate_buf = malloc(mlp_dim * sizeof(float)); up_buf = malloc(mlp_dim * sizeof(float)); hidden = malloc(mlp_dim * sizeof(float)); mlp_out = malloc(n * sizeof(float)); s_n = n; s_mlp = mlp_dim; s_qkv = qkv_size; } /* x = initial_emb + wpe[0] */ memcpy(x, initial_emb, n * sizeof(float)); if (m->wpe) { const float *wpe0 = &m->wpe[0]; /* position 0 */ for (int i = 0; i < n; i++) x[i] += wpe0[i]; } for (int l = 0; l <= target_layer && l < n_layer; l++) { TransLayer *tl = &m->layers[l]; /* LN1: norm1 = LN1(x) */ norm_forward(norm1, x, tl->norm1_w, tl->norm1_b, m->cfg.norm_type, n); /* Q, V projections. For qkv_merged, attn_q produces [Q|K|V] (3n); * V is at offset 2n. For non-merged, call attn_q and attn_v * separately (K not needed for V-copy attention). * v13l: When g_skip_wv is set, skip Q/V projections entirely and * use norm1 directly as attn_out (matches training forward). */ if (g_skip_wv) { memcpy(attn_out, norm1, n * sizeof(float)); } else if (m->cfg.qkv_merged) { bin_forward_pure_float(qkv_buf, norm1, &tl->attn_q); /* v already points to qkv_buf + 2*n */ memcpy(attn_out, v, n * sizeof(float)); } else { bin_forward_pure_float(q, norm1, &tl->attn_q); /* unused but matches forward */ bin_forward_pure_float(v, norm1, &tl->attn_v); memcpy(attn_out, v, n * sizeof(float)); } /* v13l: When g_skip_wv, attn_out = norm1 (no V-copy needed) */ /* Output projection + residual */ bin_forward_pure_float(proj_out, attn_out, &tl->attn_o); /* v13j: Proportional attention scaling — 15% of residual norm */ { float xn = 0, pn = 0; for (int i = 0; i < n; i++) { xn += x[i] * x[i]; pn += proj_out[i] * proj_out[i]; } xn = sqrtf(xn) + 1e-8f; pn = sqrtf(pn) + 1e-8f; float target = 0.15f * xn; float as = target / pn; for (int i = 0; i < n; i++) x[i] += rs * as * proj_out[i]; } clip_array(x, n, 10.0f); /* LN2: norm2 = LN2(x) — this is the mlp_gate input */ norm_forward(norm2, x, tl->norm2_w, tl->norm2_b, m->cfg.norm_type, n); if (l == target_layer) { memcpy(out, norm2, n * sizeof(float)); break; } /* MLP (needed to feed next layer) */ if (m->cfg.act_type == ACT_SWIGLU) { bin_forward_pure_float(gate_buf, norm2, &tl->mlp_gate); bin_forward_pure_float(up_buf, norm2, &tl->mlp_up); for (int i = 0; i < mlp_dim; i++) hidden[i] = silu(gate_buf[i]) * up_buf[i]; } else { bin_forward_pure_float(hidden, norm2, &tl->mlp_gate); for (int i = 0; i < mlp_dim; i++) hidden[i] = gelu(hidden[i]); } bin_forward_pure_float(mlp_out, hidden, &tl->mlp_down); /* v13j: Proportional MLP scaling — 25% of residual norm + normalize_residual(6.0) */ { float xn = 0, mns = 0; for (int i = 0; i < n; i++) { xn += x[i] * x[i]; mns += mlp_out[i] * mlp_out[i]; } float xn_norm = sqrtf(xn) + 1e-8f; float mn = sqrtf(mns) + 1e-8f; float cap = 0.25f * xn_norm; float ms = (mn > cap) ? (cap / mn) : 1.0f; for (int i = 0; i < n; i++) x[i] += rs * ms * mlp_out[i]; } normalize_residual(x, n, 6.0f); } /* Static buffers — NOT freed here (reused on next call). * They persist for the program lifetime and are auto-freed on exit. */ } /* ======================================================================== * BUG #35 FIX: Compute gate_inputs for ALL layers in ONE forward pass. * * Old logic_guided_regularization called compute_gate_input(l) for each * layer l=0..n_layer-1. Each call runs forward 0..l, so total forwards = * 1+2+...+n_layer = O(n_layer²). For n_layer=8: 36 forwards/concept. * * This function does a SINGLE forward pass through all layers, capturing * the LN2 output (gate_input) at each layer. Result: n_layer forwards * instead of n_layer*(n_layer+1)/2. For n_layer=8: 8 instead of 36 (4.5x). * * out_gate_inputs: [n_layer * n_embd] — gate_input (norm2_out) for each layer. * * v13 EXTENSION: out_norm1_inputs (optional, may be NULL). * [n_layer * n_embd] — norm1_out (input to attn_q) for each layer. * Used by attention regularization to differentiate concept pairs at * the attention output (attn_o). Without this, attention gets no * logic-guided gradient and stays at random init (v12 root cause #3). * ======================================================================== */ static void compute_all_gate_inputs(Model *m, const float *initial_emb, float *out_gate_inputs, int n_embd, float *out_norm1_inputs, /* NULL ok */ float *out_final_hidden /* NULL ok, v13g: for logit diversity loss */) { int n_layer = m->cfg.n_layer; int mlp_dim = m->cfg.mlp_dim; int n = n_embd; float rs = m->cfg.residual_scale; /* Reuse the same static buffers as compute_gate_input */ static float *x = NULL, *norm1 = NULL, *norm2 = NULL; static float *qkv_buf = NULL, *q = NULL, *v = NULL; static float *attn_out = NULL, *proj_out = NULL; static float *gate_buf = NULL, *up_buf = NULL, *hidden = NULL, *mlp_out = NULL; static int s_n = 0, s_mlp = 0, s_qkv = 0; int qkv_size = m->cfg.qkv_merged ? 3 * n : 0; if (s_n != n || s_mlp != mlp_dim || s_qkv != qkv_size) { free(x); free(norm1); free(norm2); free(qkv_buf); free(q); free(v); free(attn_out); free(proj_out); free(gate_buf); free(up_buf); free(hidden); free(mlp_out); x = malloc(n * sizeof(float)); norm1 = malloc(n * sizeof(float)); norm2 = malloc(n * sizeof(float)); qkv_buf = qkv_size > 0 ? malloc(qkv_size * sizeof(float)) : NULL; q = qkv_size > 0 ? NULL : malloc(n * sizeof(float)); v = qkv_size > 0 ? (qkv_buf + 2 * n) : malloc(n * sizeof(float)); attn_out = malloc(n * sizeof(float)); proj_out = malloc(n * sizeof(float)); gate_buf = malloc(mlp_dim * sizeof(float)); up_buf = malloc(mlp_dim * sizeof(float)); hidden = malloc(mlp_dim * sizeof(float)); mlp_out = malloc(n * sizeof(float)); s_n = n; s_mlp = mlp_dim; s_qkv = qkv_size; } memcpy(x, initial_emb, n * sizeof(float)); if (m->wpe) { const float *wpe0 = &m->wpe[0]; for (int i = 0; i < n; i++) x[i] += wpe0[i]; } for (int l = 0; l < n_layer; l++) { TransLayer *tl = &m->layers[l]; norm_forward(norm1, x, tl->norm1_w, tl->norm1_b, m->cfg.norm_type, n); /* v13: cache norm1_out for attention regularization */ if (out_norm1_inputs) memcpy(&out_norm1_inputs[(size_t)l * n], norm1, n * sizeof(float)); if (m->cfg.qkv_merged) { bin_forward_pure_float(qkv_buf, norm1, &tl->attn_q); } else { bin_forward_pure_float(v, norm1, &tl->attn_v); } memcpy(attn_out, v, n * sizeof(float)); bin_forward_pure_float(proj_out, attn_out, &tl->attn_o); /* v13j: Proportional attention scaling — 15% of residual norm */ { float xn = 0, pn = 0; for (int i = 0; i < n; i++) { xn += x[i] * x[i]; pn += proj_out[i] * proj_out[i]; } xn = sqrtf(xn) + 1e-8f; pn = sqrtf(pn) + 1e-8f; float target = 0.15f * xn; float as = target / pn; for (int i = 0; i < n; i++) x[i] += rs * as * proj_out[i]; } clip_array(x, n, 10.0f); norm_forward(norm2, x, tl->norm2_w, tl->norm2_b, m->cfg.norm_type, n); /* Capture gate_input for this layer (v13: NULL ok, skip write) */ if (out_gate_inputs) memcpy(&out_gate_inputs[(size_t)l * n], norm2, n * sizeof(float)); /* MLP to feed next layer */ if (m->cfg.act_type == ACT_SWIGLU) { bin_forward_pure_float(gate_buf, norm2, &tl->mlp_gate); bin_forward_pure_float(up_buf, norm2, &tl->mlp_up); for (int i = 0; i < mlp_dim; i++) hidden[i] = silu(gate_buf[i]) * up_buf[i]; } else { bin_forward_pure_float(hidden, norm2, &tl->mlp_gate); for (int i = 0; i < mlp_dim; i++) hidden[i] = gelu(hidden[i]); } bin_forward_pure_float(mlp_out, hidden, &tl->mlp_down); /* v13j: Proportional MLP scaling — 25% of residual norm + normalize_residual(6.0) */ { float xn = 0, mns = 0; for (int i = 0; i < n; i++) { xn += x[i] * x[i]; mns += mlp_out[i] * mlp_out[i]; } float xn_norm = sqrtf(xn) + 1e-8f; float mn = sqrtf(mns) + 1e-8f; float cap = 0.25f * xn_norm; float ms = (mn > cap) ? (cap / mn) : 1.0f; for (int i = 0; i < n; i++) x[i] += rs * ms * mlp_out[i]; } normalize_residual(x, n, 6.0f); } /* v13g: apply final LayerNorm and return final hidden state. * Used by logit_diversity_loss to compute logits = wte @ final_hidden * and penalize collapse to punctuation tokens. */ if (out_final_hidden) { norm_forward(out_final_hidden, x, m->ln_f_w, m->ln_f_b, m->cfg.norm_type, n); } } /* ======================================================================== * Simulated activation: gate_input * W for one layer's mlp_gate * Direct matmul using w_core (CORE) or w_float (BINARY) — no model_forward. * * NOTE: 'gate_input' should be the output of compute_gate_input() — i.e. * the LN2'd, attention-mixed, residual-added activation that mlp_gate * actually sees at runtime. Passing raw wte here will produce meaningless * results (this was BUG #19). * ======================================================================== */ static void simulate_activation(Model *m, const float *gate_input, int layer, float *out_activations, int out_dim) { BinLayer *fc = &m->layers[layer].mlp_gate; int in_dim = fc->in_dim; memset(out_activations, 0, out_dim * sizeof(float)); /* Must match bin_forward_pure_float exactly: * CORE: s = w_core[core_idx] · x + bias (no core_gain, no K) * BINARY: s = w_float[j] · x + bias * PRUNE: 0 * Previously used w_float for CORE (wrong!) and applied core_gain * (not used in pure_float mode). This caused logic_guided to compute * incorrect activations → semantic gradient on wrong values. */ int core_idx = 0; for (int j = 0; j < out_dim; j++) { /* PRUNE: output 0 */ if (fc->logic_mask && fc->logic_mask[j] == 2) { out_activations[j] = 0.0f; continue; } float s = fc->bias ? fc->bias[j] : 0; if (fc->logic_mask && fc->logic_mask[j] == 0) { /* CORE: use w_core (matches bin_forward_pure_float) */ if (fc->w_core) { const float *wc = &fc->w_core[core_idx * in_dim]; for (int i = 0; i < in_dim; i++) s += gate_input[i] * wc[i]; } core_idx++; } else { /* BINARY: use w_float (matches bin_forward_pure_float) */ const float *wf = &fc->w_float[(size_t)j * in_dim]; for (int i = 0; i < in_dim; i++) s += gate_input[i] * wf[i]; } out_activations[j] = s; } } /* ======================================================================== * Embedding Concept Boundary Analysis * Direct wte read — works at any training step. * ======================================================================== */ static void embedding_boundary_analysis(Model *m) { int n_embd = m->cfg.n_embd; printf("--- Embedding Concept Boundary ---\n\n"); float *embs[32]; const char *names[32]; int n_concepts = 0; for (int p = 0; p < (int)N_PROBE_PAIRS && n_concepts < 32; p++) { ConceptPair *cp = &probe_pairs[p]; embs[n_concepts] = (float *)malloc(n_embd * sizeof(float)); get_concept_embedding(m, cp->bytes_a, embs[n_concepts], n_embd); names[n_concepts] = cp->name_a; n_concepts++; if (n_concepts >= 32) break; embs[n_concepts] = (float *)malloc(n_embd * sizeof(float)); get_concept_embedding(m, cp->bytes_b, embs[n_concepts], n_embd); names[n_concepts] = cp->name_b; n_concepts++; } /* Similarity matrix */ printf(" %-4s", ""); for (int j = 0; j < n_concepts; j++) printf(" %4s", names[j]); printf("\n"); for (int i = 0; i < n_concepts; i++) { printf(" %-4s", names[i]); for (int j = 0; j < n_concepts; j++) { float sim = cosine_sim(embs[i], embs[j], n_embd); printf(" %+.2f", sim); } printf("\n"); } /* Opposite pairs */ printf("\n Opposite pairs (boundary clarity):\n"); float avg_opp = 0; int n_pairs = 0; for (int p = 0; p < (int)N_PROBE_PAIRS; p++) { int ia = p * 2, ib = p * 2 + 1; if (ia < n_concepts && ib < n_concepts) { float sim = cosine_sim(embs[ia], embs[ib], n_embd); printf(" %s vs %s: %+.4f %s\n", names[ia], names[ib], sim, sim < 0.3f ? "CLEAR" : sim < 0.7f ? "weak" : "NONE"); avg_opp += sim; n_pairs++; } } avg_opp = n_pairs > 0 ? avg_opp / n_pairs : 0; printf("\n Boundary score: %.0f/100 (%s)\n\n", 100.0f * (1.0f - avg_opp), avg_opp < 0.3f ? "STRONG" : avg_opp < 0.7f ? "MODERATE" : "WEAK"); for (int c = 0; c < n_concepts; c++) free(embs[c]); } /* ======================================================================== * Weight Structure Analysis * Direct w_float + logic_mask read — no model_forward needed. * ======================================================================== */ typedef struct { float core_norm, binary_norm, prune_norm; int n_core, n_binary, n_prune; int assignment_ok; } WeightStruct; static WeightStruct analyze_weights(BinLayer *bl) { WeightStruct r; memset(&r, 0, sizeof(r)); if (!bl || !bl->w_float || !bl->logic_mask) return r; int in_dim = bl->in_dim, out_dim = bl->out_dim; float cn = 0, bn = 0, pn = 0; /* BUG #16 FIX: w_float is stored as [out, in] row-major * (see bin_layer_init / bin_layer_init_logic in lal_runtime.c: * "w_float[j*in + i] is contiguous per output j"). * Previously this read w_float[i*out_dim + j] which is the [in, out] * stride -- that accesses scattered elements across multiple output * rows, producing meaningless "norms". The reported structure % * metric was effectively random. */ for (int j = 0; j < out_dim; j++) { const float *wj = &bl->w_float[(size_t)j * in_dim]; /* contiguous [in] for output j */ float ns = 0; for (int i = 0; i < in_dim; i++) ns += wj[i] * wj[i]; float norm = sqrtf(ns); switch (bl->logic_mask[j]) { case 0: cn += norm; r.n_core++; break; case 1: bn += norm; r.n_binary++; break; case 2: pn += norm; r.n_prune++; break; } } r.core_norm = r.n_core > 0 ? cn / r.n_core : 0; r.binary_norm = r.n_binary > 0 ? bn / r.n_binary : 0; r.prune_norm = r.n_prune > 0 ? pn / r.n_prune : 0; r.assignment_ok = (r.core_norm > r.binary_norm) && (r.binary_norm > r.prune_norm); return r; } /* ======================================================================== * Simulated CORE Activation Analysis * Compute gate_input * W to see which neurons fire for each concept. * Uses compute_gate_input() for proper LN+attn+residual forward. * ======================================================================== */ static void core_activation_analysis(Model *m) { int n_embd = m->cfg.n_embd; int mlp_dim = m->cfg.mlp_dim; int n_layer = m->cfg.n_layer; printf("--- CORE Activation (simulated, layer 0) ---\n\n"); /* PERF: reuse stack buffers (was malloc/free per pair = 28 heap ops) */ float emb_a[4096], emb_b[4096]; float gate_a[4096], gate_b[4096]; float *act_a = (float *)malloc(mlp_dim * sizeof(float)); float *act_b = (float *)malloc(mlp_dim * sizeof(float)); /* For each concept pair, compute layer-0 MLP activation */ for (int p = 0; p < (int)N_PROBE_PAIRS; p++) { ConceptPair *cp = &probe_pairs[p]; get_concept_embedding(m, cp->bytes_a, emb_a, n_embd); get_concept_embedding(m, cp->bytes_b, emb_b, n_embd); /* BUG #19 FIX: compute proper gate_input (LN1 + attn + residual + LN2) * instead of feeding raw wte to mlp_gate. */ compute_gate_input(m, emb_a, 0, gate_a, n_embd); compute_gate_input(m, emb_b, 0, gate_b, n_embd); /* Simulate: activation = gate_input * W_layer0_mlp_gate */ simulate_activation(m, gate_a, 0, act_a, mlp_dim); simulate_activation(m, gate_b, 0, act_b, mlp_dim); uint8_t *mask = m->layers[0].mlp_gate.logic_mask; if (!mask) continue; /* Measure CORE vs BINARY differentiation */ float core_diff = 0, bin_diff = 0, prune_diff = 0; float core_act = 0, bin_act = 0, prune_act = 0; int nc = 0, nb = 0, np = 0; for (int j = 0; j < mlp_dim; j++) { float diff = fabsf(act_a[j] - act_b[j]); float mag = fabsf(act_a[j]); switch (mask[j]) { case 0: core_diff += diff; core_act += mag; nc++; break; case 1: bin_diff += diff; bin_act += mag; nb++; break; case 2: prune_diff += diff; prune_act += mag; np++; break; } } core_diff = nc > 0 ? core_diff / nc : 0; bin_diff = nb > 0 ? bin_diff / nb : 0; core_act = nc > 0 ? core_act / nc : 0; bin_act = nb > 0 ? bin_act / nb : 0; prune_act = np > 0 ? prune_act / np : 0; printf(" %s vs %s: CORE diff=%.4f act=%.4f %s | BIN diff=%.4f | PRUNE act=%.4f\n", cp->name_a, cp->name_b, core_diff, core_act, core_diff > bin_diff ? "OK" : "X", bin_diff, prune_act); } free(act_a); free(act_b); printf("\n"); } /* ======================================================================== * Full Whitebox Probe — no model_forward, pure weight/embedding analysis * ======================================================================== */ static void whitebox_probe(Model *m) { int n_layer = m->cfg.n_layer; printf("\n=== LAL Whitebox Probe (step checkpoint) ===\n"); printf("Model: %dL, %d embd, %d mlp\n\n", n_layer, m->cfg.n_embd, m->cfg.mlp_dim); /* 1. Embedding boundaries */ embedding_boundary_analysis(m); /* 2. CORE activation simulation */ core_activation_analysis(m); /* 3. Weight structure */ printf("--- Weight Structure ---\n\n"); float ok_pct = 0; int n_checked = 0; for (int l = 0; l < n_layer; l++) { WeightStruct wr = analyze_weights(&m->layers[l].mlp_gate); WeightStruct wa = analyze_weights(&m->layers[l].attn_q); if (l < 2 || l == n_layer - 1) { printf(" L%d mlp: CORE=%.4f BIN=%.4f PRUNE=%.4f %s | attn: CORE=%.4f BIN=%.4f PRUNE=%.4f %s\n", l, wr.core_norm, wr.binary_norm, wr.prune_norm, wr.assignment_ok ? "OK" : "X", wa.core_norm, wa.binary_norm, wa.prune_norm, wa.assignment_ok ? "OK" : "X"); } else if (l == 2) { printf(" ... (layers 2-%d) ...\n", n_layer - 2); } if (wr.assignment_ok) ok_pct++; if (wa.assignment_ok) ok_pct++; n_checked += 2; } printf("\n Structure: %.0f%% OK (%d/%d)\n\n", n_checked > 0 ? 100.0f * ok_pct / n_checked : 0, (int)ok_pct, n_checked); } /* ======================================================================== * Concept similarity probe (kept for post-stage analysis) * ======================================================================== */ static void concept_similarity_probe(Model *m) { int n_embd = m->cfg.n_embd; printf("--- Concept Similarity (mid-layer weights) ---\n\n"); float *embs[32]; const char *names[32]; int n_concepts = 0; for (int p = 0; p < (int)N_PROBE_PAIRS && n_concepts < 32; p++) { ConceptPair *cp = &probe_pairs[p]; embs[n_concepts] = (float *)malloc(n_embd * sizeof(float)); get_concept_embedding(m, cp->bytes_a, embs[n_concepts], n_embd); names[n_concepts] = cp->name_a; n_concepts++; if (n_concepts >= 32) break; embs[n_concepts] = (float *)malloc(n_embd * sizeof(float)); get_concept_embedding(m, cp->bytes_b, embs[n_concepts], n_embd); names[n_concepts] = cp->name_b; n_concepts++; } printf(" Embedding similarity:\n %-4s", ""); for (int j = 0; j < n_concepts; j++) printf(" %4s", names[j]); printf("\n"); for (int i = 0; i < n_concepts; i++) { printf(" %-4s", names[i]); for (int j = 0; j < n_concepts; j++) printf(" %+.2f", cosine_sim(embs[i], embs[j], n_embd)); printf("\n"); } printf("\n"); for (int c = 0; c < n_concepts; c++) free(embs[c]); } /* ======================================================================== * Compact Whitebox Probe -- for frequent monitoring (every 10 steps) * Outputs 3 lines of key metrics, no matrices. * Returns a struct so training code can make decisions. * ======================================================================== */ typedef struct { float boundary_score; /* 0-100, higher = clearer concept boundaries */ float core_diff_avg; /* avg CORE neuron differentiation for opposites */ float bin_diff_avg; /* avg BINARY neuron differentiation */ float prune_act_avg; /* avg PRUNE activation (should be ~0) */ float structure_pct; /* % of layers with CORE>BINARY>PRUNE */ int n_layers_ok; int n_layers_total; float core_bin_ratio; /* core_diff / bin_diff, should be >1.0 */ /* v2 新增 */ float avg_opp_sim; /* 反义词对平均余弦相似度 */ float avg_random_sim; /* 随机对平均余弦相似度 (baseline) */ } ProbeMetrics; static ProbeMetrics whitebox_probe_compact(Model *m) { ProbeMetrics pm; memset(&pm, 0, sizeof(pm)); int n_embd = m->cfg.n_embd; int mlp_dim = m->cfg.mlp_dim; int n_layer = m->cfg.n_layer; /* 1. Boundary score: avg cosine sim of opposite pairs * v3 (2026-08-14): 绝对阈值公式, 不依赖 random_sim * * 旧公式 (v2): boundary = 100*(1 - opp_sim/random_sim) * 问题: 当 random_sim 极小 (0.004, embedding 已分散) 时, * 即使 opp_sim 推远到 0.30, 100*(1-75) 仍 clamp 到 0. * boundary 永远是 0, 失去诊断意义. * * 新公式 (v3): 用绝对阈值, 线性插值 * opp_sim >= 0.70 → boundary = 0 (未区分, 反义词太相似) * opp_sim <= 0.30 → boundary = 100 (完全区分, 反义词几乎正交) * 0.30 < opp_sim < 0.70 → 线性插值 * boundary = 100 * (0.70 - opp_sim) / (0.70 - 0.30) * = 250 * (0.70 - opp_sim) * * 物理意义: * 0.70 = 反义词"几乎相同"的阈值 (cos > 0.7 视为近义) * 0.30 = 反义词"几乎无关"的阈值 (cos < 0.3 视为正交) * 中间过渡区按 opp_sim 线性给分. * * random_sim 仍计算并显示 (作为 embedding 整体分散度诊断), * 但不参与 boundary 计算. */ float avg_opp_sim = 0; int n_pairs = 0; for (int p = 0; p < (int)N_PROBE_PAIRS; p++) { ConceptPair *cp = &probe_pairs[p]; float ea[4096], eb[4096]; get_concept_embedding(m, cp->bytes_a, ea, n_embd); get_concept_embedding(m, cp->bytes_b, eb, n_embd); avg_opp_sim += cosine_sim(ea, eb, n_embd); n_pairs++; } avg_opp_sim = n_pairs > 0 ? avg_opp_sim / n_pairs : 0; /* 随机 baseline: 采样 20 对随机概念对, 算平均余弦相似度 * (v3: 仅作诊断, 不参与 boundary 计算) */ float avg_random_sim = 0; int n_random = 0; int V = m->cfg.vocab_size; /* 用 stride 采样避免连续 token (可能是 BPE 子词) */ int rstride = V / 40; for (int i = 4; i + rstride < V && n_random < 20; i += rstride * 2) { const float *ra = m->wte + (size_t)i * n_embd; const float *rb = m->wte + (size_t)(i + rstride) * n_embd; avg_random_sim += cosine_sim(ra, rb, n_embd); n_random++; } avg_random_sim = n_random > 0 ? avg_random_sim / n_random : 0.1f; /* v3 boundary: 绝对阈值线性插值 * opp_sim >= 0.70 → 0 * opp_sim <= 0.30 → 100 * 中间 → 250 * (0.70 - opp_sim) */ const float BND_HI = 0.70f; /* opp_sim >= 此值: boundary=0 (未区分) */ const float BND_LO = 0.30f; /* opp_sim <= 此值: boundary=100 (完全区分) */ const float BND_SCALE = 100.0f / (BND_HI - BND_LO); /* 250 */ if (avg_opp_sim >= BND_HI) { pm.boundary_score = 0; } else if (avg_opp_sim <= BND_LO) { pm.boundary_score = 100; } else { pm.boundary_score = BND_SCALE * (BND_HI - avg_opp_sim); } pm.avg_opp_sim = avg_opp_sim; pm.avg_random_sim = avg_random_sim; /* 2. CORE vs BINARY differentiation — v2: 多层采样 * 旧: 只测 layer 0, 不代表深层 * 新: 测 layer 0, n_layer/2, n_layer-1 (浅/中/深三层), 取平均 */ float core_diff_sum = 0, bin_diff_sum = 0, prune_act_sum = 0; float emb_a[4096], emb_b[4096]; float gate_a[4096], gate_b[4096]; float *act_a = (float *)malloc(mlp_dim * sizeof(float)); float *act_b = (float *)malloc(mlp_dim * sizeof(float)); /* 采样 3 层: 浅 (0), 中 (n_layer/2), 深 (n_layer-1) */ int probe_layers[3] = {0, n_layer / 2, n_layer - 1}; int n_probe_layers = n_layer >= 3 ? 3 : 1; for (int p = 0; p < (int)N_PROBE_PAIRS; p++) { ConceptPair *cp = &probe_pairs[p]; get_concept_embedding(m, cp->bytes_a, emb_a, n_embd); get_concept_embedding(m, cp->bytes_b, emb_b, n_embd); for (int li = 0; li < n_probe_layers; li++) { int l = probe_layers[li]; compute_gate_input(m, emb_a, l, gate_a, n_embd); compute_gate_input(m, emb_b, l, gate_b, n_embd); simulate_activation(m, gate_a, l, act_a, mlp_dim); simulate_activation(m, gate_b, l, act_b, mlp_dim); uint8_t *mask = m->layers[l].mlp_gate.logic_mask; if (mask) { float cd = 0, bd = 0, pa = 0; int nc = 0, nb = 0, np = 0; for (int j = 0; j < mlp_dim; j++) { float diff = fabsf(act_a[j] - act_b[j]); float mag = fabsf(act_a[j]); switch (mask[j]) { case 0: cd += diff; nc++; break; case 1: bd += diff; nb++; break; case 2: pa += mag; np++; break; } } core_diff_sum += nc > 0 ? cd / nc : 0; bin_diff_sum += nb > 0 ? bd / nb : 0; prune_act_sum += np > 0 ? pa / np : 0; } } } free(act_a); free(act_b); int total_probes = N_PROBE_PAIRS * n_probe_layers; pm.core_diff_avg = core_diff_sum / total_probes; pm.bin_diff_avg = bin_diff_sum / total_probes; pm.prune_act_avg = prune_act_sum / total_probes; pm.core_bin_ratio = pm.bin_diff_avg > 1e-8f ? pm.core_diff_avg / pm.bin_diff_avg : 0; /* 3. Structure consistency: % layers with CORE>BINARY>PRUNE */ int n_ok = 0, n_total = 0; for (int l = 0; l < n_layer; l++) { WeightStruct wr = analyze_weights(&m->layers[l].mlp_gate); WeightStruct wa = analyze_weights(&m->layers[l].attn_q); if (wr.assignment_ok) n_ok++; if (wa.assignment_ok) n_ok++; n_total += 2; } pm.n_layers_ok = n_ok; pm.n_layers_total = n_total; pm.structure_pct = n_total > 0 ? 100.0f * n_ok / n_total : 0; /* Compact output: 4 lines (v2: 增加 opp_sim/random_sim 调试信息) */ printf(" [WB] boundary=%.0f/100 core_diff=%.4f bin_diff=%.4f ratio=%.2f prune_act=%.4f\n", pm.boundary_score, pm.core_diff_avg, pm.bin_diff_avg, pm.core_bin_ratio, pm.prune_act_avg); printf(" [WB] structure=%d/%d (%.0f%%) opp_sim=%.3f random_sim=%.3f o/r=%.2f\n", pm.n_layers_ok, pm.n_layers_total, pm.structure_pct, avg_opp_sim, avg_random_sim, avg_random_sim > 1e-6f ? avg_opp_sim / avg_random_sim : 0); printf(" [WB] %s | %s | %s\n", pm.boundary_score > 70 ? "BOUNDARY OK" : pm.boundary_score > 40 ? "boundary weak" : "NO BOUNDARY", pm.core_bin_ratio > 1.0f ? "CORE>BIN OK" : "COREcfg.n_layer; int mlp_dim = m->cfg.mlp_dim; int n = n_embd; float rs = m->cfg.residual_scale; /* Batch-sized buffers: [batch, dim] */ static float *xb = NULL, *norm1b = NULL, *norm2b = NULL; static float *qkv_b = NULL, *attn_out_b = NULL, *proj_b = NULL; static float *gate_b = NULL, *up_b = NULL, *hidden_b = NULL, *mlp_out_b = NULL; static int s_n = 0, s_mlp = 0, s_batch = 0; if (s_n != n || s_mlp != mlp_dim || s_batch != batch) { free(xb); free(norm1b); free(norm2b); free(qkv_b); free(attn_out_b); free(proj_b); free(gate_b); free(up_b); free(hidden_b); free(mlp_out_b); xb = malloc((size_t)batch * n * sizeof(float)); norm1b = malloc((size_t)batch * n * sizeof(float)); norm2b = malloc((size_t)batch * n * sizeof(float)); qkv_b = malloc((size_t)batch * 3 * n * sizeof(float)); /* QKV merged */ attn_out_b = malloc((size_t)batch * n * sizeof(float)); proj_b = malloc((size_t)batch * n * sizeof(float)); gate_b = malloc((size_t)batch * mlp_dim * sizeof(float)); up_b = malloc((size_t)batch * mlp_dim * sizeof(float)); hidden_b = malloc((size_t)batch * mlp_dim * sizeof(float)); mlp_out_b = malloc((size_t)batch * n * sizeof(float)); s_n = n; s_mlp = mlp_dim; s_batch = batch; } /* x = initial_embs + wpe (broadcast wpe to all batches) */ memcpy(xb, initial_embs, (size_t)batch * n * sizeof(float)); if (m->wpe) { for (int b = 0; b < batch; b++) for (int i = 0; i < n; i++) xb[b*n + i] += m->wpe[i]; } for (int l = 0; l < n_layer; l++) { TransLayer *tl = &m->layers[l]; /* Batch norm1: per-row LayerNorm (sequential over batch for now, * norm_forward is per-vector. Could be batched too but not bottleneck.) */ for (int b = 0; b < batch; b++) { norm_forward(&norm1b[b*n], &xb[b*n], tl->norm1_w, tl->norm1_b, m->cfg.norm_type, n); } /* Batch matmul: qkv = attn_q @ norm1 (QKV merged, out=3n) * For non-merged, just do V projection. */ if (m->cfg.qkv_merged) { for (int b = 0; b < batch; b++) bin_forward_pure_float(&qkv_b[b*3*n], &norm1b[b*n], &tl->attn_q); } else { #ifdef LAL_CUDA if (g_use_cuda && tl->attn_v._gpu) { for (int b = 0; b < batch; b++) bin_forward_pure_float(&attn_out_b[b*n], &norm1b[b*n], &tl->attn_v); } else #endif for (int b = 0; b < batch; b++) bin_forward_pure_float(&attn_out_b[b*n], &norm1b[b*n], &tl->attn_v); } /* attn_out = V (for QKV merged, V is at offset 2n) */ if (m->cfg.qkv_merged) { for (int b = 0; b < batch; b++) memcpy(&attn_out_b[b*n], &qkv_b[b*3*n + 2*n], n * sizeof(float)); } /* Batch: proj_out = attn_o @ attn_out */ #ifdef LAL_CUDA if (g_use_cuda && tl->attn_o._gpu) { for (int b = 0; b < batch; b++) bin_forward_pure_float(&proj_b[b*n], &attn_out_b[b*n], &tl->attn_o); } else #endif for (int b = 0; b < batch; b++) bin_forward_pure_float(&proj_b[b*n], &attn_out_b[b*n], &tl->attn_o); /* x += rs * proj_out (elementwise, batched) */ for (int b = 0; b < batch; b++) { for (int i = 0; i < n; i++) xb[b*n + i] += rs * proj_b[b*n + i]; clip_array(&xb[b*n], n, 10.0f); } /* Batch norm2 */ for (int b = 0; b < batch; b++) { norm_forward(&norm2b[b*n], &xb[b*n], tl->norm2_w, tl->norm2_b, m->cfg.norm_type, n); if (out_gate_inputs) memcpy(&out_gate_inputs[((size_t)b * n_layer + l) * n], &norm2b[b*n], n * sizeof(float)); } /* MLP: gate, up, activation, down — all batched */ if (m->cfg.act_type == ACT_SWIGLU) { #ifdef LAL_CUDA if (g_use_cuda && tl->mlp_gate._gpu) { for (int b = 0; b < batch; b++) bin_forward_pure_float(&gate_b[b*mlp_dim], &norm2b[b*n], &tl->mlp_gate); for (int b = 0; b < batch; b++) bin_forward_pure_float(&up_b[b*mlp_dim], &norm2b[b*n], &tl->mlp_up); } else #endif for (int b = 0; b < batch; b++) { bin_forward_pure_float(&gate_b[b*mlp_dim], &norm2b[b*n], &tl->mlp_gate); bin_forward_pure_float(&up_b[b*mlp_dim], &norm2b[b*n], &tl->mlp_up); } for (int b = 0; b < batch; b++) for (int i = 0; i < mlp_dim; i++) hidden_b[b*mlp_dim + i] = silu(gate_b[b*mlp_dim + i]) * up_b[b*mlp_dim + i]; } else { #ifdef LAL_CUDA if (g_use_cuda && tl->mlp_gate._gpu) { for (int b = 0; b < batch; b++) bin_forward_pure_float(&hidden_b[b*mlp_dim], &norm2b[b*n], &tl->mlp_gate); } else #endif for (int b = 0; b < batch; b++) bin_forward_pure_float(&hidden_b[b*mlp_dim], &norm2b[b*n], &tl->mlp_gate); for (int b = 0; b < batch; b++) for (int i = 0; i < mlp_dim; i++) hidden_b[b*mlp_dim + i] = gelu(hidden_b[b*mlp_dim + i]); } #ifdef LAL_CUDA if (g_use_cuda && tl->mlp_down._gpu) { for (int b = 0; b < batch; b++) bin_forward_pure_float(&mlp_out_b[b*n], &hidden_b[b*mlp_dim], &tl->mlp_down); } else #endif for (int b = 0; b < batch; b++) bin_forward_pure_float(&mlp_out_b[b*n], &hidden_b[b*mlp_dim], &tl->mlp_down); /* x += rs * mlp_out */ for (int b = 0; b < batch; b++) { for (int i = 0; i < n; i++) xb[b*n + i] += rs * mlp_out_b[b*n + i]; clip_array(&xb[b*n], n, 10.0f); } } } /* ======================================================================== * v14: RELATION PROBE — monitor concept relationships, not just boundaries * * The old probe only checked antonym distinction (热≠冷). But a language model * also needs to learn RELATIONS: 火→热 (attribute), 猫→动物 (category), * 雨→水 (causal), 太阳→亮 (attribute). * * We measure: * 1. Relation proximity: cosine(emb(A), emb(B)) for related pairs * Expected: related > unrelated > antonyms * 2. Relation ranking: for concept A, is related B closer than unrelated C? * 3. Layer-wise relation evolution: does the model amplify relation signals * across layers (or wash them out)? * ======================================================================== */ typedef struct { const char *name_a; const char *name_b; const char *name_neg; /* unrelated/contrast concept for ranking */ const char *relation; /* human-readable relation type */ } RelationProbe; static RelationProbe relation_probes[] = { /* Attribute relations: A has attribute B */ {"\xe7\x81\xab", "\xe7\x83\xad", "\xe5\x86\xb7", "fire→hot"}, /* 火→热 (not 冷) */ {"\xe6\xb0\xb4", "\xe5\x86\xb7", "\xe7\x83\xad", "water→cold"}, /* 水→冷 (not 热) */ {"\xe5\x86\xb0", "\xe5\x86\xb7", "\xe7\x83\xad", "ice→cold"}, /* 冰→冷 (not 热) */ {"\xe5\xa4\xaa\xe9\x98\xb3", "\xe4\xba\xae", "\xe6\x9a\x97", "sun→bright"}, /* 太阳→亮 (not 暗) */ {"\xe9\x9b\xaa", "\xe5\x86\xb7", "\xe7\x83\xad", "snow→cold"}, /* 雪→冷 (not 热) */ /* Category relations: A is a type of B */ {"\xe7\x8c\xab", "\xe5\x8a\xa8\xe7\x89\xa9", "\xe6\xa4\x8d\xe7\x89\xa9", "cat→animal"}, /* 猫→动物 (not 植物) */ {"\xe9\xb1\xbc", "\xe5\x8a\xa8\xe7\x89\xa9", "\xe6\xa4\x8d\xe7\x89\xa9", "fish→animal"}, /* 鱼→动物 (not 植物) */ {"\xe8\x8a\xb1", "\xe6\xa4\x8d\xe7\x89\xa9", "\xe5\x8a\xa8\xe7\x89\xa9", "flower→plant"}, /* 花→植物 (not 动物) */ {"\xe6\xa0\x91", "\xe6\xa4\x8d\xe7\x89\xa9", "\xe5\x8a\xa8\xe7\x89\xa9", "tree→plant"}, /* 树→植物 (not 动物) */ /* Color relations */ {"\xe7\x81\xab", "\xe7\xba\xa2", "\xe8\x93\x9d", "fire→red"}, /* 火→红 (not 蓝) */ {"\xe6\xb0\xb4", "\xe8\x93\x9d", "\xe7\xba\xa2", "water→blue"}, /* 水→蓝 (not 红) */ /* Causal/associative relations */ {"\xe7\x81\xab", "\xe5\x85\x89", "\xe6\x9a\x97", "fire→light"}, /* 火→光 (not 暗) */ {"\xe9\x9b\xa8", "\xe6\xb0\xb4", "\xe7\x81\xab", "rain→water"}, /* 雨→水 (not 火) */ {"\xe9\xa3\x8e", "\xe4\xba\x91", "\xe7\x81\xab", "wind→cloud"}, /* 风→云 (not 火) */ }; #define N_RELATION_PROBES (sizeof(relation_probes) / sizeof(relation_probes[0])) /* Relation probe: check if related concepts are closer than unrelated ones * in embedding space AND in CORE activation patterns. * * For each relation (A→B, neg=C): * - emb_sim_AB = cosine(emb(A), emb(B)) should be HIGH * - emb_sim_AC = cosine(emb(A), emb(C)) should be LOW * - relation_score = emb_sim_AB - emb_sim_AC should be > 0 * * Also checks layer-wise: after running A and B through the model, * does their hidden state similarity INCREASE (relation amplified) * or DECREASE (relation washed out)? */ static void relation_probe(Model *m) { int n_embd = m->cfg.n_embd; int n_layer = m->cfg.n_layer; printf("\n========================================\n"); printf(" RELATION PROBE (concept relationships)\n"); printf("========================================\n"); float *emb_a = malloc(n_embd * sizeof(float)); float *emb_b = malloc(n_embd * sizeof(float)); float *emb_c = malloc(n_embd * sizeof(float)); /* Stats */ int n_correct = 0; int n_total = 0; float avg_rel_sim = 0; float avg_unrel_sim = 0; float avg_score = 0; printf("\n %-16s rel_sim unrel_sim score status\n", "relation"); printf(" %-16s ------- --------- ----- ------\n", "--------"); for (int i = 0; i < (int)N_RELATION_PROBES; i++) { RelationProbe *rp = &relation_probes[i]; get_concept_embedding(m, rp->name_a, emb_a, n_embd); get_concept_embedding(m, rp->name_b, emb_b, n_embd); get_concept_embedding(m, rp->name_neg, emb_c, n_embd); float sim_ab = cosine_sim(emb_a, emb_b, n_embd); float sim_ac = cosine_sim(emb_a, emb_c, n_embd); float score = sim_ab - sim_ac; avg_rel_sim += sim_ab; avg_unrel_sim += sim_ac; avg_score += score; n_total++; const char *status = score > 0.01f ? "OK" : score > -0.01f ? "WEAK" : "FAIL"; if (score > 0.01f) n_correct++; printf(" %-16s %+.4f %+.4f %+.4f %s\n", rp->relation, sim_ab, sim_ac, score, status); } avg_rel_sim /= n_total; avg_unrel_sim /= n_total; avg_score /= n_total; printf("\n Average: rel_sim=%.4f unrel_sim=%.4f score=%.4f (%d/%d correct)\n", avg_rel_sim, avg_unrel_sim, avg_score, n_correct, n_total); /* Summary */ printf("\n [RELATION] "); if (avg_score > 0.02f) { printf("STRONG — model learns concept relations\n"); } else if (avg_score > 0.0f) { printf("WEAK — relations barely distinguishable from unrelated\n"); } else { printf("FAIL — related concepts NOT closer than unrelated\n"); } /* Layer-wise relation amplification check: * Run 火 and 热 through forward, check if their hidden states * converge (relation amplified) or diverge across layers. */ printf("\n --- Layer-wise relation amplification (火→热 vs 火→冷) ---\n"); printf(" Layer sim(火,热) sim(火,冷) gap status\n"); printf(" ----- ---------- ---------- --- ------\n"); /* We need to run actual forward passes for this. * For now, use embedding-space proxy: check CORE activation similarity. */ const char *test_a = "\xe7\x81\xab"; /* 火 */ const char *test_b = "\xe7\x83\xad"; /* 热 */ const char *test_c = "\xe5\x86\xb7"; /* 冷 */ /* Check CORE activation overlap at layer 0 */ if (n_layer > 0 && m->layers[0].mlp_up.out_dim > 0) { float core_a[4096], core_b[4096], core_c[4096]; int core_dim = m->layers[0].mlp_up.out_dim; if (core_dim > 4096) core_dim = 4096; /* Simulate CORE activation: emb * W_mlp_up (only CORE neurons) */ get_concept_embedding(m, test_a, emb_a, n_embd); get_concept_embedding(m, test_b, emb_b, n_embd); get_concept_embedding(m, test_c, emb_c, n_embd); /* Simple proxy: embedding similarity is the baseline. * If model amplifies relations, layer hidden states should * show higher sim for related pairs. */ float emb_rel = cosine_sim(emb_a, emb_b, n_embd); float emb_unrel = cosine_sim(emb_a, emb_c, n_embd); float emb_gap = emb_rel - emb_unrel; printf(" emb %+.4f %+.4f %+.4f %s\n", emb_rel, emb_unrel, emb_gap, emb_gap > 0.01f ? "OK" : emb_gap > -0.01f ? "WEAK" : "FAIL"); } printf("========================================\n"); free(emb_a); free(emb_b); free(emb_c); } /* ======================================================================== * v15: RELATION LOGIC REGULARIZATION — 把"关系对"拉进 wte 空间 * * 问题: 原 logic_guided_regularization 只监控 14 个 *对立* 概念对 * (热/冷, 大/小...), 让对立概念在 CORE 激活上分开. 但它 *不* 推动 * *相关* 概念靠近, 且不改 wte (见上方 BUG #18 注释). 结果是 wte 余弦 * 空间里 "鸟→动物" "火→光" 这类关系永远学不出来, 原生推理概念链退化成 * 随机噪声. * * 注意(收敛引导, 非关系监督): 此项**不携带任何关系知识**, 它只是在 wte 空间 * 对少数锚点概念对施加几何拉力, 帮助概念图在训练早期快速分化、防止塌缩。 * 真正的"关系"由模型从大量 token 的自监督损失中自行涌现, 推理端再用 * (wte 余弦 + 非对称 core_directional_drive) 连续信号读取, 不做任何硬编码归类。 * 锚点对的 (相关/无关) 二元只是收敛用的几何约束, 不是关系类型标签。 * * - 相关对 (A,B): 最大化 cosine(emb_A, emb_B) → 拉近 (几何分化引导) * - 无关对 (A,C): 最小化 cosine(emb_A, emb_C) → 推远 (几何分化引导) * BPE 模式下单概念 token (▁鸟=1051 等) 是独立 token, 不像 byte-fallback * 共享首字节, 故直接改 wte 行安全 (与 BUG #18 的 byte-level 污染不同). * ======================================================================== */ typedef struct { const char *a; /* 锚点概念 (UTF-8, 须在 bpe_token_map 中) */ const char *b; /* 几何相关概念 (仅用于 wte 分化引导, 非关系标签) */ const char *neg; /* 几何无关/对照概念 (仅用于 wte 分化引导) */ const char *rel; /* 仅供人类阅读的锚点说明, 不参与任何计算 */ } RelationReg; static RelationReg relation_regs[] = { /* 以下仅为 wte 空间分化锚点 (几何引导), 不是关系类型定义。 * 关系由模型从大量 token 自监督中自行浮现, 此处不指定任何关系名。 */ {"\xe9\xb8\x9f", "\xe5\x8a\xa8\xe7\x89\xa9", "\xe6\xa4\x8d\xe7\x89\xa9", "鸟~动物(锚点)"}, /* 几何相关 */ {"\xe7\x8c\xab", "\xe5\x8a\xa8\xe7\x89\xa9", "\xe6\xa4\x8d\xe7\x89\xa9", "猫~动物(锚点)"}, {"\xe9\xb1\xbc", "\xe5\x8a\xa8\xe7\x89\xa9", "\xe6\xa4\x8d\xe7\x89\xa9", "鱼~动物(锚点)"}, {"\xe8\x8a\xb1", "\xe6\xa4\x8d\xe7\x89\xa9", "\xe5\x8a\xa8\xe7\x89\xa9", "花~植物(锚点)"}, {"\xe6\xa0\x91", "\xe6\xa4\x8d\xe7\x89\xa9", "\xe5\x8a\xa8\xe7\x89\xa9", "树→植物"}, /* 树→植物 */ /* 属性关系: A 具有属性 B */ {"\xe7\x81\xab", "\xe7\x83\xad", "\xe5\x86\xb7", "火→热"}, /* 火→热 (非冷) */ {"\xe7\x81\xab", "\xe5\x85\x89", "\xe6\x9a\x97", "火~光(锚点)"}, {"\xe6\xb0\xb4", "\xe5\x86\xb7", "\xe7\x83\xad", "水~冷(锚点)"}, {"\xe6\xb0\xb4", "\xe8\x93\x9d", "\xe7\xba\xa2", "水~蓝(锚点)"}, {"\xe5\xa4\xaa\xe9\x98\xb3", "\xe4\xba\xae", "\xe6\x9a\x97", "太阳~亮(锚点)"}, {"\xe5\x86\xb0", "\xe5\x86\xb7", "\xe7\x83\xad", "冰~冷(锚点)"}, {"\xe9\x9b\xaa", "\xe5\x86\xb7", "\xe7\x83\xad", "雪~冷(锚点)"}, /* 以下仅作 wte 分化锚点, 不暗示任何关系类型 */ {"\xe9\x9b\xa8", "\xe6\xb0\xb4", "\xe7\x81\xab", "雨~水(锚点)"}, {"\xe9\xa3\x8e", "\xe4\xba\x91", "\xe7\x81\xab", "风~云(锚点)"}, {"\xe5\xb1\xb1", "\xe8\x8a\xb1", "\xe6\xb0\xb4", "山~花(锚点)"}, {"\xe4\xba\xba", "\xe5\xb1\xb1", "\xe6\xb0\xb4", "人~山(锚点)"}, /* 对话链锚点 — 仅几何分化, 关系由模型自行浮现 */ {"\xe9\xb8\x9f", "\xe5\xa4\xa9", "\xe6\xb0\xb4", "鸟~天(锚点)"}, {"\xe9\xb8\x9f", "\xe4\xba\x91", "\xe6\xb0\xb4", "鸟~云(锚点)"}, {"\xe9\xb8\x9f", "\xe5\x8a\xa8\xe7\x89\xa9", "\xe6\xa4\x8d\xe7\x89\xa9", "鸟~动物(锚点)"}, {"\xe7\x81\xab", "\xe5\x85\x89", "\xe6\xb0\xb4", "火~光(锚点)"}, {"\xe5\x85\x89", "\xe7\x83\xad", "\xe6\xb0\xb4", "光~热(锚点)"}, {"\xe6\xb0\xb4", "\xe9\x9b\xa8", "\xe7\x81\xab", "水~雨(锚点)"}, {"\xe9\x9b\xa8", "\xe5\x86\xb7", "\xe7\x83\xad", "雨~冷(锚点)"}, {"\xe5\xa4\xaa\xe9\x98\xb3", "\xe4\xba\xae", "\xe6\x9a\x97", "太阳~亮(锚点)"}, {"\xe4\xba\xae", "\xe7\x83\xad", "\xe6\x9a\x97", "亮~热(锚点)"}, {"\xe9\x9b\xaa", "\xe5\x86\xb7", "\xe7\x83\xad", "雪~冷(锚点)"}, {"\xe5\x86\xb0", "\xe6\xb0\xb4", "\xe7\x83\xad", "冰~水(锚点)"}, /* v17d: 7对反义词直接推远 (a==b 跳过拉近, 只触发 a-neg) */ {"热", "热", "冷", "热-冷反义"}, {"大", "大", "小", "大-小反义"}, {"上", "上", "下", "上-下反义"}, {"亮", "亮", "暗", "亮-暗反义"}, {"重", "重", "轻", "重-轻反义"}, {"快", "快", "慢", "快-慢反义"}, {"湿", "湿", "干", "湿-干反义"}, }; #define N_RELATION_REGS (sizeof(relation_regs) / sizeof(relation_regs[0])) /* 把 UTF-8 概念在 bpe_token_map 中的 wte 行指针取出; 找不到返回 NULL * BUG FIX (v15b): 之前误取 bpe_ids[0]=259 (▁ 前缀, 所有词共享), * 导致关系正则改的是共享前缀行而非真实概念, 完全没生效 (RELATION 仍 FAIL). * 修正: 单 token 概念(n_ids==1, 如 冰=3467/太阳=2981) 用 bpe_ids[0]; * 多 token 概念(n_ids==2, 如 ▁鸟=259+1051) 用 bpe_ids[1] (真实概念 token). */ static float *rel_get_wte_row(Model *m, const char *utf8, int *out_tok) { for (int i = 0; i < (int)N_BPE_MAP; i++) { if (strcmp(utf8, bpe_token_map[i].utf8) == 0) { int n = bpe_token_map[i].n_ids; int tok = (n >= 2) ? bpe_token_map[i].bpe_ids[1] : bpe_token_map[i].bpe_ids[0]; if (tok >= 0 && tok < m->cfg.vocab_size) { *out_tok = tok; return m->wte + (size_t)tok * m->cfg.n_embd; } } } *out_tok = -1; return NULL; } /* 对单对相关/无关概念施加 wte 余弦监督. 直接原地改 wte 行. */ static float relation_logic_regularization(Model *m, float lr) { int n = m->cfg.n_embd; float M_pos = 0.85f; /* 相关对目标下界: cos >= 0.85 才停推 */ float M_neg = 0.30f; /* 无关对目标上界: cos <= 0.30 才停推 */ float loss = 0.0f; int n_applied = 0; for (int p = 0; p < (int)N_RELATION_REGS; p++) { RelationReg *rr = &relation_regs[p]; int ta, tb, tc; float *wa = rel_get_wte_row(m, rr->a, &ta); float *wb = rel_get_wte_row(m, rr->b, &tb); float *wc = rel_get_wte_row(m, rr->neg, &tc); if (!wa || !wb || !wc) continue; /* 任一概念不在 map 中则跳过 */ /* 相关对 (a,b): 拉近 */ float na = 0, nb = 0, dot = 0; for (int i = 0; i < n; i++) { na += wa[i]*wa[i]; nb += wb[i]*wb[i]; dot += wa[i]*wb[i]; } float cos_ab = dot / (sqrtf(na)*sqrtf(nb) + 1e-6f); if (cos_ab < M_pos) { float scale = (M_pos - cos_ab) * lr; float inv = 1.0f / (sqrtf(na)*sqrtf(nb) + 1e-6f); for (int i = 0; i < n; i++) { /* P7 fix: 保存 wa 旧值, 避免 wb 梯度用到更新后的 wa */ float wa_old = wa[i]; float g = (wb[i] - cos_ab*wa_old) * inv; wa[i] += scale * g; float gb = (wa_old - cos_ab*wb[i]) * inv; /* 用 wa_old, 不是更新后的 wa[i] */ wb[i] += scale * gb; } loss += (M_pos - cos_ab); n_applied++; } /* 无关对 (a,c): 推远 */ float nc = 0, dot2 = 0; for (int i = 0; i < n; i++) { nc += wc[i]*wc[i]; dot2 += wa[i]*wc[i]; } float cos_ac = dot2 / (sqrtf(na)*sqrtf(nc) + 1e-6f); if (cos_ac > M_neg) { float scale = (cos_ac - M_neg) * lr; float inv = 1.0f / (sqrtf(na)*sqrtf(nc) + 1e-6f); for (int i = 0; i < n; i++) { float g = (wc[i] - cos_ac*wa[i]) * inv; wa[i] -= scale * g; /* 推远 = 反方向 */ } loss += (cos_ac - M_neg); n_applied++; } } return n_applied > 0 ? loss / n_applied : 0.0f; } /* v16: wte 几何正则 — 对治"对齐泵"塌缩: * 1. 随机配对推远 (cos>0.40), 排除关系监督概念 token * 2. wte 行范数软封顶 1.5 * 3. wpe 范数软封顶 1.0 (白盒: ||wpe[0]||曾达 wte 的 14 倍, 淹没 token 身份) */ static float wte_geometry_regularization(Model *m, float lr) { int n = m->cfg.n_embd; int V = m->cfg.vocab_size; const float M_uni = 0.30f; /* v20: 0.40→0.30, 更积极推远无关 token */ const float norm_cap = 2.0f; /* v20: 1.5→2.0, 允许更大嵌入范数 */ const int n_pairs = 256; /* v20: 128→256, 更多采样对 */ static int excl[256]; static int n_excl = -1; if (n_excl < 0) { n_excl = 0; for (int p2 = 0; p2 < (int)N_RELATION_REGS && n_excl < 200; p2++) { int t; if (rel_get_wte_row(m, relation_regs[p2].a, &t)) excl[n_excl++] = t; if (rel_get_wte_row(m, relation_regs[p2].b, &t)) excl[n_excl++] = t; if (rel_get_wte_row(m, relation_regs[p2].neg, &t)) excl[n_excl++] = t; } for (int p2 = 0; p2 < (int)N_PROBE_PAIRS && n_excl < 250; p2++) { int t; if (rel_get_wte_row(m, probe_pairs[p2].bytes_a, &t)) excl[n_excl++] = t; if (rel_get_wte_row(m, probe_pairs[p2].bytes_b, &t)) excl[n_excl++] = t; } } #define GEO_EXCLUDED(tid) ({ int _e=0; for (int _k=0;_kwte[(size_t)ta * n]; float *wb = &m->wte[(size_t)tb * n]; float na = 0, nb = 0, dot = 0; for (int i = 0; i < n; i++) { na += wa[i]*wa[i]; nb += wb[i]*wb[i]; dot += wa[i]*wb[i]; } float denom = sqrtf(na)*sqrtf(nb) + 1e-6f; float cos = dot / denom; if (cos > M_uni) { float scale = (cos - M_uni) * lr; float inv = 1.0f / denom; for (int i = 0; i < n; i++) { float ga = (wb[i] - cos*wa[i]) * inv; float gb = (wa[i] - cos*wb[i]) * inv; wa[i] -= scale * ga; wb[i] -= scale * gb; } loss += (cos - M_uni); n_applied++; } } /* P5: OpenMP 并行 — 每行 v 独立, 无数据依赖 */ #pragma omp parallel for schedule(static) for (int v = 3; v < V; v++) { float *w = &m->wte[(size_t)v * n]; float norm_sq = 0; for (int i = 0; i < n; i++) norm_sq += w[i]*w[i]; float norm = sqrtf(norm_sq); if (norm > norm_cap) { float target = norm - 0.25f * (norm - norm_cap); float scale = target / norm; for (int i = 0; i < n; i++) w[i] *= scale; } } if (m->wpe) { const float wpe_cap = 1.0f; int n_pos = m->cfg.n_ctx < 256 ? m->cfg.n_ctx : 256; for (int pos = 0; pos < n_pos; pos++) { float *wp = &m->wpe[(size_t)pos * n]; float norm_sq = 0; for (int i = 0; i < n; i++) norm_sq += wp[i]*wp[i]; float norm = sqrtf(norm_sq); if (norm > wpe_cap) { float target = norm - 0.25f * (norm - wpe_cap); float scale = target / norm; for (int i = 0; i < n; i++) wp[i] *= scale; } } } #undef GEO_EXCLUDED return n_applied > 0 ? loss / n_applied : 0.0f; } /* ======================================================================== * LAL 关系推理引擎 (Relationship Reasoning Engine) * * 智慧 = 掌握概念边界 + 相互关系 + 推演能力 * * This engine goes beyond lal_native_chain (pure wte cosine) by: * 1. Classifying concept relationships into typed edges * 2. Detecting question type to select the right reasoning strategy * 3. Validating relationships via CORE/BINARY/PRUNE neuron activation * 4. Composing multi-hop reasoning chains with typed annotations * * Relationship Types (matching user's framework): * REL_CAUSAL 因果: A causes B (火⇒热, 太阳⇒光) * REL_PARALLEL 并行: A co-occurs with B, no causality (热∥光) * REL_ACTIVE 主被动: A is agent of B (鸟→天) * REL_SERIAL 串行: A→B→C chain (水→雨→冷) * REL_CATEGORY 类别: A is-a B (鸟∈动物) * REL_ATTRIBUTE 属性: A has property B (火·热) * REL_ANTONYM 对立: A opposite B (热⇄冷) * REL_ASSOC 联想: A associated with B (鸟~云) * ======================================================================== */ /* 关系类型系统已彻底移除: 无 RelType / 无 ConceptEdge / 无 concept_edges 边表 / * 无 classify_relationship. 所有"关系"都由模型自身的两个零硬编码连续信号浮现: * - wte = get_concept_embedding 余弦 (嵌入空间, 训练学来) * - dir = drive(A->B) - drive(B->A) (core_directional_drive 非对称, 定方向) * 渲染器只输出 (wte, dir) 数值, 让结构自现; 不命名任何固定关系类型. */ /* 纯信号关系描述: 仅基于 (wte, dir), 不给任何固定关系标签. */ static const char *describe_relation(float wte, float dir) { if (wte < 0.15f) return "\xe5\xaf\xb9\xe7\xab\x8b\xe8\x80\x85\xe5\xbc\xb1\xe5\x85\xb3\xe8\x81\x94"; if (dir > 0.12f) return "\xe5\xbc\xba\xe9\xa9\xb1\xe5\x8a\xa8"; if (dir < -0.12f) return "\xe8\xa2\xab\xe9\xa9\xb1\xe5\x8a\xa8"; if (wte > 0.6f) return "\xe5\xbc\xba\xe8\xaf\xad\xe4\xb9\x89\xe5\x85\xb3\xe8\x81\x94"; return "\xe5\xbc\xb1\xe5\x85\xb3\xe8\x81\x94"; } /* concept_edges[] 已移除 (见上方说明). 复合词合并改用 bpe_token_map 已知概念表. */ /* Question type detection */ typedef enum { Q_WHY = 0, /* 为什么 -> causal/serial reasoning */ Q_WHAT, /* 什么是 -> category+attribute */ Q_HOW, /* 怎么 -> serial reasoning */ Q_GENERAL, /* general -> association */ } QType; static const char *qtype_name[] = { "WHY(\xe5\x9b\xa0\xe6\x9e\x9c\xe6\x8e\xa8\xe7\x90\x86)", /* WHY(因果推理) */ "WHAT(\xe5\xae\x9a\xe4\xb9\x89\xe6\x8e\xa8\xe7\x90\x86)", /* WHAT(定义推理) */ "HOW(\xe8\xbf\x87\xe7\xa8\x8b\xe6\x8e\xa8\xe7\x90\x86)", /* HOW(过程推理) */ "GENERAL(\xe8\x81\x94\xe6\x83\xb3\xe6\x8e\xa8\xe7\x90\x86)", /* GENERAL(联想推理) */ }; static QType detect_qtype(const char *prompt) { /* 为什么 = \xe4\xb8\xba\xe4\xbb\x80\xe4\xb9\x88 */ if (strstr(prompt, "\xe4\xb8\xba\xe4\xbb\x80\xe4\xb9\x88") || strstr(prompt, "\xe4\xb8\xba\xe4\xbd\x95") || strstr(prompt, "\xe5\x87\xad\xe4\xbb\x80\xe4\xb9\x88")) return Q_WHY; /* 什么是 = \xe4\xbb\x80\xe4\xb9\x88\xe6\x98\xaf, 是什么 = \xe6\x98\xaf\xe4\xbb\x80\xe4\xb9\x88 */ if (strstr(prompt, "\xe4\xbb\x80\xe4\xb9\x88\xe6\x98\xaf") || strstr(prompt, "\xe6\x98\xaf\xe4\xbb\x80\xe4\xb9\x88") || strstr(prompt, "\xe4\xbb\x80\xe4\xb9\x88\xe5\x8f\xab")) return Q_WHAT; /* 怎么 = \xe6\x80\x8e\xe4\xb9\x88, 如何 = \xe5\xa6\x82\xe4\xbd\x95 */ if (strstr(prompt, "\xe6\x80\x8e\xe4\xb9\x88") || strstr(prompt, "\xe5\xa6\x82\xe4\xbd\x95")) return Q_HOW; return Q_GENERAL; } /* CORE activation overlap between two concepts. * Computes cosine similarity of CORE-only neuron activation patterns at layer 0. * * High overlap -> concepts share semantic circuitry (parallel/category) * Low overlap -> concepts are distinct (causal/antonym) * * Also outputs CORE diff (mean absolute difference of CORE activations). */ static float core_activation_overlap(Model *m, const char *concept_a, const char *concept_b, float *out_core_diff) { int n_embd = m->cfg.n_embd; int mlp_dim = m->cfg.mlp_dim; float emb_a[4096], emb_b[4096]; float gate_a[4096], gate_b[4096]; float *act_a = (float *)malloc(mlp_dim * sizeof(float)); float *act_b = (float *)malloc(mlp_dim * sizeof(float)); get_concept_embedding(m, concept_a, emb_a, n_embd); get_concept_embedding(m, concept_b, emb_b, n_embd); compute_gate_input(m, emb_a, 0, gate_a, n_embd); compute_gate_input(m, emb_b, 0, gate_b, n_embd); simulate_activation(m, gate_a, 0, act_a, mlp_dim); simulate_activation(m, gate_b, 0, act_b, mlp_dim); uint8_t *mask = m->layers[0].mlp_gate.logic_mask; if (!mask) { free(act_a); free(act_b); if (out_core_diff) *out_core_diff = 0; return 0; } /* Extract CORE-only activations */ float *core_a = (float *)malloc(mlp_dim * sizeof(float)); float *core_b = (float *)malloc(mlp_dim * sizeof(float)); int n_core = 0; for (int j = 0; j < mlp_dim; j++) { if (mask[j] == 0) { /* CORE neuron */ core_a[n_core] = act_a[j]; core_b[n_core] = act_b[j]; n_core++; } } float overlap = 0, diff = 0; if (n_core > 0) { overlap = cosine_sim(core_a, core_b, n_core); float sum_diff = 0; for (int j = 0; j < n_core; j++) sum_diff += fabsf(core_a[j] - core_b[j]); diff = sum_diff / n_core; } free(act_a); free(act_b); free(core_a); free(core_b); if (out_core_diff) *out_core_diff = diff; return overlap; } /* ======================================================================== * NON-SYMMETRIC directional drive between two concepts. * * core_activation_overlap() above is symmetric (cosine of two CORE * patterns), so dir = ov(A,B) - ov(B,A) collapses to 0 and the model * can never tell "A drives B" from "B drives A" —主谓宾/因果方向无法浮现. * * To break symmetry we re-run the driver's forward with a small residual * bias injected along the *target's* embedding direction (beta≈0.3). The * resulting CORE activation pattern reflects "how much of driver's logic * circuit is pulled toward target". We then measure its cosine overlap * with target's own CORE pattern: * * drive(A->B) = cosine( CORE(A biased by B), CORE(B) ) * drive(B->A) = cosine( CORE(B biased by A), CORE(A) ) * * These are typically unequal: if A naturally activates circuitry that B * also uses (A is the "subject/agent"), drive(A->B) > drive(B->A) and * dir>0. No fixed relationship labels — just a continuous directional * signal that lets 主谓宾 / 因果 / 主动被动 emerge from the model itself. * ======================================================================== */ static float core_directional_drive(Model *m, const char *driver, const char *target, float beta, float *out_core_diff) { int n_embd = m->cfg.n_embd; int mlp_dim = m->cfg.mlp_dim; float emb_drv[4096], emb_tgt[4096]; float gate_drv[4096], gate_tgt[4096]; float *act_drv = (float *)malloc(mlp_dim * sizeof(float)); float *act_tgt = (float *)malloc(mlp_dim * sizeof(float)); get_concept_embedding(m, driver, emb_drv, n_embd); get_concept_embedding(m, target, emb_tgt, n_embd); /* Biased driver embedding: pull driver slightly toward target dir. * This perturbs the residual stream so the driver's downstream CORE * circuitry partially aligns with target's. */ for (int i = 0; i < n_embd; i++) emb_drv[i] += beta * emb_tgt[i]; compute_gate_input(m, emb_drv, 0, gate_drv, n_embd); compute_gate_input(m, emb_tgt, 0, gate_tgt, n_embd); simulate_activation(m, gate_drv, 0, act_drv, mlp_dim); simulate_activation(m, gate_tgt, 0, act_tgt, mlp_dim); uint8_t *mask = m->layers[0].mlp_gate.logic_mask; if (!mask) { free(act_drv); free(act_tgt); if (out_core_diff) *out_core_diff = 0; return 0; } float *core_drv = (float *)malloc(mlp_dim * sizeof(float)); float *core_tgt = (float *)malloc(mlp_dim * sizeof(float)); int n_core = 0; for (int j = 0; j < mlp_dim; j++) { if (mask[j] == 0) { /* CORE neuron */ core_drv[n_core] = act_drv[j]; core_tgt[n_core] = act_tgt[j]; n_core++; } } float overlap = 0, diff = 0; if (n_core > 0) { overlap = cosine_sim(core_drv, core_tgt, n_core); float sum_diff = 0; for (int j = 0; j < n_core; j++) sum_diff += fabsf(core_drv[j] - core_tgt[j]); diff = sum_diff / n_core; } free(act_drv); free(act_tgt); free(core_drv); free(core_tgt); if (out_core_diff) *out_core_diff = diff; return overlap; } /* Token-id 变体: 直接对 wte[ta]/wte[tb] 行算非对称驱动方向, 不依赖 UTF-8 * 概念名查找. 用于训练期对**任意随机概念对**施加方向收敛引导, 使所有关系 * (不只 bpe_token_map 里的锚点) 都能被模型在 CORE 回路层面结构化. * 逻辑与 core_directional_drive 完全一致, 只是入口从字符串改为 token id. */ static float core_directional_drive_id(Model *m, int ta, int tb, float beta, float *out_core_diff) { int n_embd = m->cfg.n_embd; int mlp_dim = m->cfg.mlp_dim; if (ta < 0 || tb < 0 || ta >= m->cfg.vocab_size || tb >= m->cfg.vocab_size) return 0; float *emb_a = &m->wte[(size_t)ta * n_embd]; float *emb_b = &m->wte[(size_t)tb * n_embd]; float drv[4096], tgt[4096]; for (int i = 0; i < n_embd; i++) { drv[i] = emb_a[i]; tgt[i] = emb_b[i]; } /* 偏置 driver(ta) 朝 target(tb) 方向, 打破对称 */ for (int i = 0; i < n_embd; i++) drv[i] += beta * tgt[i]; float gate_drv[4096], gate_tgt[4096]; /* P3: static thread-local 缓冲, 避免每调用 malloc/free (256×2×4 = 2048 次/step) */ static __thread float *act_drv = NULL, *act_tgt = NULL; static __thread float *core_drv = NULL, *core_tgt = NULL; static __thread int dd_mlp = 0; if (dd_mlp != mlp_dim) { free(act_drv); free(act_tgt); free(core_drv); free(core_tgt); act_drv = (float *)malloc(mlp_dim * sizeof(float)); act_tgt = (float *)malloc(mlp_dim * sizeof(float)); core_drv = (float *)malloc(mlp_dim * sizeof(float)); core_tgt = (float *)malloc(mlp_dim * sizeof(float)); dd_mlp = mlp_dim; } compute_gate_input(m, drv, 0, gate_drv, n_embd); compute_gate_input(m, tgt, 0, gate_tgt, n_embd); simulate_activation(m, gate_drv, 0, act_drv, mlp_dim); simulate_activation(m, gate_tgt, 0, act_tgt, mlp_dim); uint8_t *mask = m->layers[0].mlp_gate.logic_mask; if (!mask) { if (out_core_diff)*out_core_diff=0; return 0; } int n_core = 0; for (int j = 0; j < mlp_dim; j++) if (mask[j] == 0) { core_drv[n_core] = act_drv[j]; core_tgt[n_core] = act_tgt[j]; n_core++; } float overlap = 0, diff = 0; if (n_core > 0) { overlap = cosine_sim(core_drv, core_tgt, n_core); float sum = 0; for (int j = 0; j < n_core; j++) sum += fabsf(core_drv[j]-core_tgt[j]); diff = sum / n_core; } /* P3: static buffers 不需要 free */ if (out_core_diff) *out_core_diff = diff; return overlap; } /* ======================================================================== * 非对称驱动方向收敛引导 (directional_drive_regularization) * * 设计原则 (零关系知识): 不指定谁驱动谁, 不指定任何关系名, 不依赖写死概念对. * 仅对随机采样的概念对 (ta,tb) 施加结构性收敛目标, 让 CORE 回路层面的驱动 * 方向 (core_directional_drive) 在训练中稳定可归类: * * - 方向塌缩惩罚: |drive(ta->tb) - drive(tb->ta)| 太小时, 说明该对的驱动 * 方向无信息量(对称), 鼓励模型把概念对的驱动方向拉开 → 主谓宾/因果方向 * 才能从结构里浮现, 而非全部 dir≈0. * - 方向过饱和惩罚: 若某对 drive 同时过高(双向都≈1, 退化), 轻微回拉, * 避免所有对都变成强双向关联. * * 注意: 此项只在"驱动方向"这一维做几何引导; 关系的具体归类 (主谓宾/因果/ * 并列/对立) 完全由模型从大量 token 的 CE 损失中自行定位到 (wte, dir) 平面, * 此处不携带任何关系类型. 覆盖所有随机概念对 → 处理所有关系. * ======================================================================== */ static float directional_drive_regularization(Model *m, float lr) { int V = m->cfg.vocab_size; const int n_pairs = 256; const float dir_floor = 0.04f; const float sym_cap = 0.92f; float loss = 0.0f; int n_applied = 0; /* P4: 串行 (OpenMP atomic 在 Windows MSYS2 上崩溃, 回退为串行) */ for (int p2 = 0; p2 < n_pairs; p2++) { int ta = 3 + (rand() % (V - 3)); int tb = 3 + (rand() % (V - 3)); if (ta == tb) continue; float d_ab = core_directional_drive_id(m, ta, tb, 0.3f, NULL); float d_ba = core_directional_drive_id(m, tb, ta, 0.3f, NULL); float dir = d_ab - d_ba; if (fabsf(dir) < dir_floor) { float pen = (dir_floor - fabsf(dir)); int n_embd = m->cfg.n_embd; float *wa = &m->wte[(size_t)ta * n_embd]; float *wb = &m->wte[(size_t)tb * n_embd]; float na = 0, nb = 0, dot = 0; for (int i = 0; i < n_embd; i++) { na += wa[i]*wa[i]; nb += wb[i]*wb[i]; dot += wa[i]*wb[i]; } float denom = sqrtf(na)*sqrtf(nb) + 1e-6f; float cos = dot / denom; float scale = pen * lr; float inv = 1.0f / denom; for (int i = 0; i < n_embd; i++) { float ga = (wb[i] - cos*wa[i]) * inv; wa[i] -= scale * ga; } loss += pen; n_applied++; } if (d_ab > sym_cap && d_ba > sym_cap) { int n_embd = m->cfg.n_embd; float *wa = &m->wte[(size_t)ta * n_embd]; float norm_sq = 0; for (int i = 0; i < n_embd; i++) norm_sq += wa[i]*wa[i]; float norm = sqrtf(norm_sq) + 1e-6f; float scale = (1.0f - 0.05f * (d_ab + d_ba - 2*sym_cap) / 2.0f); if (scale < 0.95f) scale = 0.95f; for (int i = 0; i < n_embd; i++) { wa[i] *= scale; } } } return n_applied > 0 ? loss / n_applied : 0.0f; } /* classify_relationship / rel_type_matches_qtype / find_edge_type(_bidir) / * rel_explanation_cn 已全部移除: 不再把 (wte, core_ov, core_diff) 硬分类成 * REL_* 固定标签, 也不再按问句类型过滤固定关系. 关系完全由模型自身的连续 * 信号 (wte 余弦 + 非对称 core_directional_drive) 浮现. * * 若渲染器需要可读中文, 调用 describe_relation(wte, dir) —— 它只描述 * "语义强度/驱动方向", 不命名任何固定关系类别 (因果/主被动/串行/并行...). */ #endif /* LAL_WHITEBOX_PROBE_H */