Publish artifacts for clinical-rag-safety-gateway-20260815

README.md

@@ -0,0 +1,61 @@
+---
+license: mit
+library_name: custom
+pipeline_tag: question-answering
+datasets:
+- RKB109/clinical-rag-safety-gateway-20260815-dataset
+tags:
+- synthetic-data
+- transparent-baseline
+- healthcare-ai
+- question-answering
+- sentence-similarity
+- text-classification
+- summarization
+metrics:
+- accuracy
+---
+
+# Clinical RAG Safety Gateway Baseline Model
+
+## Model Description
+
+This repository contains a small, transparent prototype model for
+**Clinical assistants need retrieval, source attribution, and explicit abstention before answers reach care teams.**
+
+The model combines per-label token weights with IDF-weighted evidence
+retrieval. It was generated for reproducible architecture demonstrations and
+does not call a hosted LLM.
+
+## Evaluation
+
+- Held-out synthetic examples: 4
+- Accuracy: 1
+- Intended metrics: retrieval_accuracy, abstention_coverage, citation_coverage
+
+## Intended Use
+
+- Architecture prototyping
+- CI and evaluation examples
+- Local baseline comparisons
+- Educational experimentation
+
+## Hugging Face Task Coverage
+
+- `question-answering`
+- `sentence-similarity`
+- `text-classification`
+- `summarization`
+
+## Limitations and Risks
+
+Synthetic educational data only. The baseline must not provide diagnosis, treatment, or emergency medical advice.
+
+The dataset is synthetic and small. Do not use this model for consequential
+decisions without representative data, expert review, and production-grade
+evaluation.
+
+## Reproducibility
+
+The linked GitHub repository includes `train.py`, the exact dataset split,
+evaluation code, and the model JSON format.

evaluation.json

@@ -0,0 +1,5 @@
+{
+  "test_examples": 4,
+  "accuracy": 1,
+  "synthetic_evaluation": true
+}

inference.py

@@ -0,0 +1,80 @@
+"""Transparent baseline pipeline for the generated AI project."""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from pathlib import Path
+
+
+def tokenize(value: str) -> list[str]:
+    return re.findall(r"[a-z0-9]+", value.lower())
+
+
+class Pipeline:
+    def __init__(self, model: dict):
+        self.model = model
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> "Pipeline":
+        return cls(json.loads(Path(path).read_text(encoding="utf-8")))
+
+    def classify(self, text: str) -> tuple[str, float]:
+        tokens = tokenize(text)
+        scores = {
+            label: sum(weights.get(token, 0) for token in tokens)
+            for label, weights in self.model["prototypes"].items()
+        }
+        ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
+        label, best = ranked[0]
+        total = sum(max(score, 0) for _, score in ranked) or 1
+        return label, best / total
+
+    def search(self, query: str, limit: int = 3) -> list[dict]:
+        query_tokens = set(tokenize(query))
+        ranked = []
+        for document in self.model["documents"]:
+            document_tokens = set(tokenize(document["text"]))
+            lexical = sum(
+                self.model["idf"].get(token, 1.0)
+                for token in query_tokens & document_tokens
+            )
+            ranked.append({**document, "score": round(lexical, 6)})
+        return sorted(ranked, key=lambda item: (-item["score"], item["id"]))[:limit]
+
+    def graph_evidence(self, text: str) -> list[dict]:
+        tokens = set(tokenize(text))
+        matches = []
+        for subject, relation, target in self.model.get("graph_edges", []):
+            edge_tokens = set(tokenize(f"{subject} {relation} {target}"))
+            overlap = len(tokens & edge_tokens)
+            if overlap:
+                matches.append(
+                    {
+                        "subject": subject,
+                        "relation": relation,
+                        "target": target,
+                        "overlap": overlap,
+                    }
+                )
+        return sorted(matches, key=lambda item: -item["overlap"])
+
+    def run(self, text: str) -> dict:
+        label, confidence = self.classify(text)
+        evidence = self.search(text)
+        result = {
+            "prediction": label,
+            "confidence": round(confidence, 4),
+            "requires_review": confidence < self.model["confidence_threshold"],
+            "evidence": evidence,
+        }
+        if self.model["mode"] == "graph":
+            result["graph_evidence"] = self.graph_evidence(text)
+        if self.model["mode"] == "agent":
+            result["proposed_tool"] = label
+            result["approval_required"] = label in {
+                "request-approval",
+                "request-human-help",
+            }
+        return result

model.json

@@ -0,0 +1,239 @@
+{
+  "format": "daily-project-prototype-v1",
+  "project": "clinical-rag-safety",
+  "title": "Clinical RAG Safety Gateway",
+  "domain": "healthcare-ai",
+  "mode": "retrieval",
+  "labels": [
+    "medication-policy",
+    "triage-policy",
+    "privacy-policy"
+  ],
+  "prototypes": {
+    "medication-policy": {
+      "when": 3,
+      "should": 3,
+      "medication": 8,
+      "reconciliation": 6,
+      "be": 3,
+      "completed": 3,
+      "complete": 3,
+      "during": 3,
+      "admission": 3,
+      "transfer": 3,
+      "and": 3,
+      "discharge": 3,
+      "in": 2,
+      "an": 4,
+      "operations": 2,
+      "review": 6,
+      "for": 2,
+      "evaluation": 2,
+      "case": 2,
+      "what": 2,
+      "requires": 2,
+      "pharmacist": 4,
+      "high": 2,
+      "risk": 2,
+      "changes": 2,
+      "require": 2,
+      "before": 2,
+      "release": 2
+    },
+    "triage-policy": {
+      "when": 2,
+      "should": 2,
+      "urgent": 4,
+      "symptoms": 4,
+      "be": 2,
+      "escalated": 2,
+      "or": 2,
+      "life": 2,
+      "threatening": 2,
+      "require": 2,
+      "immediate": 2,
+      "escalation": 2,
+      "to": 4,
+      "qualified": 4,
+      "clinical": 2,
+      "staff": 2,
+      "for": 1,
+      "an": 2,
+      "evaluation": 1,
+      "case": 1,
+      "can": 2,
+      "the": 6,
+      "assistant": 4,
+      "diagnose": 2,
+      "a": 4,
+      "patient": 2,
+      "must": 2,
+      "abstain": 2,
+      "from": 2,
+      "diagnosis": 2,
+      "and": 2,
+      "direct": 2,
+      "user": 2,
+      "clinician": 2,
+      "in": 1,
+      "operations": 1,
+      "review": 1
+    },
+    "privacy-policy": {
+      "how": 3,
+      "should": 3,
+      "patient": 6,
+      "identifiers": 6,
+      "be": 5,
+      "handled": 3,
+      "use": 5,
+      "the": 3,
+      "minimum": 3,
+      "necessary": 3,
+      "information": 3,
+      "and": 5,
+      "never": 3,
+      "place": 3,
+      "in": 7,
+      "public": 3,
+      "logs": 3,
+      "an": 4,
+      "operations": 2,
+      "review": 2,
+      "for": 4,
+      "evaluation": 4,
+      "case": 2,
+      "can": 2,
+      "protected": 2,
+      "health": 2,
+      "data": 2,
+      "used": 2,
+      "test": 2,
+      "prompts": 2,
+      "synthetic": 2,
+      "or": 2,
+      "properly": 2,
+      "de": 2,
+      "identified": 2,
+      "records": 2,
+      "testing": 2
+    }
+  },
+  "idf": {
+    "complete": 2.252763,
+    "medication": 1.847298,
+    "reconciliation": 2.252763,
+    "during": 2.252763,
+    "admission": 2.252763,
+    "transfer": 2.252763,
+    "and": 1.336472,
+    "discharge": 2.252763,
+    "high": 2.252763,
+    "risk": 2.252763,
+    "changes": 2.252763,
+    "require": 1.847298,
+    "pharmacist": 2.252763,
+    "review": 2.252763,
+    "before": 2.252763,
+    "release": 2.252763,
+    "urgent": 2.252763,
+    "or": 1.847298,
+    "life": 2.252763,
+    "threatening": 2.252763,
+    "symptoms": 2.252763,
+    "immediate": 2.252763,
+    "escalation": 2.252763,
+    "to": 1.847298,
+    "qualified": 1.847298,
+    "clinical": 2.252763,
+    "staff": 2.252763,
+    "the": 1.847298,
+    "assistant": 2.252763,
+    "must": 2.252763,
+    "abstain": 2.252763,
+    "from": 2.252763,
+    "diagnosis": 2.252763,
+    "direct": 2.252763,
+    "user": 2.252763,
+    "a": 2.252763,
+    "clinician": 2.252763,
+    "use": 1.847298,
+    "minimum": 2.252763,
+    "necessary": 2.252763,
+    "patient": 2.252763,
+    "information": 2.252763,
+    "never": 2.252763,
+    "place": 2.252763,
+    "identifiers": 2.252763,
+    "in": 2.252763,
+    "public": 2.252763,
+    "logs": 2.252763,
+    "synthetic": 2.252763,
+    "properly": 2.252763,
+    "de": 2.252763,
+    "identified": 2.252763,
+    "records": 2.252763,
+    "for": 2.252763,
+    "testing": 2.252763,
+    "evaluation": 2.252763
+  },
+  "documents": [
+    {
+      "id": "med-policy-01",
+      "label": "medication-policy",
+      "text": "Complete medication reconciliation during admission, transfer, and discharge.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "healthcare-ai"
+      }
+    },
+    {
+      "id": "med-policy-02",
+      "label": "medication-policy",
+      "text": "High-risk medication changes require pharmacist review before release.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "healthcare-ai"
+      }
+    },
+    {
+      "id": "triage-01",
+      "label": "triage-policy",
+      "text": "Urgent or life-threatening symptoms require immediate escalation to qualified clinical staff.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "healthcare-ai"
+      }
+    },
+    {
+      "id": "triage-02",
+      "label": "triage-policy",
+      "text": "The assistant must abstain from diagnosis and direct the user to a qualified clinician.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "healthcare-ai"
+      }
+    },
+    {
+      "id": "privacy-01",
+      "label": "privacy-policy",
+      "text": "Use the minimum necessary patient information and never place identifiers in public logs.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "healthcare-ai"
+      }
+    },
+    {
+      "id": "privacy-02",
+      "label": "privacy-policy",
+      "text": "Use synthetic or properly de-identified records for testing and evaluation.",
+      "metadata": {
+        "synthetic": true,
+        "domain": "healthcare-ai"
+      }
+    }
+  ],
+  "graph_edges": [],
+  "confidence_threshold": 0.18,
+  "trained_on_synthetic_data": true
+}

project.json

@@ -0,0 +1,52 @@
+{
+  "name": "Clinical RAG Safety Gateway",
+  "problem": "Clinical assistants need retrieval, source attribution, and explicit abstention before answers reach care teams.",
+  "domain": "healthcare-ai",
+  "architecture": "retrieval",
+  "hugging_face_tasks": [
+    "question-answering",
+    "sentence-similarity",
+    "text-classification",
+    "summarization"
+  ],
+  "recommended_stack": [
+    "FastAPI for typed service endpoints",
+    "LangGraph for durable safety and escalation workflows",
+    "LlamaIndex for ingestion and retrieval abstractions",
+    "PostgreSQL plus pgvector for filtered vector search",
+    "Redis for caching and rate limiting",
+    "OpenTelemetry plus Phoenix for traces and evaluation"
+  ],
+  "real_world_data_sources": [
+    {
+      "name": "ClinicalTrials.gov API v2",
+      "url": "https://clinicaltrials.gov/api/v2/studies?pageSize=5",
+      "purpose": "Public study metadata for retrieval and citation tests"
+    },
+    {
+      "name": "openFDA drug labels",
+      "url": "https://api.fda.gov/drug/label.json?limit=5",
+      "purpose": "Public drug-label text for safety-oriented ingestion"
+    }
+  ],
+  "job_description_skills": [
+    "Production RAG design and retrieval evaluation",
+    "Healthcare AI safety, abstention, and auditability",
+    "Vector databases and metadata-aware retrieval",
+    "Agent orchestration with human review",
+    "LLMOps tracing, regression tests, and release gates"
+  ],
+  "impact_targets": [
+    "Reach Recall@5 >= 0.85 on a reviewed retrieval set",
+    "Maintain 100% citation coverage for non-abstained answers",
+    "Block 100% of diagnosis or emergency-advice test prompts",
+    "Keep p95 retrieval latency below 300 ms at 50 requests/second"
+  ],
+  "baseline_evaluation": {
+    "test_examples": 4,
+    "accuracy": 1,
+    "synthetic_evaluation": true
+  },
+  "estimated_delivery": "8-12 weeks for one engineer",
+  "generated_baseline_is_production_ready": false
+}