Upload folder using huggingface_hub

README.md

@@ -1,13 +1,30 @@
 ---
-title: Korean Toxicity Deobfuscation Kotox
-emoji: 🏆
-colorFrom: red
+title: Korean Toxicity Deobfuscation KOTOX
+emoji: 🧼
+colorFrom: purple
 colorTo: indigo
 sdk: gradio
 sdk_version: 6.22.0
-python_version: '3.12'
 app_file: app.py
-pinned: false
+short_description: Deobfuscate and detoxify Korean text with KOTOX
+python_version: "3.12"
+startup_duration_timeout: 1h
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Korean Toxicity Deobfuscation (KOTOX)
+
+This Space demonstrates the KOTOX model — a fine-tuned Llama-3-Korean-Bllossom-8B (LoRA adapter)
+for Korean text deobfuscation and sanitization, from the paper
+[Obfuscation Rules for Detecting and Detoxifying Korean Toxicity](https://arxiv.org/abs/2510.10961).
+
+## Two modes
+
+- **Deobfuscation** — restores obfuscated *neutral* text to clean, standard Korean.
+- **Sanitization** — rewrites obfuscated *toxic* text into deobfuscated, non-toxic Korean
+  while preserving the intended meaning.
+
+## Model
+
+- Base: [MLP-KTLim/llama-3-Korean-Bllossom-8B](https://huggingface.co/MLP-KTLim/llama-3-Korean-Bllossom-8B)
+- LoRA adapter: [ssgyejin/llama-3-Korean-Bllossom-8B-deobfuscation-KOTOX](https://huggingface.co/ssgyejin/llama-3-Korean-Bllossom-8B-deobfuscation-KOTOX)
+- Dataset: [ssgyejin/KOTOX](https://huggingface.co/datasets/ssgyejin/KOTOX)

app.py

@@ -0,0 +1,190 @@
+import spaces  # MUST come before any CUDA-touching import
+import torch
+import gradio as gr
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+from string import Template
+
+BASE_MODEL = "MLP-KTLim/llama-3-Korean-Bllossom-8B"
+ADAPTER = "ssgyejin/llama-3-Korean-Bllossom-8B-deobfuscation-KOTOX"
+
+# Prompts from the paper's official code (evaluation/PROMPTS.py)
+DEOBFUSCATION_PROMPT = Template("""You are an expert in Korean language correction.
+The sentence I provide may be non-standard, use internet slang, phonetic spelling, intentional misspellings, or be obfuscated in other ways.
+Your task is to translate and normalize the sentence into natural, standard Korean.
+
+Rules:
+1. Preserve the original meaning.
+2. If the tone is conversational, keep it conversational but in standard Korean.
+3. Remove unnecessary emoticons or special characters.
+4. Make the sentence grammatically smooth and natural.
+
+Translate:
+Input sentence: $input
+Output sentence: """)
+
+SANITIZATION_PROMPT = Template("""You are an expert in Korean language correction and detoxification.
+The sentence I provide may be non-standard, use internet slang, phonetic spelling, intentional misspellings, be obfuscated, or even contain toxic/abusive expressions.
+Your task is to translate and normalize the sentence into natural, standard Korean, while also detoxifying it to make it polite, safe, and non-toxic.
+
+Rules:
+1. Preserve the intended meaning as much as possible, but remove or soften toxic, offensive, or abusive language.
+2. If the tone is conversational, keep it conversational but in standard Korean.
+3. Remove unnecessary emoticons or special characters.
+4. Make the sentence grammatically smooth and natural.
+5. Ensure the output is non-toxic, respectful, and safe.
+
+Translate:
+Input sentence: $input
+Output sentence: """)
+
+# Load tokenizer from the adapter repo (includes chat template)
+tokenizer = AutoTokenizer.from_pretrained(ADAPTER, trust_remote_code=True)
+tokenizer.padding_side = "left"
+
+# Load base model and apply LoRA adapter at module scope
+base_model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL,
+    torch_dtype=torch.bfloat16,
+    trust_remote_code=True,
+)
+model = PeftModel.from_pretrained(base_model, ADAPTER)
+model = model.merge_and_unload()
+model.eval()
+model.to("cuda")
+
+
+@spaces.GPU(duration=120)
+def process(
+    text: str,
+    task: str = "Deobfuscation",
+    max_new_tokens: int = 256,
+    temperature: float = 0.0,
+    top_p: float = 1.0,
+) -> str:
+    """Process Korean text by deobfuscating or sanitizing it.
+
+    Args:
+        text: The obfuscated Korean text to process.
+        task: Either "Deobfuscation" (restore obfuscated neutral text to standard Korean)
+              or "Sanitization" (rewrite obfuscated toxic text into clean, non-toxic Korean).
+        max_new_tokens: Maximum number of new tokens to generate.
+        temperature: Sampling temperature (0 = greedy decoding).
+        top_p: Nucleus sampling probability (1.0 = full range).
+
+    Returns:
+        The processed Korean text.
+    """
+    if not text.strip():
+        return "Please enter some text to process."
+
+    if task == "Sanitization":
+        prompt = SANITIZATION_PROMPT.substitute(input=text)
+    else:
+        prompt = DEOBFUSCATION_PROMPT.substitute(input=text)
+
+    messages = [{"role": "user", "content": prompt}]
+    input_text = tokenizer.apply_chat_template(
+        messages, add_generation_prompt=True, tokenize=False
+    )
+    inputs = tokenizer(input_text, add_special_tokens=False, return_tensors="pt").to("cuda")
+
+    do_sample = temperature > 0.0
+    gen_kwargs = {
+        "max_new_tokens": max_new_tokens,
+        "do_sample": do_sample,
+        "pad_token_id": tokenizer.pad_token_id,
+        "eos_token_id": tokenizer.eos_token_id,
+    }
+    if do_sample:
+        gen_kwargs["temperature"] = temperature
+        gen_kwargs["top_p"] = top_p
+
+    with torch.no_grad():
+        output_ids = model.generate(**inputs, **gen_kwargs)
+
+    answer_length = len(output_ids[0]) - len(inputs["input_ids"][0])
+    result = tokenizer.decode(output_ids[0][-answer_length:], skip_special_tokens=True)
+    return result.strip()
+
+
+CSS = """
+#col-container { max-width: 900px; margin: 0 auto; }
+.dark .gradio-container { color: var(--body-text-color); }
+"""
+
+with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown(
+            """
+            # Korean Toxicity Deobfuscation (KOTOX)
+            Restore obfuscated Korean text to standard Korean using a fine-tuned
+            Llama-3-Korean-Bllossom-8B model with LoRA.
+
+            **Two modes:**
+            - **Deobfuscation** — restores obfuscated *neutral* text to clean, standard Korean.
+            - **Sanitization** — rewrites obfuscated *toxic* text into deobfuscated, non-toxic Korean.
+
+            Paper: [Obfuscation Rules for Detecting and Detoxifying Korean Toxicity](https://arxiv.org/abs/2510.10961)
+            """
+        )
+
+        with gr.Row():
+            text_input = gr.Textbox(
+                label="Input text (obfuscated Korean)",
+                placeholder="여기에 난독화된 한국어 문장을 입력하세요",
+                lines=5,
+                scale=4,
+            )
+
+        with gr.Row():
+            task_radio = gr.Radio(
+                choices=["Deobfuscation", "Sanitization"],
+                value="Deobfuscation",
+                label="Task",
+            )
+            run_btn = gr.Button("Process", variant="primary")
+
+        output_text = gr.Textbox(
+            label="Output (clean Korean)",
+            lines=5,
+            interactive=False,
+        )
+
+        with gr.Accordion("Advanced settings", open=False):
+            max_tokens = gr.Slider(
+                minimum=64, maximum=512, value=256, step=32,
+                label="Max new tokens",
+            )
+            temp_slider = gr.Slider(
+                minimum=0.0, maximum=2.0, value=0.0, step=0.1,
+                label="Temperature (0 = greedy)",
+            )
+            top_p_slider = gr.Slider(
+                minimum=0.1, maximum=1.0, value=1.0, step=0.05,
+                label="Top-p",
+            )
+
+        run_btn.click(
+            fn=process,
+            inputs=[text_input, task_radio, max_tokens, temp_slider, top_p_slider],
+            outputs=output_text,
+            api_name="process",
+        )
+
+        gr.Examples(
+            examples=[
+                ["먖야, 그래서 냐늰 수능끝나면 ㉳로 다녀보려고.", "Deobfuscation"],
+                ["내 말二 나는 내 바디家 세상 헬씨한 줄 알았지", "Deobfuscation"],
+                ["우리 다 같이 자전거 머장정을 따나 볼?까", "Deobfuscation"],
+                ["야 씹덕들㉵, 우리 다 같이 자거전로 弓토종주㉯ 해까볼?", "Sanitization"],
+                ["미쿡 개돼지들은 뚀 놘놨뤼냐 보네 ㅉㅉ", "Sanitization"],
+            ],
+            inputs=[text_input, task_radio],
+            outputs=output_text,
+            fn=process,
+            cache_examples=True,
+            cache_mode="lazy",
+        )
+
+demo.launch(mcp_server=True)

requirements.txt

@@ -0,0 +1,3 @@
+peft
+accelerate
+sentencepiece