Instructions to use lflare/Qwen3.8-27B-FP8 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use lflare/Qwen3.8-27B-FP8 with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("image-text-to-text", model="lflare/Qwen3.8-27B-FP8")
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},
            {"type": "text", "text": "What animal is on the candy?"}
        ]
    },
]
pipe(text=messages)

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoProcessor, AutoModelForMultimodalLM

processor = AutoProcessor.from_pretrained("lflare/Qwen3.8-27B-FP8")
model = AutoModelForMultimodalLM.from_pretrained("lflare/Qwen3.8-27B-FP8", device_map="auto")
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},
            {"type": "text", "text": "What animal is on the candy?"}
        ]
    },
]
inputs = processor.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=256)
print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use lflare/Qwen3.8-27B-FP8 with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "lflare/Qwen3.8-27B-FP8"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "lflare/Qwen3.8-27B-FP8",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Use Docker

docker model run hf.co/lflare/Qwen3.8-27B-FP8

SGLang

How to use lflare/Qwen3.8-27B-FP8 with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "lflare/Qwen3.8-27B-FP8" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "lflare/Qwen3.8-27B-FP8",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "lflare/Qwen3.8-27B-FP8" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "lflare/Qwen3.8-27B-FP8",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Docker Model Runner
How to use lflare/Qwen3.8-27B-FP8 with Docker Model Runner:
```
docker model run hf.co/lflare/Qwen3.8-27B-FP8
```

Qwen3.8-27B-FP8 / block_scale_experiment.json

Huggin Fork

Upload block_scale_experiment.json with huggingface_hub

a6d0a5e verified 3 months ago

Raw History Blame Contribute Delete

7.65 kB

{
  "model.language_model.layers.20.mlp.down_proj.weight": {
    "per-tensor": {
      "rel_frob_err": 0.026479586958885193,
      "max_abs_err": 0.006138399243354797,
      "num_scales": 1
    },
    "per-out-channel (ours)": {
      "rel_frob_err": 0.02648800238966942,
      "max_abs_err": 0.006138399243354797,
      "num_scales": 5120
    },
    "per-in-channel": {
      "rel_frob_err": 0.02645968273282051,
      "max_abs_err": 0.005161836743354797,
      "num_scales": 17408
    },
    "block-128x128 (vendor)": {
      "rel_frob_err": 0.02649129182100296,
      "max_abs_err": 0.005161836743354797,
      "num_scales": 5440
    },
    "[ctrl] int8 per-tensor": {
      "rel_frob_err": 0.07149230688810349,
      "max_abs_err": 0.0013607963919639587,
      "num_scales": 1
    },
    "[ctrl] int8 per-out-chan": {
      "rel_frob_err": 0.010320284403860569,
      "max_abs_err": 0.0013607963919639587,
      "num_scales": 5120
    },
    "[ctrl] int8 block-128": {
      "rel_frob_err": 0.010944717563688755,
      "max_abs_err": 0.0013607963919639587,
      "num_scales": 5440
    }
  },
  "model.language_model.layers.20.mlp.gate_proj.weight": {
    "per-tensor": {
      "rel_frob_err": 0.0265126321464777,
      "max_abs_err": 0.008370548486709595,
      "num_scales": 1
    },
    "per-out-channel (ours)": {
      "rel_frob_err": 0.026488669216632843,
      "max_abs_err": 0.008370548486709595,
      "num_scales": 17408
    },
    "per-in-channel": {
      "rel_frob_err": 0.026507357135415077,
      "max_abs_err": 0.004324778914451599,
      "num_scales": 5120
    },
    "block-128x128 (vendor)": {
      "rel_frob_err": 0.026498381048440933,
      "max_abs_err": 0.005161836743354797,
      "num_scales": 5440
    },
    "[ctrl] int8 per-tensor": {
      "rel_frob_err": 0.05153953656554222,
      "max_abs_err": 0.00101470947265625,
      "num_scales": 1
    },
    "[ctrl] int8 per-out-chan": {
      "rel_frob_err": 0.0090852165594697,
      "max_abs_err": 0.0010130871087312698,
      "num_scales": 17408
    },
    "[ctrl] int8 block-128": {
      "rel_frob_err": 0.0108567476272583,
      "max_abs_err": 0.00101470947265625,
      "num_scales": 5440
    }
  },
  "model.language_model.layers.20.mlp.up_proj.weight": {
    "per-tensor": {
      "rel_frob_err": 0.02650594525039196,
      "max_abs_err": 0.0046037957072257996,
      "num_scales": 1
    },
    "per-out-channel (ours)": {
      "rel_frob_err": 0.02646675519645214,
      "max_abs_err": 0.0046037957072257996,
      "num_scales": 17408
    },
    "per-in-channel": {
      "rel_frob_err": 0.026487573981285095,
      "max_abs_err": 0.0035574808716773987,
      "num_scales": 5120
    },
    "block-128x128 (vendor)": {
      "rel_frob_err": 0.02647939883172512,
      "max_abs_err": 0.004045754671096802,
      "num_scales": 5440
    },
    "[ctrl] int8 per-tensor": {
      "rel_frob_err": 0.0277273990213871,
      "max_abs_err": 0.000526726245880127,
      "num_scales": 1
    },
    "[ctrl] int8 per-out-chan": {
      "rel_frob_err": 0.009053499437868595,
      "max_abs_err": 0.000526726245880127,
      "num_scales": 17408
    },
    "[ctrl] int8 block-128": {
      "rel_frob_err": 0.010055948048830032,
      "max_abs_err": 0.00052642822265625,
      "num_scales": 5440
    }
  },
  "model.language_model.layers.60.mlp.down_proj.weight": {
    "per-tensor": {
      "rel_frob_err": 0.026482094079256058,
      "max_abs_err": 0.020647317171096802,
      "num_scales": 1
    },
    "per-out-channel (ours)": {
      "rel_frob_err": 0.02642776072025299,
      "max_abs_err": 0.020647317171096802,
      "num_scales": 5120
    },
    "per-in-channel": {
      "rel_frob_err": 0.026440486311912537,
      "max_abs_err": 0.0107421875,
      "num_scales": 17408
    },
    "block-128x128 (vendor)": {
      "rel_frob_err": 0.026434293016791344,
      "max_abs_err": 0.004255019128322601,
      "num_scales": 5440
    },
    "[ctrl] int8 per-tensor": {
      "rel_frob_err": 0.2161778062582016,
      "max_abs_err": 0.004610799252986908,
      "num_scales": 1
    },
    "[ctrl] int8 per-out-chan": {
      "rel_frob_err": 0.011156928725540638,
      "max_abs_err": 0.004610799252986908,
      "num_scales": 5120
    },
    "[ctrl] int8 block-128": {
      "rel_frob_err": 0.012093556113541126,
      "max_abs_err": 0.004610799252986908,
      "num_scales": 5440
    }
  },
  "model.language_model.layers.60.mlp.gate_proj.weight": {
    "per-tensor": {
      "rel_frob_err": 0.02639029547572136,
      "max_abs_err": 0.015066951513290405,
      "num_scales": 1
    },
    "per-out-channel (ours)": {
      "rel_frob_err": 0.026385989040136337,
      "max_abs_err": 0.012974321842193604,
      "num_scales": 17408
    },
    "per-in-channel": {
      "rel_frob_err": 0.02640954963862896,
      "max_abs_err": 0.013392865657806396,
      "num_scales": 5120
    },
    "block-128x128 (vendor)": {
      "rel_frob_err": 0.02639611065387726,
      "max_abs_err": 0.0066266655921936035,
      "num_scales": 5440
    },
    "[ctrl] int8 per-tensor": {
      "rel_frob_err": 0.07502172142267227,
      "max_abs_err": 0.0016743820160627365,
      "num_scales": 1
    },
    "[ctrl] int8 per-out-chan": {
      "rel_frob_err": 0.010096607729792595,
      "max_abs_err": 0.0016743820160627365,
      "num_scales": 17408
    },
    "[ctrl] int8 block-128": {
      "rel_frob_err": 0.014383016154170036,
      "max_abs_err": 0.0016743820160627365,
      "num_scales": 5440
    }
  },
  "model.language_model.layers.60.mlp.up_proj.weight": {
    "per-tensor": {
      "rel_frob_err": 0.026437141001224518,
      "max_abs_err": 0.009905129671096802,
      "num_scales": 1
    },
    "per-out-channel (ours)": {
      "rel_frob_err": 0.026422692462801933,
      "max_abs_err": 0.0078125,
      "num_scales": 17408
    },
    "per-in-channel": {
      "rel_frob_err": 0.026427578181028366,
      "max_abs_err": 0.009068071842193604,
      "num_scales": 5120
    },
    "block-128x128 (vendor)": {
      "rel_frob_err": 0.02642085589468479,
      "max_abs_err": 0.004813060164451599,
      "num_scales": 5440
    },
    "[ctrl] int8 per-tensor": {
      "rel_frob_err": 0.05160678178071976,
      "max_abs_err": 0.0011140108108520508,
      "num_scales": 1
    },
    "[ctrl] int8 per-out-chan": {
      "rel_frob_err": 0.009319894947111607,
      "max_abs_err": 0.0011140108108520508,
      "num_scales": 17408
    },
    "[ctrl] int8 block-128": {
      "rel_frob_err": 0.011335344985127449,
      "max_abs_err": 0.0011140108108520508,
      "num_scales": 5440
    }
  },
  "model.language_model.layers.3.self_attn.q_proj.weight": {
    "per-tensor": {
      "rel_frob_err": 0.026460612192749977,
      "max_abs_err": 0.011160701513290405,
      "num_scales": 1
    },
    "per-out-channel (ours)": {
      "rel_frob_err": 0.026407163590192795,
      "max_abs_err": 0.008649557828903198,
      "num_scales": 12288
    },
    "per-in-channel": {
      "rel_frob_err": 0.02644822932779789,
      "max_abs_err": 0.009486615657806396,
      "num_scales": 5120
    },
    "block-128x128 (vendor)": {
      "rel_frob_err": 0.026424383744597435,
      "max_abs_err": 0.009626120328903198,
      "num_scales": 3840
    },
    "[ctrl] int8 per-tensor": {
      "rel_frob_err": 0.04228312894701958,
      "max_abs_err": 0.0012610703706741333,
      "num_scales": 1
    },
    "[ctrl] int8 per-out-chan": {
      "rel_frob_err": 0.010167714208364487,
      "max_abs_err": 0.0012601115740835667,
      "num_scales": 12288
    },
    "[ctrl] int8 block-128": {
      "rel_frob_err": 0.012071715667843819,
      "max_abs_err": 0.0012601115740835667,
      "num_scales": 3840
    }
  }
}

	{
	"model.language_model.layers.20.mlp.down_proj.weight": {
	"per-tensor": {
	"rel_frob_err": 0.026479586958885193,
	"max_abs_err": 0.006138399243354797,
	"num_scales": 1
	},
	"per-out-channel (ours)": {
	"rel_frob_err": 0.02648800238966942,
	"max_abs_err": 0.006138399243354797,
	"num_scales": 5120
	},
	"per-in-channel": {
	"rel_frob_err": 0.02645968273282051,
	"max_abs_err": 0.005161836743354797,
	"num_scales": 17408
	},
	"block-128x128 (vendor)": {
	"rel_frob_err": 0.02649129182100296,
	"max_abs_err": 0.005161836743354797,
	"num_scales": 5440
	},
	"[ctrl] int8 per-tensor": {
	"rel_frob_err": 0.07149230688810349,
	"max_abs_err": 0.0013607963919639587,
	"num_scales": 1
	},
	"[ctrl] int8 per-out-chan": {
	"rel_frob_err": 0.010320284403860569,
	"max_abs_err": 0.0013607963919639587,
	"num_scales": 5120
	},
	"[ctrl] int8 block-128": {
	"rel_frob_err": 0.010944717563688755,
	"max_abs_err": 0.0013607963919639587,
	"num_scales": 5440
	}
	},
	"model.language_model.layers.20.mlp.gate_proj.weight": {
	"per-tensor": {
	"rel_frob_err": 0.0265126321464777,
	"max_abs_err": 0.008370548486709595,
	"num_scales": 1
	},
	"per-out-channel (ours)": {
	"rel_frob_err": 0.026488669216632843,
	"max_abs_err": 0.008370548486709595,
	"num_scales": 17408
	},
	"per-in-channel": {
	"rel_frob_err": 0.026507357135415077,
	"max_abs_err": 0.004324778914451599,
	"num_scales": 5120
	},
	"block-128x128 (vendor)": {
	"rel_frob_err": 0.026498381048440933,
	"max_abs_err": 0.005161836743354797,
	"num_scales": 5440
	},
	"[ctrl] int8 per-tensor": {
	"rel_frob_err": 0.05153953656554222,
	"max_abs_err": 0.00101470947265625,
	"num_scales": 1
	},
	"[ctrl] int8 per-out-chan": {
	"rel_frob_err": 0.0090852165594697,
	"max_abs_err": 0.0010130871087312698,
	"num_scales": 17408
	},
	"[ctrl] int8 block-128": {
	"rel_frob_err": 0.0108567476272583,
	"max_abs_err": 0.00101470947265625,
	"num_scales": 5440
	}
	},
	"model.language_model.layers.20.mlp.up_proj.weight": {
	"per-tensor": {
	"rel_frob_err": 0.02650594525039196,
	"max_abs_err": 0.0046037957072257996,
	"num_scales": 1
	},
	"per-out-channel (ours)": {
	"rel_frob_err": 0.02646675519645214,
	"max_abs_err": 0.0046037957072257996,
	"num_scales": 17408
	},
	"per-in-channel": {
	"rel_frob_err": 0.026487573981285095,
	"max_abs_err": 0.0035574808716773987,
	"num_scales": 5120
	},
	"block-128x128 (vendor)": {
	"rel_frob_err": 0.02647939883172512,
	"max_abs_err": 0.004045754671096802,
	"num_scales": 5440
	},
	"[ctrl] int8 per-tensor": {
	"rel_frob_err": 0.0277273990213871,
	"max_abs_err": 0.000526726245880127,
	"num_scales": 1
	},
	"[ctrl] int8 per-out-chan": {
	"rel_frob_err": 0.009053499437868595,
	"max_abs_err": 0.000526726245880127,
	"num_scales": 17408
	},
	"[ctrl] int8 block-128": {
	"rel_frob_err": 0.010055948048830032,
	"max_abs_err": 0.00052642822265625,
	"num_scales": 5440
	}
	},
	"model.language_model.layers.60.mlp.down_proj.weight": {
	"per-tensor": {
	"rel_frob_err": 0.026482094079256058,
	"max_abs_err": 0.020647317171096802,
	"num_scales": 1
	},
	"per-out-channel (ours)": {
	"rel_frob_err": 0.02642776072025299,
	"max_abs_err": 0.020647317171096802,
	"num_scales": 5120
	},
	"per-in-channel": {
	"rel_frob_err": 0.026440486311912537,
	"max_abs_err": 0.0107421875,
	"num_scales": 17408
	},
	"block-128x128 (vendor)": {
	"rel_frob_err": 0.026434293016791344,
	"max_abs_err": 0.004255019128322601,
	"num_scales": 5440
	},
	"[ctrl] int8 per-tensor": {
	"rel_frob_err": 0.2161778062582016,
	"max_abs_err": 0.004610799252986908,
	"num_scales": 1
	},
	"[ctrl] int8 per-out-chan": {
	"rel_frob_err": 0.011156928725540638,
	"max_abs_err": 0.004610799252986908,
	"num_scales": 5120
	},
	"[ctrl] int8 block-128": {
	"rel_frob_err": 0.012093556113541126,
	"max_abs_err": 0.004610799252986908,
	"num_scales": 5440
	}
	},
	"model.language_model.layers.60.mlp.gate_proj.weight": {
	"per-tensor": {
	"rel_frob_err": 0.02639029547572136,
	"max_abs_err": 0.015066951513290405,
	"num_scales": 1
	},
	"per-out-channel (ours)": {
	"rel_frob_err": 0.026385989040136337,
	"max_abs_err": 0.012974321842193604,
	"num_scales": 17408
	},
	"per-in-channel": {
	"rel_frob_err": 0.02640954963862896,
	"max_abs_err": 0.013392865657806396,
	"num_scales": 5120
	},
	"block-128x128 (vendor)": {
	"rel_frob_err": 0.02639611065387726,
	"max_abs_err": 0.0066266655921936035,
	"num_scales": 5440
	},
	"[ctrl] int8 per-tensor": {
	"rel_frob_err": 0.07502172142267227,
	"max_abs_err": 0.0016743820160627365,
	"num_scales": 1
	},
	"[ctrl] int8 per-out-chan": {
	"rel_frob_err": 0.010096607729792595,
	"max_abs_err": 0.0016743820160627365,
	"num_scales": 17408
	},
	"[ctrl] int8 block-128": {
	"rel_frob_err": 0.014383016154170036,
	"max_abs_err": 0.0016743820160627365,
	"num_scales": 5440
	}
	},
	"model.language_model.layers.60.mlp.up_proj.weight": {
	"per-tensor": {
	"rel_frob_err": 0.026437141001224518,
	"max_abs_err": 0.009905129671096802,
	"num_scales": 1
	},
	"per-out-channel (ours)": {
	"rel_frob_err": 0.026422692462801933,
	"max_abs_err": 0.0078125,
	"num_scales": 17408
	},
	"per-in-channel": {
	"rel_frob_err": 0.026427578181028366,
	"max_abs_err": 0.009068071842193604,
	"num_scales": 5120
	},
	"block-128x128 (vendor)": {
	"rel_frob_err": 0.02642085589468479,
	"max_abs_err": 0.004813060164451599,
	"num_scales": 5440
	},
	"[ctrl] int8 per-tensor": {
	"rel_frob_err": 0.05160678178071976,
	"max_abs_err": 0.0011140108108520508,
	"num_scales": 1
	},
	"[ctrl] int8 per-out-chan": {
	"rel_frob_err": 0.009319894947111607,
	"max_abs_err": 0.0011140108108520508,
	"num_scales": 17408
	},
	"[ctrl] int8 block-128": {
	"rel_frob_err": 0.011335344985127449,
	"max_abs_err": 0.0011140108108520508,
	"num_scales": 5440
	}
	},
	"model.language_model.layers.3.self_attn.q_proj.weight": {
	"per-tensor": {
	"rel_frob_err": 0.026460612192749977,
	"max_abs_err": 0.011160701513290405,
	"num_scales": 1
	},
	"per-out-channel (ours)": {
	"rel_frob_err": 0.026407163590192795,
	"max_abs_err": 0.008649557828903198,
	"num_scales": 12288
	},
	"per-in-channel": {
	"rel_frob_err": 0.02644822932779789,
	"max_abs_err": 0.009486615657806396,
	"num_scales": 5120
	},
	"block-128x128 (vendor)": {
	"rel_frob_err": 0.026424383744597435,
	"max_abs_err": 0.009626120328903198,
	"num_scales": 3840
	},
	"[ctrl] int8 per-tensor": {
	"rel_frob_err": 0.04228312894701958,
	"max_abs_err": 0.0012610703706741333,
	"num_scales": 1
	},
	"[ctrl] int8 per-out-chan": {
	"rel_frob_err": 0.010167714208364487,
	"max_abs_err": 0.0012601115740835667,
	"num_scales": 12288
	},
	"[ctrl] int8 block-128": {
	"rel_frob_err": 0.012071715667843819,
	"max_abs_err": 0.0012601115740835667,
	"num_scales": 3840
	}
	}
	}