Python with Hugging Face transformers

Use this when you need Python-level control — custom generation logic, embedding extraction, research, or fine-tuning later. This path uses the original safetensors weights, not GGUF.

pip install transformers accelerate torch bitsandbytes --break-system-packages

from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

model_id = "Qwen/Qwen3-8B-Instruct"

# 4-bit quantization at load time (the bitsandbytes library, analogous
# in spirit to GGUF quantization but done in Python at runtime)
quant_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16)

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=quant_config,
    device_map="auto",   # automatically places layers on available GPU(s)/CPU
)

messages = [{"role": "user", "content": "What's the time complexity of merge sort?"}]
inputs = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True).to(model.device)

output = model.generate(inputs, max_new_tokens=300, temperature=0.7, do_sample=True)
print(tokenizer.decode(output[0][inputs.shape[-1]:], skip_special_tokens=True))