Small Language Fashions with Hugging Face transformers Library + smolLM3

# Python 3.10 or newer required
python --version

# Create and activate a digital atmosphere
python -m venv smollm-env
supply smollm-env/bin/activate       # macOS / Linux
smollm-envScriptsactivate          # Home windows

# Set up all dependencies
pip set up 
  "transformers>=4.53.0" 
  "torch>=2.3.0" 
  "speed up>=0.30.0" 
  "bitsandbytes>=0.43.0" 
  "sentencepiece" 
  "trl>=0.9.0" 
  "peft>=0.11.0" 
  "datasets>=2.19.0"

# device_check.py
# Run this earlier than the rest to substantiate your setup and choose the fitting dtype.

def detect_device():
    """
    Detect the perfect out there compute machine.
    Returns (device_str, dtype_str, load_kwargs) to be used with from_pretrained.
    """
    attempt:
        import torch
    besides ImportError:
        elevate RuntimeError("PyTorch not discovered. Set up with: pip set up torch")

    if torch.cuda.is_available():
        vram_gb = torch.cuda.get_device_properties(0).total_memory / 1e9
        print(f"CUDA GPU detected: {torch.cuda.get_device_name(0)} ({vram_gb:.1f} GB VRAM)")
        # bfloat16 is advisable for SmolLM3 -- it is the coaching dtype
        return "cuda", torch.bfloat16, {"device_map": "auto", "torch_dtype": torch.bfloat16}

    elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
        print("Apple Silicon MPS detected")
        # MPS helps float16 however not all bfloat16 ops -- use float16 on Apple Silicon
        return "mps", torch.float16, {"device_map": "mps", "torch_dtype": torch.float16}

    else:
        print("No GPU discovered -- working on CPU (slower however useful)")
        return "cpu", torch.float32, {"device_map": "cpu", "torch_dtype": torch.float32}


if __name__ == "__main__":
    machine, dtype, kwargs = detect_device()
    print(f"Machine : {machine}")
    print(f"Dtype  : {dtype}")
    print(f"Kwargs : {kwargs}")

CUDA GPU detected: NVIDIA GeForce RTX 3060 (12.0 GB VRAM)
Machine : cuda
Dtype  : torch.bfloat16
Kwargs : {'device_map': 'auto', 'torch_dtype': torch.bfloat16}

# first_inference.py
# Conditions: transformers>=4.53.0, torch, speed up
# Run: python first_inference.py

import re
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID = "HuggingFaceTB/SmolLM3-3B"

# ── 1. Load tokenizer and mannequin ───────────────────────────────────────────────

print(f"Loading {MODEL_ID}...")

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

mannequin = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    torch_dtype=torch.bfloat16,    # Match the coaching dtype; use float16 on Apple Silicon
    device_map="auto",             # Spreads throughout all out there GPUs, or CPU if none
)
mannequin.eval()

print(f"Mannequin loaded on: {mannequin.machine}")

# ── 2. Technology helper ──────────────────────────────────────────────────────

def generate(messages: listing[dict], max_new_tokens: int = 512) -> str:
    """
    Apply the SmolLM3 chat template, tokenize, generate, and decode.
    Strips the ... block from the output routinely
    so callers all the time obtain the ultimate reply solely.
    """
    # apply_chat_template codecs messages utilizing SmolLM3's built-in chat template
    textual content = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True,
    )
    inputs = tokenizer(textual content, return_tensors="pt").to(mannequin.machine)

    with torch.no_grad():
        output_ids = mannequin.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=0.6,   # Really helpful by the SmolLM3 crew for balanced output
            top_p=0.95,        # Nucleus sampling -- retains output targeted with out being repetitive
            do_sample=True,
        )

    # Decode solely the newly generated tokens, not the enter immediate
    new_tokens = output_ids[0][inputs["input_ids"].form[-1]:]
    uncooked = tokenizer.decode(new_tokens, skip_special_tokens=True)

    # Strip the chain-of-thought block if current.
    # In assume mode the mannequin prefixes its response with ....
    # Callers often solely want the ultimate reply that follows.
    last = re.sub(r".*?", "", uncooked, flags=re.DOTALL).strip()
    return last


# ── 3. Evaluate assume vs no_think on the identical immediate ──────────────────────────

immediate = "A buyer is charged twice for a similar order. What are three concrete steps assist ought to take?"

# no_think: quick, direct reply -- good for high-throughput classification and replies
no_think_messages = [
    {"role": "system", "content": "/no_think"},
    {"role": "user",   "content": prompt},
]

# assume: reasoning hint earlier than reply -- good for advanced choices and edge circumstances
think_messages = [
    {"role": "system", "content": "/think"},
    {"role": "user",   "content": prompt},
]

print("n── no_think mode ──")
print(generate(no_think_messages, max_new_tokens=256))

print("n── assume mode ──")
print(generate(think_messages, max_new_tokens=512))

python first_inference.py

# ticket_router.py
# Conditions: transformers>=4.53.0, torch, speed up
# Run: python ticket_router.py

import re
import json
import torch
from dataclasses import dataclass
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID      = "HuggingFaceTB/SmolLM3-3B"
ESCALATE_AT   = 0.70   # Tickets with confidence under this go to a human agent

# ── Information class for a routing outcome ──────────────────────────────────────────

@dataclass
class RoutingResult:
    ticket: str
    class: str           # billing | technical | account | common
    confidence: float       # 0.0-1.0 self-reported by the mannequin
    reply: str              # Generated in the identical language because the ticket
    escalate: bool          # True when confidence < ESCALATE_AT
    raw_output: str         # Full mannequin output for debugging


# ── System immediate ─────────────────────────────────────────────────────────────

SYSTEM_PROMPT = """You're a multilingual buyer assist router for a SaaS firm.
Your job is to categorise assist tickets and draft a useful, skilled reply.

Guidelines:
- Detect the language of the ticket routinely.
- Classify into EXACTLY ONE of: billing, technical, account, common.
- Reply within the SAME language because the ticket.
- Fee your confidence actually from 0.0 to 1.0. Low confidence means the ticket is ambiguous or exterior your information.
- Reply ONLY with a single JSON object -- no preamble, no clarification exterior the JSON.

Required format:
{"class": "", "confidence": <0.0-1.0>, "reply": ""}"""


# ── Router class ──────────────────────────────────────────────────────────────

class TicketRouter:
    def __init__(self, model_id: str = MODEL_ID):
        print(f"Loading {model_id}...")
        self.tokenizer = AutoTokenizer.from_pretrained(model_id)
        self.mannequin = AutoModelForCausalLM.from_pretrained(
            model_id,
            torch_dtype=torch.bfloat16,
            device_map="auto",
        )
        self.mannequin.eval()
        print(f"Prepared on {self.mannequin.machine}")

    def _call_model(self, ticket: str) -> str:
        """
        Format the ticket right into a chat message, run inference in no_think mode
        (sooner for classification), and return the uncooked decoded output.
        """
        messages = [
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user",   "content": ticket},
        ]
        textual content = self.tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            add_generation_prompt=True,
            enable_thinking=False,   # Quick path -- no chain-of-thought for routine classification
        )
        inputs = self.tokenizer(textual content, return_tensors="pt").to(self.mannequin.machine)

        with torch.no_grad():
            output_ids = self.mannequin.generate(
                **inputs,
                max_new_tokens=256,
                temperature=0.3,   # Decrease temp for classification -- extra deterministic output
                top_p=0.9,
                do_sample=True,
            )

        new_tokens = output_ids[0][inputs["input_ids"].form[-1]:]
        return self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()

    def _parse_output(self, uncooked: str) -> dict:
        """
        Extract the JSON object from the mannequin's output.
        Falls again to a default 'common' class with zero confidence if parsing fails.
        This prevents a JSON parse failure from crashing the pipeline.
        """
        # Discover any JSON object within the output, even when surrounded by stray textual content
        match = re.search(r"{.*?}", uncooked, re.DOTALL)
        if not match:
            return {"class": "common", "confidence": 0.0, "reply": uncooked}
        attempt:
            return json.hundreds(match.group())
        besides json.JSONDecodeError:
            return {"class": "common", "confidence": 0.0, "reply": uncooked}

    def route(self, ticket: str) -> RoutingResult:
        """
        Route a single ticket. Returns a RoutingResult with classification,
        confidence, reply, and escalation flag.
        """
        uncooked = self._call_model(ticket)
        parsed = self._parse_output(uncooked)

        class   = parsed.get("class", "common")
        confidence = float(parsed.get("confidence", 0.0))
        reply      = parsed.get("reply", "Thanks for reaching out. We'll comply with up shortly.")

        return RoutingResult(
            ticket=ticket,
            class=class,
            confidence=confidence,
            reply=reply,
            escalate=confidence < ESCALATE_AT,
            raw_output=uncooked,
        )

    def route_batch(self, tickets: listing[str]) -> listing[RoutingResult]:
        """Route an inventory of tickets sequentially. Returns leads to enter order."""
        return [self.route(t) for t in tickets]


# ── Run it ────────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    router = TicketRouter()

    test_tickets = [
        "I was charged twice for my subscription this month. Please refund the duplicate charge.",
        "L'application se bloque chaque fois que j'essaie d'exporter un fichier PDF.",   # French
        "No puedo iniciar sesión en mi cuenta desde hace dos días.",                     # Spanish
        "Die Rechnung für März fehlt in meinem Abrechnungsbereich.",                     # German
        "Il mio abbonamento non si rinnova automaticamente nonostante il pagamento.",    # Italian
    ]

    print("n" + "=" * 70)
    outcomes = router.route_batch(test_tickets)

    for r in outcomes:
        flag = "🔴 ESCALATE" if r.escalate else "🟢 AUTO"
        print(f"n{flag}")
        print(f"Ticket     : {r.ticket[:70]}...")
        print(f"Class   : {r.class}")
        print(f"Confidence : {r.confidence:.2f}")
        print(f"Reply      : {r.reply[:100]}...")

    escalated = [r for r in results if r.escalate]
    print(f"n{'─'*70}")
    print(f"Complete tickets : {len(outcomes)}")
    print(f"Auto-routed   : {len(outcomes) - len(escalated)}")
    print(f"Escalated     : {len(escalated)}")

# tool_calling.py
# Conditions: transformers>=4.53.0, torch, speed up
# Run: python tool_calling.py

import re
import json
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID  = "HuggingFaceTB/SmolLM3-3B"
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
mannequin     = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, torch_dtype=torch.bfloat16, device_map="auto"
)
mannequin.eval()

# ── Software definition ───────────────────────────────────────────────────────────
# SmolLM3 accepts device definitions as JSON Schema objects beneath xml_tools.
# The mannequin makes use of the title and outline to resolve when to name the device.
# The parameters schema tells it what arguments to incorporate within the name.

TOOLS = [
    {
        "name": "lookup_order_status",
        "description": (
            "Look up the current status, estimated delivery date, and carrier "
            "for a specific customer order. Call this when the customer mentions "
            "an order number or asks where their order is."
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "order_id": {
                    "type": "string",
                    "description": "The order ID, usually in the format ORD-XXXXXX."
                }
            },
            "required": ["order_id"]
        }
    }
]

# ── Simulated order database ──────────────────────────────────────────────────

def lookup_order_status(order_id: str) -> dict:
    """
    In manufacturing, substitute this with an actual database or API name.
    Returns a dict the mannequin can learn and summarize for the client.
    """
    database = {
        "ORD-4821": {"standing": "shipped",    "eta": "June 18, 2026", "service": "DHL"},
        "ORD-3307": {"standing": "processing", "eta": "June 20, 2026", "service": None},
        "ORD-1190": {"standing": "delivered",  "eta": None,            "service": "FedEx"},
    }
    return database.get(order_id, {"standing": "not_found", "eta": None, "service": None})

# ── Software name parser ──────────────────────────────────────────────────────────

def parse_tool_call(output: str):
    """
    Extract a device name from the mannequin's output.
    SmolLM3 emits: {"title": "...", "arguments": {...}}
    Returns (tool_name, arguments) or (None, None) if no device name is current.
    """
    match = re.search(r"(.*?)", output, re.DOTALL)
    if not match:
        return None, None
    attempt:
        payload = json.hundreds(match.group(1).strip())
        return payload.get("title"), payload.get("arguments", {})
    besides json.JSONDecodeError:
        return None, None

# ── Full tool-call spherical journey ─────────────────────────────────────────────────

def respond_with_tools(user_message: str) -> str:
    """
    Full agentic loop:
    1. Ship person message + device definitions to the mannequin.
    2. If the mannequin emits a device name, execute it and inject the outcome.
    3. Generate the ultimate customer-facing response.
    """
    # Flip 1: give the mannequin the person message and out there instruments
    messages = [{"role": "user", "content": user_message}]

    inputs = tokenizer.apply_chat_template(
        messages,
        xml_tools=TOOLS,           # Move device definitions right here
        enable_thinking=False,
        add_generation_prompt=True,
        tokenize=True,
        return_tensors="pt",
    ).to(mannequin.machine)

    with torch.no_grad():
        output_ids = mannequin.generate(
            inputs, max_new_tokens=256, temperature=0.3, top_p=0.9, do_sample=True
        )
    turn1 = tokenizer.decode(
        output_ids[0][inputs.shape[-1]:], skip_special_tokens=True
    )

    # Examine if the mannequin needs to name a device
    tool_name, tool_args = parse_tool_call(turn1)

    if tool_name == "lookup_order_status":
        # Execute the actual operate
        tool_result = lookup_order_status(**tool_args)
        print(f"  [Tool called] {tool_name}({tool_args}) → {tool_result}")

        # Flip 2: inject the device outcome and ask for the ultimate response
        messages += [
            {"role": "assistant", "content": turn1},
            {"role": "tool",      "content": json.dumps(tool_result), "name": tool_name},
        ]
        inputs2 = tokenizer.apply_chat_template(
            messages,
            xml_tools=TOOLS,
            enable_thinking=False,
            add_generation_prompt=True,
            tokenize=True,
            return_tensors="pt",
        ).to(mannequin.machine)

        with torch.no_grad():
            output_ids2 = mannequin.generate(
                inputs2, max_new_tokens=256, temperature=0.3, top_p=0.9, do_sample=True
            )
        return tokenizer.decode(
            output_ids2[0][inputs2.shape[-1]:], skip_special_tokens=True
        ).strip()

    # No device name -- mannequin answered instantly
    return turn1.strip()


# ── Check it ───────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    queries = [
        "Where is my order ORD-4821? It's been a week.",
        "My order ORD-3307 hasn't shipped yet -- what's the status?",
        "I just want to change my email address.",  # No tool needed
    ]

    for question in queries:
        print(f"nCustomer : {question}")
        response = respond_with_tools(question)
        print(f"Agent    : {response}")

# finetune.py
# Extra stipulations: pip set up trl>=0.9.0 peft>=0.11.0 datasets>=2.19.0
# Run: python finetune.py
# Time: ~8-12 minutes on an RTX 3060 for 3 epochs over 50 examples

import json
import torch
from datasets import Dataset
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig

MODEL_ID   = "HuggingFaceTB/SmolLM3-3B"
OUTPUT_DIR = "./smollm3-ticket-router"

# ── System immediate (similar because the inference router) ──────────────────────────────

SYSTEM_PROMPT = """You're a multilingual buyer assist router for a SaaS firm.
Classify the assist ticket and generate a useful reply in the identical language because the ticket.
Reply ONLY with JSON: {"class": "", "confidence": <0.0-1.0>, "reply": ""}"""

# ── Coaching knowledge ─────────────────────────────────────────────────────────────
# In manufacturing you'd load a whole lot of actual labelled tickets.
# This minimal set demonstrates the format -- broaden together with your actual knowledge.

raw_examples = [
    ("I was charged twice for my subscription.", "billing",
     "We're sorry for the duplicate charge. Our billing team will review and issue a refund within 3-5 business days."),
    ("The app crashes every time I try to export a PDF.", "technical",
     "We apologize for the inconvenience. Our engineering team has been notified and will investigate."),
    ("I can't log into my account since yesterday.", "account",
     "We're sorry you're having trouble. Please try resetting your password. If the issue continues, we'll escalate to our account team."),
    ("Die App stürzt beim Exportieren von PDFs ab.", "technical",
     "Wir entschuldigen uns für die Unannehmlichkeiten. Unser Technikteam wurde benachrichtigt und untersucht das Problem."),
    ("L'application se bloque quand j'exporte un fichier.", "technical",
     "Nous nous excusons pour la gêne occasionnée. Notre équipe technique a été informée et travaille sur ce problème."),
    ("My March invoice is missing from the billing section.", "billing",
     "Thank you for flagging this. Our billing team will locate your March invoice and resend it within 24 hours."),
    ("No puedo iniciar sesión desde ayer por la noche.", "account",
     "Lamentamos el problema de acceso. Por favor, restablezca su contraseña. Si el problema persiste, escalaremos su caso."),
    ("How do I upgrade my plan to the Pro tier?", "general",
     "You can upgrade to Pro directly from Settings → Subscription. The new rate applies from your next billing cycle."),
]

def format_example(ticket: str, class: str, reply: str) -> dict:
    """
    Format a single instance into the SmolLM3 messages format.
    The assistant flip comprises the goal JSON the mannequin ought to be taught to supply.
    """
    return {
        "messages": [
            {"role": "system",    "content": SYSTEM_PROMPT},
            {"role": "user",      "content": ticket},
            {"role": "assistant", "content": json.dumps({
                "category": category, "confidence": 0.95, "reply": reply
            })},
        ]
    }

dataset = Dataset.from_list([format_example(*ex) for ex in raw_examples])

# ── Tokenizer ─────────────────────────────────────────────────────────────────

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
tokenizer.pad_token = tokenizer.eos_token   # SmolLM3 has no separate pad token

# ── Mannequin (4-bit quantized base for QLoRA) ────────────────────────────────────

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
)
mannequin = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    quantization_config=bnb_config,
    device_map="auto",
)

# ── LoRA config ───────────────────────────────────────────────────────────────
# We goal the eye and MLP projection layers -- these carry essentially the most
# task-specific sign and provides the perfect accuracy/parameter trade-off.

lora_config = LoraConfig(
    r=16,              # Rank of the LoRA replace matrices -- larger = extra expressive, extra reminiscence
    lora_alpha=32,     # Scaling issue; conventionally set to 2*r
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",   # Attention projections
        "gate_proj", "up_proj", "down_proj",        # MLP projections (SwiGLU)
    ],
)
mannequin = get_peft_model(mannequin, lora_config)
mannequin.print_trainable_parameters()
# Anticipated: trainable params: ~13M (0.4% of 3B whole)

# ── Coaching config ───────────────────────────────────────────────────────────

sft_config = SFTConfig(
    output_dir=OUTPUT_DIR,
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,   # Efficient batch dimension = 8
    learning_rate=2e-4,
    warmup_ratio=0.1,
    lr_scheduler_type="cosine",
    bf16=True,
    logging_steps=5,
    save_strategy="epoch",
    max_seq_length=512,              # Tickets are brief -- no want for the total context window
)

# ── Practice ─────────────────────────────────────────────────────────────────────

coach = SFTTrainer(
    mannequin=mannequin,
    tokenizer=tokenizer,
    train_dataset=dataset,
    args=sft_config,
)
coach.prepare()

# ── Save and merge ────────────────────────────────────────────────────────────
# Save the LoRA adapter -- small file, straightforward to share or model.
coach.save_model(f"{OUTPUT_DIR}/adapter")

# Merge the adapter again into the bottom mannequin weights for standalone deployment.
# The merged mannequin hundreds precisely like the bottom mannequin -- no PEFT dependency at inference.
merged = mannequin.merge_and_unload()
merged.save_pretrained(f"{OUTPUT_DIR}/merged")
tokenizer.save_pretrained(f"{OUTPUT_DIR}/merged")

print(f"nFine-tuned mannequin saved to {OUTPUT_DIR}/merged")
print("Load it with: AutoModelForCausalLM.from_pretrained('./smollm3-ticket-router/merged')")

pip set up trl>=0.9.0 peft>=0.11.0 datasets>=2.19.0
python finetune.py

trainable params: 13,631,488 || all params: 3,085,123,584 || trainable%: 0.4420
{'loss': 1.842, 'learning_rate': 2e-04, 'epoch': 0.5}
{'loss': 0.923, 'learning_rate': 1.4e-04, 'epoch': 1.0}
{'loss': 0.461, 'learning_rate': 6e-05, 'epoch': 2.0}
{'loss': 0.287, 'learning_rate': 0.0, 'epoch': 3.0}