absorb.md

Sebastian Raschka

Chronological feed of everything captured from Sebastian Raschka.

github_gist / rasbt / May 8 / failed

Compute memory usage for various LLMs like Gemma, Llama, etc.

# Sebastian Raschka 2025 # # # Usage: # python llama-vs-gemma.py \ # --auth_token hf_... \ # --model_name meta-llama/Llama-3.2-1B \ # --prompt medium import argparse from packaging import version import requests import torch import transformers from transformers import AutoTokenizer, AutoModelForCausalLM def load_text(): url = "https://raw.githubusercontent.com/rasbt/LLMs-from-scratch/refs/heads/main/ch02/01_main-chapter-code/the-verdict.txt" response = requests.get(url) text = response.text return text def print_gpu_memory_usage(stage): if torch.cuda.is_available(): allocated = torch.cuda.memory_allocated(0) / (1024 ** 2) reserved = torch.cuda.memory_reserved(0) / (1024 ** 2) peak_memory = torch.cuda.max_memory_allocated(0) / (1024 ** 2) print(f"{stage} - GPU allocated: {allocated:.2f} MB, GPU Max allocated: {peak_memory:.2f} MB, reserved: {reserved:.2f} MB") else: print("CUDA is not available") def main(args): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print_gpu_memory_usage("Before loading model") tokenizer = AutoTokenizer.from_pretrained(args.model_name, token=args.auth_token) model = AutoModelForCausalLM.from_pretrained( args.model_name, token=args.auth_token, torch_dtype=torch.bfloat16 ).to(device) print_gpu_memory_usage("After loading model") if args.prompt == "short": prompt = "Hello world" elif args.prompt == "medium": prompt = load_text()*3 elif args.prompt == "long": prompt = load_text()*8 else: prompt = args.prompt inputs = tokenizer(prompt, return_tensors="pt").to(device) print("Prompt length:", len(inputs.input_ids[0])) if "gemma-3" in args.model_name: if version.parse(transformers.__version__) < version.parse("4.50.0.dev0"): raise RuntimeError(f"Detected transformers {transformers.__version__} but requires transformers >= 4.50.0") with torch.no_grad(): outputs = model.generate(**inputs, max_new_tokens=50) print_gpu_memory_usage("After generation") result = tokenizer.decode(outputs[0], skip_special_tokens=True) print("Generated output:", result[-100:]) if __name__ == "__main__": parser = argparse.ArgumentParser( description="Load Llama 3.2 1B model, generate text, and measure GPU memory usage." ) parser.add_argument("--auth_token", type=str, required=True, help="Hugging Face access token") parser.add_argument("--model_name", type=str, default="meta-llama/Llama-3.2-1B", help="Model name to load from Hugging Face") parser.add_argument("--prompt", type=str, default="Hello world", help="Prompt text to generate text for") args = parser.parse_args() main(args)

github_gist / rasbt / May 8 / failed

Reasoning Model Inference Scaling Example with DeepSeek and GitHub Models

{ "cells": [ { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "BASE_PROMPT = \"What is (13 × 4) + (6 × 7)\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Plain DeepSeek-V3" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Completion ===\n", "To solve the expression \\((13 \\times 4) + (6 \\times 7)\\), follow these steps:\n", "\n", "1. **Calculate each multiplication separately:**\n", " \\[\n", " 13 \\times 4 = 52\n", " \\]\n", " \\[\n", " 6 \\times 7 = 42\n", " \\]\n", "\n", "2. **Add the results of the multiplications:**\n", " \\[\n", " 52 + 42 = 94\n", " \\]\n", "\n", "**Final Answer:**\n", "\\[\n", "\\boxed{94}\n", "\\]\n", "\n", "=== Token Usage ===\n", "Prompt tokens: 18\n", "Completion tokens: 107\n", "Total tokens: 125\n" ] } ], "source": [ "\"\"\"Run this model in Python\n", "\n", "> pip install azure-ai-inference\n", "\"\"\"\n", "import os\n", "from azure.ai.inference import ChatCompletionsClient\n", "from azure.ai.inference.models import SystemMessage\n", "from azure.ai.inference.models import UserMessage\n", "from azure.core.credentials import AzureKeyCredential\n", "\n", "# To authenticate with the model you will need to generate a personal access token (PAT) in your GitHub settings. \n", "# Create your PAT token by following instructions here: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens\n", "client = ChatCompletionsClient(\n", " endpoint=\"https://models.inference.ai.azure.com\",\n", " credential=AzureKeyCredential(os.environ[\"GITHUB_TOKEN\"]),\n", ")\n", "\n", "response = client.complete(\n", " messages=[\n", " SystemMessage(\"\"\"\"\"\"),\n", " UserMessage(BASE_PROMPT),\n", " ],\n", " model=\"DeepSeek-V3\",\n", " temperature=0.8,\n", " max_tokens=2048,\n", " top_p=0.1\n", ")\n", "\n", "# Print content\n", "content = response.choices[0].message.content\n", "print(\"=== Completion ===\")\n", "print(content)\n", "\n", "# Try to print token usage if available\n", "try:\n", " usage = response.usage\n", " print(\"\\n=== Token Usage ===\")\n", " print(f\"Prompt tokens: {usage.prompt_tokens}\")\n", " print(f\"Completion tokens: {usage.completion_tokens}\")\n", " print(f\"Total tokens: {usage.total_tokens}\")\n", "except AttributeError:\n", " print(\"\\n[⚠️] Token usage info not available in this response.\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## DeepSeek-V3: Think step by step" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Completion ===\n", "Sure! Let's break it down step by step.\n", "\n", "**Step 1:** Calculate \\(13 \\times 4\\). \n", "\\[\n", "13 \\times 4 = 52\n", "\\]\n", "\n", "**Step 2:** Calculate \\(6 \\times 7\\). \n", "\\[\n", "6 \\times 7 = 42\n", "\\]\n", "\n", "**Step 3:** Add the results from Step 1 and Step 2. \n", "\\[\n", "52 + 42 = 94\n", "\\]\n", "\n", "**Final Answer:** \n", "\\[\n", "\\boxed{94}\n", "\\]\n", "\n", "=== Token Usage ===\n", "Prompt tokens: 23\n", "Completion tokens: 108\n", "Total tokens: 131\n" ] } ], "source": [ "response = client.complete(\n", " messages=[\n", " SystemMessage(\"\"\"\"\"\"),\n", " UserMessage(BASE_PROMPT + \"Think step by step.\"),\n", " ],\n", " model=\"DeepSeek-V3\",\n", " temperature=0.8,\n", " max_tokens=2048,\n", " top_p=0.1\n", ")\n", "\n", "# Print content\n", "content = response.choices[0].message.content\n", "print(\"=== Completion ===\")\n", "print(content)\n", "\n", "# Try to print token usage if available\n", "try:\n", " usage = response.usage\n", " print(\"\\n=== Token Usage ===\")\n", " print(f\"Prompt tokens: {usage.prompt_tokens}\")\n", " print(f\"Completion tokens: {usage.completion_tokens}\")\n", " print(f\"Total tokens: {usage.total_tokens}\")\n", "except AttributeError:\n", " print(\"\\n[⚠️] Token usage info not available in this response.\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "##

tweet / @rasbt / May 8 / failed
tweet / @rasbt / May 8 / failed
tweet / @rasbt / May 8 / failed
tweet / @rasbt / May 8 / failed
tweet / @rasbt / May 8 / failed
tweet / @rasbt / May 8 / failed
tweet / @rasbt / May 8 / failed
tweet / @rasbt / May 8 / failed
tweet / @rasbt / May 8 / failed
tweet / @rasbt / May 8 / failed