llama4_inference_multimodal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fc652877-6e04-43fb-b4ce-0a08a07f7b99",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'AutoAWQ'...\n",
      "remote: Enumerating objects: 3588, done.\u001b[K\n",
      "remote: Counting objects: 100% (842/842), done.\u001b[K\n",
      "remote: Compressing objects: 100% (303/303), done.\u001b[K\n",
      "remote: Total 3588 (delta 724), reused 561 (delta 539), pack-reused 2746 (from 3)\u001b[K\n",
      "Receiving objects: 100% (3588/3588), 7.16 MiB | 29.69 MiB/s, done.\n",
      "Resolving deltas: 100% (2276/2276), done.\n",
      "Obtaining file:///workspace/AutoAWQ\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (2.4.1+cu124)\n",
      "Requirement already satisfied: triton in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (3.0.0)\n",
      "Requirement already satisfied: transformers>=4.45.0 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (4.49.0)\n",
      "Requirement already satisfied: tokenizers>=0.12.1 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (0.21.0)\n",
      "Requirement already satisfied: typing_extensions>=4.8.0 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (4.12.2)\n",
      "Requirement already satisfied: accelerate in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (1.4.0)\n",
      "Requirement already satisfied: datasets>=2.20 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (3.3.2)\n",
      "Requirement already satisfied: zstandard in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (0.23.0)\n",
      "Requirement already satisfied: huggingface_hub>=0.26.5 in /usr/local/lib/python3.11/dist-packages (from autoawq==0.2.8) (0.29.1)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (3.13.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (1.26.4)\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (19.0.1)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (0.3.8)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (2.2.3)\n",
      "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (2.32.3)\n",
      "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (4.67.1)\n",
      "Requirement already satisfied: xxhash in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (3.5.0)\n",
      "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.12.0,>=2023.1.0 in /usr/local/lib/python3.11/dist-packages (from fsspec[http]<=2024.12.0,>=2023.1.0->datasets>=2.20->autoawq==0.2.8) (2024.2.0)\n",
      "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (3.11.12)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (24.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.20->autoawq==0.2.8) (6.0.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/dist-packages (from transformers>=4.45.0->autoawq==0.2.8) (2024.11.6)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.11/dist-packages (from transformers>=4.45.0->autoawq==0.2.8) (0.5.2)\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.11/dist-packages (from accelerate->autoawq==0.2.8) (6.0.0)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (3.1.3)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (9.1.0.70)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.4.2.65 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.2.65)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.2.0.44 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (11.2.0.44)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.5.119 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (10.3.5.119)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.6.0.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (11.6.0.99)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.3.0.142 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.3.0.142)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (2.20.5)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12==12.4.99 in /usr/local/lib/python3.11/dist-packages (from torch->autoawq==0.2.8) (12.4.99)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (2.4.6)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (1.3.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (24.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (1.5.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (6.1.0)\n",
      "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (0.3.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.20->autoawq==0.2.8) (1.18.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets>=2.20->autoawq==0.2.8) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets>=2.20->autoawq==0.2.8) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets>=2.20->autoawq==0.2.8) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets>=2.20->autoawq==0.2.8) (2024.8.30)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch->autoawq==0.2.8) (2.1.5)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets>=2.20->autoawq==0.2.8) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets>=2.20->autoawq==0.2.8) (2025.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets>=2.20->autoawq==0.2.8) (2025.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.11/dist-packages (from sympy->torch->autoawq==0.2.8) (1.3.0)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas->datasets>=2.20->autoawq==0.2.8) (1.16.0)\n",
      "Installing collected packages: autoawq\n",
      "  Attempting uninstall: autoawq\n",
      "    Found existing installation: autoawq 0.2.7.post3\n",
      "    Uninstalling autoawq-0.2.7.post3:\n",
      "      Successfully uninstalled autoawq-0.2.7.post3\n",
      "\u001b[33m  DEPRECATION: Legacy editable install of autoawq==0.2.8 from file:///workspace/AutoAWQ (setup.py develop) is deprecated. pip 25.0 will enforce this behaviour change. A possible replacement is to add a pyproject.toml or enable --use-pep517, and use setuptools >= 64. If the resulting installation is not behaving as expected, try using --config-settings editable_mode=compat. Please consult the setuptools documentation for more information. Discussion can be found at https://github.com/pypa/pip/issues/11457\u001b[0m\u001b[33m\n",
      "\u001b[0m  Running setup.py develop for autoawq\n",
      "Successfully installed autoawq-0.2.8\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: transformers in /usr/local/lib/python3.11/dist-packages (4.49.0)\n",
      "Collecting transformers\n",
      "  Downloading transformers-4.51.3-py3-none-any.whl.metadata (38 kB)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from transformers) (3.13.1)\n",
      "Collecting huggingface-hub<1.0,>=0.30.0 (from transformers)\n",
      "  Downloading huggingface_hub-0.30.2-py3-none-any.whl.metadata (13 kB)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from transformers) (1.26.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from transformers) (24.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from transformers) (6.0.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/dist-packages (from transformers) (2024.11.6)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from transformers) (2.32.3)\n",
      "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.11/dist-packages (from transformers) (0.21.0)\n",
      "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.11/dist-packages (from transformers) (0.5.2)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.11/dist-packages (from transformers) (4.67.1)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub<1.0,>=0.30.0->transformers) (2024.2.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub<1.0,>=0.30.0->transformers) (4.12.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (2024.8.30)\n",
      "Downloading transformers-4.51.3-py3-none-any.whl (10.4 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.4/10.4 MB\u001b[0m \u001b[31m168.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading huggingface_hub-0.30.2-py3-none-any.whl (481 kB)\n",
      "Installing collected packages: huggingface-hub, transformers\n",
      "  Attempting uninstall: huggingface-hub\n",
      "    Found existing installation: huggingface-hub 0.29.1\n",
      "    Uninstalling huggingface-hub-0.29.1:\n",
      "      Successfully uninstalled huggingface-hub-0.29.1\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 4.49.0\n",
      "    Uninstalling transformers-4.49.0:\n",
      "      Successfully uninstalled transformers-4.49.0\n",
      "Successfully installed huggingface-hub-0.30.2 transformers-4.51.3\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!git clone -b llama4 https://github.com/kIshizaki-sci/AutoAWQ.git\n",
    "!pip install -e ./AutoAWQ\n",
    "!pip install -U transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "846f4a3e-5960-4b52-8c79-74eee1f72468",
   "metadata": {},
   "outputs": [],
   "source": [
    "from awq import AutoAWQForCausalLM\n",
    "import torch\n",
    "import transformers\n",
    "from transformers import AutoProcessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fc58d0e4-e98a-4116-b27c-87ef1adf3620",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch version :  2.4.1+cu124\n",
      "transformers version :  4.51.3\n"
     ]
    }
   ],
   "source": [
    "print('torch version : ', torch.__version__)\n",
    "print('transformers version : ', transformers.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "47a07cd8-7874-4b57-9805-ed1f5b0dbd17",
   "metadata": {},
   "outputs": [],
   "source": [
    "quant_path = 'kishizaki-sci/Llama-4-Scout-17B-16E-Instruct-AWQ'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "adaaeee2-c985-430c-8aca-d53821418536",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/AutoAWQ/awq/models/llama4.py:312: UserWarning: Multimodal input has not been implemented in Llama4AWQForConditionalGeneration yet.\n",
      "  warnings.warn(\"Multimodal input has not been implemented in Llama4AWQForConditionalGeneration yet.\", UserWarning)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26119589fe994dd494eb7717c011fcc0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 25 files:   0%|          | 0/25 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8726ec737a8d4d5d863a6a04c0bdf47f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Replacing MoE Block...:   0%|          | 0/48 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Replacing layers...: 100%|██████████| 48/48 [00:08<00:00,  5.64it/s]\n",
      "/workspace/AutoAWQ/awq/models/base.py:539: UserWarning: Skipping fusing modules because AWQ extension is not installed.No module named 'awq_ext'\n",
      "  warnings.warn(\"Skipping fusing modules because AWQ extension is not installed.\" + msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 44min 10s, sys: 24min 2s, total: 2h 8min 12s\n",
      "Wall time: 22min 57s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model = AutoAWQForCausalLM.from_quantized(quant_path, torch_dtype=torch.float16, use_cache=True, device_map='auto')\n",
    "processor = AutoProcessor.from_pretrained(quant_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6033656a-3e4d-4f8a-b62d-5e9c69076288",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri May  2 23:45:15 2025       \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 570.86.15              Driver Version: 570.86.15      CUDA Version: 12.8     |\n",
      "|-----------------------------------------+------------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                        |               MIG M. |\n",
      "|=========================================+========================+======================|\n",
      "|   0  NVIDIA H100 NVL                On  |   00000000:C7:00.0 Off |                    0 |\n",
      "| N/A   36C    P0             92W /  310W |   60518MiB /  95830MiB |      0%      Default |\n",
      "|                                         |                        |             Disabled |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "                                                                                         \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                              |\n",
      "|  GPU   GI   CI              PID   Type   Process name                        GPU Memory |\n",
      "|        ID   ID                                                               Usage      |\n",
      "|=========================================================================================|\n",
      "+-----------------------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "60c7a4d3-9a9e-45ae-8eb5-31becfdc1865",
   "metadata": {},
   "outputs": [],
   "source": [
    "url1 = \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg\"\n",
    "url2 = \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png\"\n",
    "messages = [\n",
    "    {\n",
    "        \"role\": \"user\",\n",
    "        \"content\": [\n",
    "            {\"type\": \"image\", \"url\": url1},\n",
    "            {\"type\": \"image\", \"url\": url2},\n",
    "            {\"type\": \"text\", \"text\": \"Can you describe how these two images are similar, and how they differ?\"},\n",
    "        ]\n",
    "    },\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "474aa24f-1044-4ae4-8cc2-8625424df11b",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = processor.apply_chat_template(\n",
    "    messages,\n",
    "    add_generation_prompt=True,\n",
    "    tokenize=True,\n",
    "    return_dict=True,\n",
    "    return_tensors=\"pt\",\n",
    ").to(model.model.device, torch.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ab67b7dc-36a7-46f7-bb68-769a1cb8da7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 59.6 s, sys: 3.29 s, total: 1min 2s\n",
      "Wall time: 1min 2s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "outputs = model.generate(\n",
    "    **inputs,\n",
    "    max_new_tokens=256,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b9149cea-befc-4cd7-8eaa-77794a7baee5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The two images depict anthropomorphic animals, a rabbit and a cat, dressed in formal attire. The similarities between the images include:\n",
      "\n",
      "*   **Formal Attire**: Both characters wear formal clothing, with the rabbit dressed in a blue coat, brown vest, white shirt, and brown bow tie, while the cat wears a similar outfit consisting of a blue jacket, tan vest, white shirt, and red bow tie.\n",
      "*   **Anthropomorphic Features**: Both characters have human-like features such as standing on two legs (rabbit) or sitting upright (cat), and wearing clothes like humans.\n",
      "*   **Natural Setting**: Both images are set in a natural environment, with the rabbit standing on a dirt path surrounded by flowers and trees, and the cat sitting on a stone path in a garden-like setting.\n",
      "\n",
      "Despite these similarities, there are also some notable differences:\n",
      "\n",
      "*   **Species**: The most obvious difference is that one character is a rabbit and the other is a cat.\n",
      "*   **Posture**: The rabbit stands upright on two legs, while the cat sits on its hind legs.\n",
      "*   **Background**: Although both images feature a natural setting, the backgrounds differ slightly. The rabbit's image shows a more rural scene with a house and rolling hills, whereas the cat's\n"
     ]
    }
   ],
   "source": [
    "response = processor.batch_decode(outputs[:, inputs[\"input_ids\"].shape[-1]:])[0]\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "511b0483-65bc-47fc-8bc1-3c37b5d6003b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}