{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f714fa20-78bc-4c1a-89e4-fa31fe7e6c27",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: bentoml==1.1.11 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (1.1.11)\n",
      "Requirement already satisfied: openllm==0.4.44 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (0.4.44)\n",
      "Requirement already satisfied: openllm-client==0.4.44 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (0.4.44)\n",
      "Requirement already satisfied: aiohttp in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (3.12.2)\n",
      "Requirement already satisfied: attrs>=21.1.0 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (24.2.0)\n",
      "Requirement already satisfied: cattrs<23.2.0,>=22.1.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (23.1.2)\n",
      "Requirement already satisfied: circus!=0.17.2,>=0.17.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.19.0)\n",
      "Requirement already satisfied: click-option-group in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.5.7)\n",
      "Requirement already satisfied: click>=7.0 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (8.1.7)\n",
      "Requirement already satisfied: cloudpickle>=2.0.0 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (2.2.1)\n",
      "Requirement already satisfied: deepmerge in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (2.0)\n",
      "Requirement already satisfied: fs in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (2.4.16)\n",
      "Requirement already satisfied: httpx in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (0.27.2)\n",
      "Requirement already satisfied: inflection in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.5.1)\n",
      "Requirement already satisfied: jinja2>=3.0.1 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (3.1.4)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (1.26.4)\n",
      "Requirement already satisfied: nvidia-ml-py<12 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (11.525.150)\n",
      "Requirement already satisfied: opentelemetry-api==1.20.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (1.20.0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-aiohttp-client==0.41b0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.41b0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.41b0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.41b0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation==0.41b0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.41b0)\n",
      "Requirement already satisfied: opentelemetry-sdk==1.20.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (1.20.0)\n",
      "Requirement already satisfied: opentelemetry-semantic-conventions==0.41b0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.41b0)\n",
      "Requirement already satisfied: opentelemetry-util-http==0.41b0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.41b0)\n",
      "Requirement already satisfied: packaging>=22.0 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (24.1)\n",
      "Requirement already satisfied: pathspec in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.12.1)\n",
      "Requirement already satisfied: pip-requirements-parser>=31.2.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (32.0.1)\n",
      "Requirement already satisfied: pip-tools>=6.6.2 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (7.3.0)\n",
      "Requirement already satisfied: prometheus-client>=0.10.0 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (0.21.0)\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (6.0.0)\n",
      "Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (2.9.0)\n",
      "Requirement already satisfied: python-json-logger in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (2.0.7)\n",
      "Requirement already satisfied: python-multipart in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.0.20)\n",
      "Requirement already satisfied: pyyaml>=5.0 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (6.0.2)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (2.32.3)\n",
      "Requirement already satisfied: rich>=11.2.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (14.0.0)\n",
      "Requirement already satisfied: schema in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.7.7)\n",
      "Requirement already satisfied: simple-di>=0.1.4 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (0.1.5)\n",
      "Requirement already satisfied: starlette>=0.24.0 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (0.41.0)\n",
      "Requirement already satisfied: uvicorn in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (0.32.0)\n",
      "Requirement already satisfied: watchfiles>=0.15.0 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (0.24.0)\n",
      "Requirement already satisfied: accelerate in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm==0.4.44) (1.7.0)\n",
      "Requirement already satisfied: bitsandbytes<0.42 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm==0.4.44) (0.41.3.post2)\n",
      "Requirement already satisfied: build<1 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from build[virtualenv]<1->openllm==0.4.44) (0.10.0)\n",
      "Requirement already satisfied: cuda-python in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm==0.4.44) (12.9.0)\n",
      "Requirement already satisfied: einops in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm==0.4.44) (0.8.1)\n",
      "Requirement already satisfied: ghapi in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm==0.4.44) (1.0.6)\n",
      "Requirement already satisfied: openllm-core>=0.4.44 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm==0.4.44) (0.4.44)\n",
      "Requirement already satisfied: optimum>=1.12.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm==0.4.44) (1.25.3)\n",
      "Requirement already satisfied: safetensors in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm==0.4.44) (0.5.3)\n",
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.11/site-packages (from openllm==0.4.44) (1.11.3)\n",
      "Requirement already satisfied: sentencepiece in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm==0.4.44) (0.2.0)\n",
      "Requirement already satisfied: transformers>=4.36.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from transformers[tokenizers,torch]>=4.36.0->openllm==0.4.44) (4.52.3)\n",
      "Requirement already satisfied: anyio in /opt/conda/lib/python3.11/site-packages (from openllm-client==0.4.44) (4.6.2.post1)\n",
      "Requirement already satisfied: distro in /opt/conda/lib/python3.11/site-packages (from openllm-client==0.4.44) (1.9.0)\n",
      "Requirement already satisfied: deprecated>=1.2.6 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-api==1.20.0->bentoml==1.1.11) (1.2.14)\n",
      "Requirement already satisfied: importlib-metadata<7.0,>=6.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from opentelemetry-api==1.20.0->bentoml==1.1.11) (6.11.0)\n",
      "Requirement already satisfied: setuptools>=16.0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-instrumentation==0.41b0->bentoml==1.1.11) (75.1.0)\n",
      "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-instrumentation==0.41b0->bentoml==1.1.11) (1.16.0)\n",
      "Requirement already satisfied: asgiref~=3.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from opentelemetry-instrumentation-asgi==0.41b0->bentoml==1.1.11) (3.8.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-sdk==1.20.0->bentoml==1.1.11) (4.12.2)\n",
      "Requirement already satisfied: pyproject_hooks in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from build<1->build[virtualenv]<1->openllm==0.4.44) (1.2.0)\n",
      "Requirement already satisfied: virtualenv>=20.0.35 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from build[virtualenv]<1->openllm==0.4.44) (20.31.2)\n",
      "Requirement already satisfied: pyzmq>=17.0 in /opt/conda/lib/python3.11/site-packages (from circus!=0.17.2,>=0.17.0->bentoml==1.1.11) (26.2.0)\n",
      "Requirement already satisfied: tornado>=5.0.2 in /opt/conda/lib/python3.11/site-packages (from circus!=0.17.2,>=0.17.0->bentoml==1.1.11) (6.4.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.11/site-packages (from jinja2>=3.0.1->bentoml==1.1.11) (3.0.2)\n",
      "Requirement already satisfied: mypy-extensions in /opt/conda/lib/python3.11/site-packages (from openllm-core>=0.4.44->openllm==0.4.44) (1.0.0)\n",
      "Requirement already satisfied: orjson in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from openllm-core>=0.4.44->openllm==0.4.44) (3.10.18)\n",
      "Requirement already satisfied: torch>=1.11 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from optimum>=1.12.0->openllm==0.4.44) (2.6.0)\n",
      "Requirement already satisfied: huggingface-hub>=0.8.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from optimum>=1.12.0->openllm==0.4.44) (0.32.2)\n",
      "Requirement already satisfied: pyparsing in /opt/conda/lib/python3.11/site-packages (from pip-requirements-parser>=31.2.0->bentoml==1.1.11) (3.2.0)\n",
      "Requirement already satisfied: pip>=22.2 in /opt/conda/lib/python3.11/site-packages (from pip-tools>=6.6.2->bentoml==1.1.11) (24.2)\n",
      "Requirement already satisfied: wheel in /opt/conda/lib/python3.11/site-packages (from pip-tools>=6.6.2->bentoml==1.1.11) (0.44.0)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from rich>=11.2.0->bentoml==1.1.11) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.11/site-packages (from rich>=11.2.0->bentoml==1.1.11) (2.18.0)\n",
      "Requirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.11/site-packages (from anyio->openllm-client==0.4.44) (3.10)\n",
      "Requirement already satisfied: sniffio>=1.1 in /opt/conda/lib/python3.11/site-packages (from anyio->openllm-client==0.4.44) (1.3.1)\n",
      "Requirement already satisfied: filelock in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from transformers>=4.36.0->transformers[tokenizers,torch]>=4.36.0->openllm==0.4.44) (3.18.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from transformers>=4.36.0->transformers[tokenizers,torch]>=4.36.0->openllm==0.4.44) (2024.11.6)\n",
      "Requirement already satisfied: tokenizers<0.22,>=0.21 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from transformers>=4.36.0->transformers[tokenizers,torch]>=4.36.0->openllm==0.4.44) (0.21.1)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.11/site-packages (from transformers>=4.36.0->transformers[tokenizers,torch]>=4.36.0->openllm==0.4.44) (4.66.5)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from aiohttp->bentoml==1.1.11) (2.6.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from aiohttp->bentoml==1.1.11) (1.3.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from aiohttp->bentoml==1.1.11) (1.6.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from aiohttp->bentoml==1.1.11) (6.4.4)\n",
      "Requirement already satisfied: propcache>=0.2.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from aiohttp->bentoml==1.1.11) (0.3.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.17.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from aiohttp->bentoml==1.1.11) (1.20.0)\n",
      "Requirement already satisfied: cuda-bindings~=12.9.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from cuda-python->openllm==0.4.44) (12.9.0)\n",
      "Requirement already satisfied: appdirs~=1.4.3 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from fs->bentoml==1.1.11) (1.4.4)\n",
      "Requirement already satisfied: six~=1.10 in /opt/conda/lib/python3.11/site-packages (from fs->bentoml==1.1.11) (1.16.0)\n",
      "Requirement already satisfied: fastcore>=1.7.2 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from ghapi->openllm==0.4.44) (1.8.2)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.11/site-packages (from httpx->bentoml==1.1.11) (2024.8.30)\n",
      "Requirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.11/site-packages (from httpx->bentoml==1.1.11) (1.0.6)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /opt/conda/lib/python3.11/site-packages (from httpcore==1.*->httpx->bentoml==1.1.11) (0.14.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests->bentoml==1.1.11) (3.4.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests->bentoml==1.1.11) (1.26.20)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.11/site-packages (from huggingface-hub>=0.8.0->optimum>=1.12.0->openllm==0.4.44) (2024.10.0)\n",
      "Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from huggingface-hub>=0.8.0->optimum>=1.12.0->openllm==0.4.44) (1.1.2)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.11/site-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api==1.20.0->bentoml==1.1.11) (3.20.2)\n",
      "Requirement already satisfied: mdurl~=0.1 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.2.0->bentoml==1.1.11) (0.1.2)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (3.4.2)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.4.127 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (12.4.127)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.4.127 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (12.4.127)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.4.127 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (12.4.127)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (9.1.0.70)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.4.5.8 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (12.4.5.8)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.2.1.3 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (11.2.1.3)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.5.147 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (10.3.5.147)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.6.1.9 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (11.6.1.9)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.3.1.170 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (12.3.1.170)\n",
      "Requirement already satisfied: nvidia-cusparselt-cu12==0.6.2 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (0.6.2)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (2.21.5)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.4.127 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (12.4.127)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12==12.4.127 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (12.4.127)\n",
      "Requirement already satisfied: triton==3.2.0 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (3.2.0)\n",
      "Requirement already satisfied: sympy==1.13.1 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (1.13.1)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/conda/lib/python3.11/site-packages (from sympy==1.13.1->torch>=1.11->optimum>=1.12.0->openllm==0.4.44) (1.3.0)\n",
      "Requirement already satisfied: distlib<1,>=0.3.7 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from virtualenv>=20.0.35->build[virtualenv]<1->openllm==0.4.44) (0.3.9)\n",
      "Requirement already satisfied: platformdirs<5,>=3.9.1 in /opt/conda/lib/python3.11/site-packages (from virtualenv>=20.0.35->build[virtualenv]<1->openllm==0.4.44) (4.3.6)\n",
      "Requirement already satisfied: filetype in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (1.2.0)\n",
      "Requirement already satisfied: pillow in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (11.0.0)\n",
      "Requirement already satisfied: pydantic<2 in /home/harshal.patil-gmail.com/.local/lib/python3.11/site-packages (from bentoml==1.1.11) (1.10.22)\n",
      "Requirement already satisfied: pandas>=1 in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (2.1.4)\n",
      "Requirement already satisfied: pyarrow in /opt/conda/lib/python3.11/site-packages (from bentoml==1.1.11) (11.0.0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.11/site-packages (from pandas>=1->bentoml==1.1.11) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.11/site-packages (from pandas>=1->bentoml==1.1.11) (2024.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install bentoml==1.1.11 openllm==0.4.44 openllm-client==0.4.44"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "80fc4bbf-a1c6-401e-806e-7c6bca953ec5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import mlflow\n",
    "import bentoml.sklearn\n",
    "\n",
    "from sklearn import svm\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "03066446-19c9-483b-b409-e6161203c1ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load training data set\n",
    "iris = datasets.load_iris()\n",
    "X, y = iris.data, iris.target\n",
    "\n",
    "# Train the model\n",
    "clf = svm.SVC(gamma='scale')\n",
    "model = clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c87e50bc-7e1b-49cb-9a65-869f8a70a0c6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token successfully refreshed.\n"
     ]
    }
   ],
   "source": [
    "%update_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "353a85fa-82c7-40dd-a689-9f049edb6041",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Experiment: artifact_location='s3://mlflow.pcaigeneva/18', creation_time=1742367792267, experiment_id='18', last_update_time=1742367792267, lifecycle_stage='active', name='iris_experiment', tags={}>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create MLflow experiment and generate MLflow run name\n",
    "mlflow.set_experiment(\"iris_experiment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0d7c927f-81fc-491d-beed-a45afd8733bb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mlflow.end_run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "554341f6-2992-45b3-ae9c-70f1163ccb63",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025/05/28 01:33:44 WARNING mlflow.models.model: Model logged without a signature and input example. Please set `input_example` parameter when logging the model to auto infer the model signature.\n",
      "Registered model 'IrisRandomForestModel' already exists. Creating a new version of this model...\n",
      "2025/05/28 01:33:44 INFO mlflow.store.model_registry.abstract_store: Waiting up to 300 seconds for model version to finish creation. Model name: IrisRandomForestModel, version 5\n",
      "Created version '5' of model 'IrisRandomForestModel'.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<ModelVersion: aliases=[], creation_timestamp=1748396024919, current_stage='None', description='', last_updated_timestamp=1748396024919, name='IrisRandomForestModel', run_id='0803459368b84c6bbef919e59a7ca940', run_link='', source='s3://mlflow.pcaigeneva/18/0803459368b84c6bbef919e59a7ca940/artifacts/iris_clf', status='READY', status_message='', tags={}, user_id='', version='5'>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Start MLflow run\n",
    "mlflow.start_run()\n",
    "# Log the model to MLflow\n",
    "mlflow.sklearn.log_model(model, \"iris_clf\")\n",
    "\n",
    "# Register the model in MLflow\n",
    "model_uri = f\"runs:/{mlflow.active_run().info.run_id}/iris_clf\"\n",
    "mlflow.register_model(model_uri, \"IrisRandomForestModel\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bd8d8e64-e89d-4329-a0a5-7c729a77be41",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the default model signature for scikit-learn ({'predict': {'batchable': False}}) for model \"iris_clf\".\n",
      "HTTP Request: POST https://t.bentoml.com \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model saved: Model(tag=\"iris_clf:2j3i3lb3modrcjqj\")\n"
     ]
    }
   ],
   "source": [
    "# Save the scikit-learn model using BentoML\n",
    "saved_model = bentoml.sklearn.save_model(name=\"iris_clf\", model=model)\n",
    "print(f\"Model saved: {saved_model}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "09520783-7cf7-4fd0-b3ec-71a40bcb7121",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m \u001b[0m\u001b[1mTag\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mModule\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mSize\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mCreation Time\u001b[0m\u001b[1m \u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!/home/harshal.patil-gmail.com/.local/bin/bentoml models list"
   ]
  },
  {
   "cell_type": "raw",
   "id": "96d3dc09-61d3-45c7-a1b8-1cefd1cf0393",
   "metadata": {
    "tags": []
   },
   "source": [
    "Run it in a new terminal\n",
    "> bentoml serve service:svc"
   ]
  },
  {
   "cell_type": "raw",
   "id": "b613e531-1543-448c-a324-aa350abd9f23",
   "metadata": {
    "tags": []
   },
   "source": [
    "import requests\n",
    "\n",
    "requests.post(\n",
    "   \"http://127.0.0.1:3000/classify\",\n",
    "   headers={\"content-type\": \"application/json\"},\n",
    "   data=\"[[5.9, 3, 5.1, 1.8]]\",\n",
    ").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "79b17540-bfb4-48d0-8757-6686db73b48d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[2]'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "requests.post(\n",
    "   \"http://127.0.0.1:3000/classify\",\n",
    "   headers={\"content-type\": \"application/json\"},\n",
    "   data=\"[[5.9, 3, 5.1, 1.8]]\",\n",
    ").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f8599f67-c55e-4b57-b551-426b6848d63c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error: \u001b[31m[bentoml-cli] `build` failed: no Models with name 'iris_clf' exist in BentoML store <osfs '/home/harshal.patil-gmail.com/bentoml/models'>\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!/home/harshal.patil-gmail.com/.local/bin/bentoml build -f \"./bentofile.yaml\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1eccbca5-efb1-4ed9-ac0d-de9e4b6e1a4b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m \u001b[0m\u001b[1mTag\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mSize\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mModel Size\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mCreation Time\u001b[0m\u001b[1m \u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!/home/harshal.patil-gmail.com/.local/bin/bentoml list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bed36742-dff7-4485-a5ca-344edef256cb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: line 1: bentoml: command not found\n"
     ]
    }
   ],
   "source": [
    "!bentoml export iris_classifier:latest \"/mnt/shared/test/bento/iris_demo.bento\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "84495ab8-af31-4268-8497-d23c7ad96cbc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mlflow.log_artifact(\"./iris_demo.bento\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d2ac015e-763d-4b53-a27c-06bef399e295",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025/05/28 01:30:50 INFO mlflow.tracking._tracking_service.client: 🏃 View run gentle-wolf-283 at: http://mlflow.mlflow.svc.cluster.local:5000/#/experiments/0/runs/86e3068ed7b04bd89c93102b349c50f7.\n",
      "2025/05/28 01:30:50 INFO mlflow.tracking._tracking_service.client: 🧪 View experiment at: http://mlflow.mlflow.svc.cluster.local:5000/#/experiments/0.\n"
     ]
    }
   ],
   "source": [
    "mlflow.end_run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "29764779-b707-4050-ba12-60dae0a269a6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token successfully refreshed.\n"
     ]
    }
   ],
   "source": [
    "%update_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e1fcde21-2efc-4c07-9ddf-e9cc1a5f4831",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJWTUlfR1RyOEw5eXdLcUpaU0o3ZVhDazA2X1pGUnVDZ01kS2lyc1RqUTBJIn0.eyJleHAiOjE3NDgzOTg3NzcsImlhdCI6MTc0ODM5Njk3NywiYXV0aF90aW1lIjoxNzQ4MzkzNDAyLCJqdGkiOiI3MjE0ZDE1Yy04ODgzLTQ4YjQtOTJjZi1lZDVkOGIxYjdlOTQiLCJpc3MiOiJodHRwczovL2tleWNsb2FrLmhwZXBjYWktaW5ncmVzcy5wY2FpLmhwZWNpYy5uZXQvcmVhbG1zL1VBIiwic3ViIjoiNzYxYjM0NzgtOGE5OS00OWQ5LTgzZjItYjMzYmFlNjM0MzQxIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoidWEiLCJub25jZSI6ImdfUW83QXFtejBzQjVGTjB0TmlYTjVEV1lsNnh4Nkc1U3pHRDg0WjhHS3MiLCJzZXNzaW9uX3N0YXRlIjoiZGQ1NGIyNDctNWNiNy00NWZiLTk4Y2QtMWNjNDc1NjJkYjIyIiwiYWNyIjoiMSIsInNjb3BlIjoib3BlbmlkIGVtYWlsIHBjYWlnZW5ldmE6dWEgb2ZmbGluZV9hY2Nlc3MgcHJvZmlsZSIsInNpZCI6ImRkNTRiMjQ3LTVjYjctNDVmYi05OGNkLTFjYzQ3NTYyZGIyMiIsInVpZCI6IjEwMDAxNzU3IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJnaWQiOiIxMDAxIiwibmFtZXNwYWNlIjoiaGFyc2hhbC1wYXRpbC1nLTY1MzdiYzRkIiwibmFtZSI6IkhhcnNoYWwgUGF0aWwiLCJncm91cHMiOlsidWEtZW5hYmxlZCIsIm9mZmxpbmVfYWNjZXNzIiwiYWRtaW4iLCJ1bWFfYXV0aG9yaXphdGlvbiIsImRlZmF1bHQtcm9sZXMtdWEiXSwicHJlZmVycmVkX3VzZXJuYW1lIjoiaGFyc2hhbC5wYXRpbC1nbWFpbC5jb20iLCJnaXZlbl9uYW1lIjoiSGFyc2hhbCIsInBvc2l4X3VzZXJuYW1lIjoiaGFyc2hhbC5wYXRpbC1nbWFpbC5jb20iLCJmYW1pbHlfbmFtZSI6IlBhdGlsIiwiZW1haWwiOiJoYXJzaGFsLnBhdGlsQGdtYWlsLmNvbSJ9.IambM6Rbo4htq9phRxhdzDBWXL1xEKL7ZkLU8vvtM1Ils2l4PL5E5cQ2rjaeWLFNU6j6U3Yw3PljnBCOnQ3k7sQYq_9RYjUkzxn7LdXrGk62taBaOEUUP2VrQ3bR6lt5CjSJKyk8LUqsiZhcXw8k9oAqjSQ4XKSC6o0Bcg3tGmmtShsB3-ZPFnOlH-Zl6ch-AefXSmb9shHhT3iliI22LCAivy-TNqDiY5lEiY_zxFa1WOYvkHDG3Vx_Mewy9YUtT6oTUrY1Rrc2F8YFmW8Mm75uyfgRcMtQvmiaclU_qypWn6PyC8We-Eq9A0M5xHeptxcMr5ius9oC-8eMqkh2Bg"
     ]
    }
   ],
   "source": [
    "%cat /etc/secrets/ezua/.auth_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "637b1b87-ffb0-4087-81ae-3f99d56385a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"/etc/secrets/ezua/.auth_token\",\"r\") as file:\n",
    "     TOKEN= file.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "04f709d6-8653-4d5f-b9a1-8b30b45b7b81",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/urllib3/connectionpool.py:1064: InsecureRequestWarning: Unverified HTTPS request is being made to host 'irisdeploy1-predictor-harshal-patil-g-6537bc4d.hpepcai-ingress.pcai.hpecic.net'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'[2]'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "iris_model_url = \"https://irisdeploy1-predictor-harshal-patil-g-6537bc4d.hpepcai-ingress.pcai.hpecic.net\"\n",
    "requests.post(\n",
    "   iris_model_url+\"/classify\",\n",
    "   headers={\"content-type\": \"application/json\",\n",
    "            \"Authorization\": \"Bearer \"+TOKEN,\n",
    "           },\n",
    "   data=\"[[5.9, 3, 5.1, 1.8]]\",\n",
    "    verify=False,\n",
    ").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8c4148d-1ce4-41e1-8455-97054d45a3de",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}