{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "48377ac5-883f-4fd8-b229-81acc50ef133", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Defaulting to user installation because normal site-packages is not writeable\n", "Collecting asyncio\n", " Downloading asyncio-3.4.3-py3-none-any.whl.metadata (1.7 kB)\n", "Downloading asyncio-3.4.3-py3-none-any.whl (101 kB)\n", "Installing collected packages: asyncio\n", "Successfully installed asyncio-3.4.3\n", "Defaulting to user installation because normal site-packages is not writeable\n", "Collecting openai\n", " Downloading openai-1.83.0-py3-none-any.whl.metadata (25 kB)\n", "Requirement already satisfied: anyio<5,>=3.5.0 in /opt/conda/lib/python3.11/site-packages (from openai) (4.6.2.post1)\n", "Requirement already satisfied: distro<2,>=1.7.0 in /opt/conda/lib/python3.11/site-packages (from openai) (1.9.0)\n", "Requirement already satisfied: httpx<1,>=0.23.0 in /opt/conda/lib/python3.11/site-packages (from openai) (0.27.2)\n", "Collecting jiter<1,>=0.4.0 (from openai)\n", " Downloading jiter-0.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.2 kB)\n", "Requirement already satisfied: pydantic<3,>=1.9.0 in /opt/conda/lib/python3.11/site-packages (from openai) (2.9.2)\n", "Requirement already satisfied: sniffio in /opt/conda/lib/python3.11/site-packages (from openai) (1.3.1)\n", "Requirement already satisfied: tqdm>4 in /opt/conda/lib/python3.11/site-packages (from openai) (4.66.5)\n", "Requirement already satisfied: typing-extensions<5,>=4.11 in /opt/conda/lib/python3.11/site-packages (from openai) (4.12.2)\n", "Requirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.11/site-packages (from anyio<5,>=3.5.0->openai) (3.10)\n", "Requirement already satisfied: certifi in /opt/conda/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai) (2024.8.30)\n", "Requirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai) (1.0.6)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /opt/conda/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n", "Requirement already satisfied: annotated-types>=0.6.0 in /opt/conda/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.23.4 in /opt/conda/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai) (2.23.4)\n", "Downloading openai-1.83.0-py3-none-any.whl (723 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m723.4/723.4 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading jiter-0.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (352 kB)\n", "Installing collected packages: jiter, openai\n", "\u001b[33m WARNING: The script openai is installed in '/home/harshal.patil-gmail.com/.local/bin' which is not on PATH.\n", " Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n", "\u001b[0mSuccessfully installed jiter-0.10.0 openai-1.83.0\n" ] } ], "source": [ "#AIE 1.6 Run\n", "!pip install asyncio\n", "!pip install openai" ] }, { "cell_type": "code", "execution_count": null, "id": "41411218-7b1f-45d7-9ca7-2f90d896f274", "metadata": { "jupyter": { "source_hidden": true } }, "outputs": [], "source": [ "# NOT REQD\n", "import asyncio\n", "import time\n", "from openai import AsyncOpenAI\n", "\n", "async def make_openai_request(client, prompt):\n", " \"\"\"Makes an asynchronous request to the OpenAI API.\"\"\"\n", " try:\n", " response = await client.completions.create(\n", " model=\"gpt-3.5-turbo-instruct\", # or another suitable model\n", " prompt=prompt,\n", " max_tokens=50\n", " )\n", " return response.choices[0].text.strip()\n", " except Exception as e:\n", " print(f\"Error during API call: {e}\")\n", " return None\n", "\n", "async def run_concurrent_requests(num_requests, prompt):\n", " \"\"\"Runs multiple OpenAI API requests concurrently.\"\"\"\n", " client = AsyncOpenAI()\n", " tasks = [make_openai_request(client, prompt) for _ in range(num_requests)]\n", " \n", " start_time = time.time()\n", " results = await asyncio.gather(*tasks)\n", " end_time = time.time()\n", "\n", " print(f\"Total time for {num_requests} requests: {end_time - start_time:.2f} seconds\")\n", " return results\n", "\n", "async def main():\n", " \"\"\"Main function to execute the concurrency test.\"\"\"\n", " num_requests = 20 # Adjust as needed\n", " prompt = \"Write a short sentence.\"\n", "\n", " results = await run_concurrent_requests(num_requests, prompt)\n", "\n", " successful_requests = sum(1 for result in results if result is not None)\n", " print(f\"Successful requests: {successful_requests}/{num_requests}\")\n", "\n", "if __name__ == \"__main__\":\n", " asyncio.run(main())" ] }, { "cell_type": "code", "execution_count": 1, "id": "43c48edf-4482-484c-8cc3-41ecbfa291d9", "metadata": {}, "outputs": [], "source": [ "import asyncio\n", "import time\n", "from openai import AsyncOpenAI" ] }, { "cell_type": "code", "execution_count": 25, "id": "85cc6051-5154-4841-86ef-6a43882f7816", "metadata": {}, "outputs": [], "source": [ "model_used = \"meta/llama-3.1-8b-instruct\"\n", "max_tokens_used = 50\n", "num_requests_used = 20\n", "print_response = 1" ] }, { "cell_type": "code", "execution_count": 3, "id": "1308b886-5363-4c85-8834-8ddf53423a31", "metadata": {}, "outputs": [], "source": [ "PROMPT_LIST = [\n", " \"Write a short story about a robot who learns to love.\",\n", " \"Generate a poem about the beauty of a sunset.\",\n", " \"Explain the concept of quantum entanglement in simple terms.\",\n", " \"Describe the taste of a freshly baked apple pie.\",\n", " \"Write a funny dialogue between two talking animals.\",\n", " \"Suggest five creative uses for old newspapers.\",\n", " \"Compose a song about exploring a new city.\",\n", " \"What are the pros and cons of artificial intelligence?\",\n", " \"Write a haiku about a rainy day.\",\n", " \"Describe a futuristic transportation system.\",\n", " \"Tell me a joke about a cat.\",\n", " \"Explain the process of photosynthesis.\",\n", "]" ] }, { "cell_type": "code", "execution_count": 4, "id": "8892282b-6acd-4c99-9c89-b4a53d5d3102", "metadata": {}, "outputs": [], "source": [ "# Get these details from MLIS model deployment\n", "mlis_api_key = \"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE3NDQwMTcwNTAsImlzcyI6ImFpb2xpQGhwZS5jb20iLCJzdWIiOiI3YjU1NjY1My0xYWQzLTQyYzYtYmYyYi02N2I5NmU3ZDQ1MDIiLCJ1c2VyIjoiaGFyc2hhbC5wYXRpbC1nbWFpbC5jb20ifQ.p4XORb2TbRSlSJ5oI1Ooxr9bmPwraNM4rLChy9x0WHoxF9QLzLC64Te--m-CZBgAGM6ne-RbTtnFw0pLuP4jpkXiEdEEKNxVTN3b0VBuBJmQ4_b2azXPjSB8t_PMc1mtOOehhWmePnEhs5MJ5PmsshvVYmhF8niK8cVNYxDK9u-rOLRTE2SUS7O4Ln6R2OpzGYMEV6Q4ZUr5F3U76su7WfVXIpG3iDV8-DNXNC-RllR-V_ZX1N0-hcEDHINX0KlFMc9l7-JgOmpKjytDbqbrM1UEjREXENjD255bZnlcziCSwmJjhFsVOJzzryW8jLIdQarVtWS3mKr2g_tq8bhz2Q\"\n", "mlis_api_endpoint = \"https://llama-31-nim-predictor-jordan-nanos-hp-d222129e.hpepcai-ingress.pcai.hpecic.net/v1\"\n", "mlis_model = \"meta/llama-3.1-8b-instruct\"" ] }, { "cell_type": "code", "execution_count": 10, "id": "21b9d551-fcc9-4994-97f1-0293709968c3", "metadata": {}, "outputs": [], "source": [ "# Get these details from MLIS model deployment\n", "# add v1 endpoint to URL\n", "with open(\"token.txt\",\"r\") as file:\n", " mlis_api_key= file.read()\n", "mlis_api_endpoint=\"https://llama318b-predictor-harshal-patil-g-734e396c.hpepcai-ingress.pcai.hpecic.net/v1\"\n", "mlis_model = \"meta/llama-3.1-8b-instruct\"" ] }, { "cell_type": "code", "execution_count": 11, "id": "5772721a-d04e-4c58-a192-f1e844d88f88", "metadata": {}, "outputs": [], "source": [ "#initialize OpenAI compatible client\n", "import openai\n", "import os\n", "\n", "# new\n", "from openai import OpenAI\n", "\n", "client = OpenAI(\n", " api_key= mlis_api_key, \n", " base_url= mlis_api_endpoint\n", ")" ] }, { "cell_type": "code", "execution_count": 12, "id": "c435097c-ee63-4bae-8331-e24b8137d71c", "metadata": {}, "outputs": [], "source": [ "# Ignore certificate checking - for self signed certs\n", "from openai import OpenAI\n", "import httpx\n", "httpx_client = httpx.Client(http2=True, verify=False)\n", "\n", "client = OpenAI(\n", " api_key=mlis_api_key,\n", " base_url= mlis_api_endpoint,\n", " http_client=httpx_client\n", ")" ] }, { "cell_type": "code", "execution_count": 14, "id": "4c55489c-c9db-42b5-9d52-7428bf0dc13f", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "HTTP Request: POST https://llama318b-predictor-harshal-patil-g-734e396c.hpepcai-ingress.pcai.hpecic.net/v1/chat/completions \"HTTP/2 200 OK\"\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "It was the year 1998 and the internet was still in its wild west days. Emma's family had just subscribed to America Online and she was excited to explore this mystical world of information and connections.\n", "\n", "As she booted up her computer, the familiar sound of the dial-up modem whining to life filled the air. Emma's eyes widened as the \"You've Got Mail\" jingle played, signaling her entrance into the digital realm.\n", "\n", "She clicked through the desktop, passing over Monopoly and AOL Instant Messenger, and landed on the ListServ archives. Emma had been obsessed with chat rooms for weeks, and she poured over the threads, exchanging flirtatious messages with a fellow user named \"AnimeBoy.\"\n", "\n", "As she typed furiously, the phone rang in the background, breaking Emma's concentration. \"Oh no!\" she cried, hitting the pause button. \"What if AnimeBoy thinks I hung up on him?\"\n" ] } ], "source": [ "# stream response with OpenAI API\n", "try:\n", " stream = client.chat.completions.create(\n", " model= mlis_model, \n", " messages=[\n", " {\"role\": \"user\", \"content\": \"Write a short story about a internet in 200 words or less\"}\n", " ],\n", " stream=True # Enable streaming\n", " )\n", " for chunk in stream:\n", " if chunk.choices:\n", " content = chunk.choices[0].delta.content\n", " if content is not None:\n", " print(content, end=\"\", flush=True)\n", " print() # Add a newline at the end\n", "except openai.APIError as e:\n", " print(f\"An API error occurred: {e}\")" ] }, { "cell_type": "code", "execution_count": 21, "id": "a741d10f-6d1c-4093-9a19-13b940937512", "metadata": {}, "outputs": [], "source": [ "async def make_openai_request(client, prompt):\n", " \n", " # ignore prompt param and use random prompt\n", " import random\n", " random_integer = random.randint(0, len(PROMPT_LIST)-1)\n", " prompt = PROMPT_LIST[random_integer]\n", " \n", " \"\"\"Makes an asynchronous request to the OpenAI API.\"\"\"\n", " try:\n", " response = await client.completions.create(\n", " model=model_used, # or another suitable model\n", " prompt=prompt,\n", " max_tokens=max_tokens_used\n", " )\n", " if print_response:\n", " print(response)\n", " print(\"###################\")\n", " return response.choices[0].text.strip()\n", " except Exception as e:\n", " print(f\"Error during API call: {e}\")\n", " return None" ] }, { "cell_type": "code", "execution_count": 16, "id": "f4268686-840b-43d0-acb7-55bc1ac83961", "metadata": {}, "outputs": [], "source": [ "async def run_concurrent_requests(num_requests, prompt):\n", " \"\"\"Runs multiple OpenAI API requests concurrently.\"\"\"\n", " # Ignore certificate checking - for self signed certs\n", " from openai import OpenAI\n", " import httpx\n", " httpx_client = httpx.AsyncClient(http2=True, verify=False)\n", " \n", " client = AsyncOpenAI(\n", " api_key= mlis_api_key, \n", " base_url= mlis_api_endpoint,\n", " http_client=httpx_client\n", " )\n", " tasks = [make_openai_request(client, prompt) for _ in range(num_requests)]\n", " \n", " start_time = time.time()\n", " results = await asyncio.gather(*tasks)\n", " end_time = time.time()\n", "\n", " print(f\"Total time for {num_requests} requests: {end_time - start_time:.2f} seconds\")\n", " return results" ] }, { "cell_type": "code", "execution_count": 17, "id": "95ee15b9-c5c7-46d1-8514-859603e6e0ed", "metadata": {}, "outputs": [], "source": [ "async def main():\n", "\n", " \n", " \"\"\"Main function to execute the concurrency test.\"\"\"\n", " num_requests = num_requests_used # Adjust as needed\n", " \n", " prompt = \"Write any random short sentence different from before\"\n", " results = await run_concurrent_requests(num_requests, prompt)\n", "\n", " successful_requests = sum(1 for result in results if result is not None)\n", " print(f\"Successful requests: {successful_requests}/{num_requests}\")" ] }, { "cell_type": "code", "execution_count": 18, "id": "0a03ff13-18ea-49d4-94ba-028045350bfc", "metadata": {}, "outputs": [], "source": [ "#asyncio.run(main())" ] }, { "cell_type": "code", "execution_count": 19, "id": "878dedb9-b681-4874-8d6e-ca6406829c68", "metadata": {}, "outputs": [], "source": [ "# INFO WARNING ERROR CRITICAL\n", "# turn off printing of each HTTP request \n", "import logging\n", "logging.getLogger(\"httpx\").setLevel(logging.WARNING)" ] }, { "cell_type": "code", "execution_count": 26, "id": "e6050607-fa0e-4107-91b7-ab5c3614895b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total time for 10000 requests: 160.92 seconds\n", "Successful requests: 10000/10000\n" ] } ], "source": [ "# existing Asyncio loop\n", "loop = asyncio.get_event_loop()\n", "await loop.create_task(main())" ] }, { "cell_type": "code", "execution_count": null, "id": "a2fef8cf-b600-4b50-b962-a4f25672c5ab", "metadata": {}, "outputs": [], "source": [ "# test Inference Scaling\n", "model_used = \"meta/llama-3.1-8b-instruct\"\n", "max_tokens_used = 50\n", "num_requests_used = 10000\n", "print_response = 0" ] }, { "cell_type": "code", "execution_count": null, "id": "29c57c46-2300-4f37-8af4-e350644c5fb1", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.9" } }, "nbformat": 4, "nbformat_minor": 5 }