{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "48377ac5-883f-4fd8-b229-81acc50ef133", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting asyncio\n", " Downloading asyncio-4.0.0-py3-none-any.whl.metadata (994 bytes)\n", "Downloading asyncio-4.0.0-py3-none-any.whl (5.6 kB)\n", "Installing collected packages: asyncio\n", "Successfully installed asyncio-4.0.0\n", "Collecting openai\n", " Downloading openai-2.17.0-py3-none-any.whl.metadata (29 kB)\n", "Requirement already satisfied: anyio<5,>=3.5.0 in /opt/conda/lib/python3.11/site-packages (from openai) (4.9.0)\n", "Requirement already satisfied: distro<2,>=1.7.0 in /opt/conda/lib/python3.11/site-packages (from openai) (1.9.0)\n", "Requirement already satisfied: httpx<1,>=0.23.0 in /opt/conda/lib/python3.11/site-packages (from openai) (0.27.2)\n", "Collecting jiter<1,>=0.10.0 (from openai)\n", " Downloading jiter-0.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.2 kB)\n", "Requirement already satisfied: pydantic<3,>=1.9.0 in /opt/conda/lib/python3.11/site-packages (from openai) (2.11.3)\n", "Requirement already satisfied: sniffio in /opt/conda/lib/python3.11/site-packages (from openai) (1.3.1)\n", "Requirement already satisfied: tqdm>4 in /opt/conda/lib/python3.11/site-packages (from openai) (4.67.1)\n", "Requirement already satisfied: typing-extensions<5,>=4.11 in /opt/conda/lib/python3.11/site-packages (from openai) (4.13.1)\n", "Requirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.11/site-packages (from anyio<5,>=3.5.0->openai) (3.10)\n", "Requirement already satisfied: certifi in /opt/conda/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai) (2025.1.31)\n", "Requirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai) (1.0.7)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /opt/conda/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n", "Requirement already satisfied: annotated-types>=0.6.0 in /opt/conda/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.33.1 in /opt/conda/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai) (2.33.1)\n", "Requirement already satisfied: typing-inspection>=0.4.0 in /opt/conda/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai) (0.4.0)\n", "Downloading openai-2.17.0-py3-none-any.whl (1.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m87.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading jiter-0.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (362 kB)\n", "Installing collected packages: jiter, openai\n", "Successfully installed jiter-0.13.0 openai-2.17.0\n" ] } ], "source": [ "#AIE 1.6 Run\n", "!pip install asyncio\n", "!pip install openai" ] }, { "cell_type": "code", "execution_count": null, "id": "41411218-7b1f-45d7-9ca7-2f90d896f274", "metadata": { "jupyter": { "source_hidden": true } }, "outputs": [], "source": [ "# NOT REQD\n", "import asyncio\n", "import time\n", "from openai import AsyncOpenAI\n", "\n", "async def make_openai_request(client, prompt):\n", " \"\"\"Makes an asynchronous request to the OpenAI API.\"\"\"\n", " try:\n", " response = await client.completions.create(\n", " model=\"gpt-3.5-turbo-instruct\", # or another suitable model\n", " prompt=prompt,\n", " max_tokens=50\n", " )\n", " return response.choices[0].text.strip()\n", " except Exception as e:\n", " print(f\"Error during API call: {e}\")\n", " return None\n", "\n", "async def run_concurrent_requests(num_requests, prompt):\n", " \"\"\"Runs multiple OpenAI API requests concurrently.\"\"\"\n", " client = AsyncOpenAI()\n", " tasks = [make_openai_request(client, prompt) for _ in range(num_requests)]\n", " \n", " start_time = time.time()\n", " results = await asyncio.gather(*tasks)\n", " end_time = time.time()\n", "\n", " print(f\"Total time for {num_requests} requests: {end_time - start_time:.2f} seconds\")\n", " return results\n", "\n", "async def main():\n", " \"\"\"Main function to execute the concurrency test.\"\"\"\n", " num_requests = 20 # Adjust as needed\n", " prompt = \"Write a short sentence.\"\n", "\n", " results = await run_concurrent_requests(num_requests, prompt)\n", "\n", " successful_requests = sum(1 for result in results if result is not None)\n", " print(f\"Successful requests: {successful_requests}/{num_requests}\")\n", "\n", "if __name__ == \"__main__\":\n", " asyncio.run(main())" ] }, { "cell_type": "code", "execution_count": 1, "id": "43c48edf-4482-484c-8cc3-41ecbfa291d9", "metadata": {}, "outputs": [], "source": [ "import asyncio\n", "import time\n", "from openai import AsyncOpenAI" ] }, { "cell_type": "code", "execution_count": 25, "id": "85cc6051-5154-4841-86ef-6a43882f7816", "metadata": {}, "outputs": [], "source": [ "model_used = \"meta/llama-3.1-8b-instruct\"\n", "max_tokens_used = 50\n", "num_requests_used = 20\n", "print_response = 1" ] }, { "cell_type": "code", "execution_count": 3, "id": "1308b886-5363-4c85-8834-8ddf53423a31", "metadata": {}, "outputs": [], "source": [ "PROMPT_LIST = [\n", " \"Write a short story about a robot who learns to love.\",\n", " \"Generate a poem about the beauty of a sunset.\",\n", " \"Explain the concept of quantum entanglement in simple terms.\",\n", " \"Describe the taste of a freshly baked apple pie.\",\n", " \"Write a funny dialogue between two talking animals.\",\n", " \"Suggest five creative uses for old newspapers.\",\n", " \"Compose a song about exploring a new city.\",\n", " \"What are the pros and cons of artificial intelligence?\",\n", " \"Write a haiku about a rainy day.\",\n", " \"Describe a futuristic transportation system.\",\n", " \"Tell me a joke about a cat.\",\n", " \"Explain the process of photosynthesis.\",\n", "]" ] }, { "cell_type": "code", "execution_count": 4, "id": "8892282b-6acd-4c99-9c89-b4a53d5d3102", "metadata": {}, "outputs": [], "source": [ "# Get these details from MLIS model deployment\n", "mlis_api_key = \"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE3NDQwMTcwNTAsImlzcyI6ImFpb2xpQGhwZS5jb20iLCJzdWIiOiI3YjU1NjY1My0xYWQzLTQyYzYtYmYyYi02N2I5NmU3ZDQ1MDIiLCJ1c2VyIjoiaGFyc2hhbC5wYXRpbC1nbWFpbC5jb20ifQ.p4XORb2TbRSlSJ5oI1Ooxr9bmPwraNM4rLChy9x0WHoxF9QLzLC64Te--m-CZBgAGM6ne-RbTtnFw0pLuP4jpkXiEdEEKNxVTN3b0VBuBJmQ4_b2azXPjSB8t_PMc1mtOOehhWmePnEhs5MJ5PmsshvVYmhF8niK8cVNYxDK9u-rOLRTE2SUS7O4Ln6R2OpzGYMEV6Q4ZUr5F3U76su7WfVXIpG3iDV8-DNXNC-RllR-V_ZX1N0-hcEDHINX0KlFMc9l7-JgOmpKjytDbqbrM1UEjREXENjD255bZnlcziCSwmJjhFsVOJzzryW8jLIdQarVtWS3mKr2g_tq8bhz2Q\"\n", "mlis_api_endpoint = \"https://llama-31-nim-predictor-jordan-nanos-hp-d222129e.hpepcai-ingress.pcai.hpecic.net/v1\"\n", "mlis_model = \"meta/llama-3.1-8b-instruct\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "21b9d551-fcc9-4994-97f1-0293709968c3", "metadata": {}, "outputs": [], "source": [ "mlis_api_key=\"eyJhbGciOiJSUzI1NiIsImtpZCI6IkdVRF93ZGprQW9LMjI1NlZfZUFLQUQ0TVFKaXVveDBpa0xLS0ZhbVdfNFEifQ.eyJhdWQiOlsiYXBpIiwiaXN0aW8tY2EiXSwiZXhwIjoxNzcyOTU3NDcyLCJpYXQiOjE3NzAzNjU0NzIsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwianRpIjoiYmNkZWI2NzItYWI1NS00YzI5LTliMmMtZDM4NDJkOWY3YWQ5Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJ1aSIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJpc3ZjLWVwLTE3NzAzNjU0NjY4OTQiLCJ1aWQiOiJhZTAzNjI1Yy02Yzg4LTQ0NTgtODdhMy1lNDkwYWEzNzEyZjgifX0sIm5iZiI6MTc3MDM2NTQ3Miwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50OnVpOmlzdmMtZXAtMTc3MDM2NTQ2Njg5NCJ9.SMJW10I7nK7PqThpWZNN1W5oVtCgtpl7bMAbLFvsOeEc_RxrgEH9f8Nnlfre7VDnE7YO0N3ZkMfsfSRwFH9_NS1w7pDAJWg1Qxlc4HODWz2lV9-inPRen5WuZUenw0LEAemFgs60Xj8Gy-BEWoNCZSKg0-29HgcCnN-a4kx4oqWjPKXx5A_NN6xk6Mu_AUjUgOpRtw_dEWhmzBWmU2Hwey3v3zx7RfUj5PCDZdBL9vwaDh65PVXJhgKlMKGywhLvI-jBITqjoPUXFHSKLct8so-iHVWSztzfFi4gkQ4GXiWETF04W7hhi4wnsZLlU0BPJI4HQOzZ5BiNJXYqLEFHlA\" \n", "mlis_api_endpoint=\"https://do-not-delete--gpt-oss-20b-predictor-hugo-hpe-com-7f9ace25.ai-application.hou-pcai.hpecic.net/v1\"\n", "mlis_model = \"openai/gpt-oss-20b\"" ] }, { "cell_type": "code", "execution_count": 5, "id": "5772721a-d04e-4c58-a192-f1e844d88f88", "metadata": {}, "outputs": [], "source": [ "#initialize OpenAI compatible client\n", "import openai\n", "import os\n", "\n", "# new\n", "from openai import OpenAI\n", "\n", "client = OpenAI(\n", " api_key= mlis_api_key, \n", " base_url= mlis_api_endpoint\n", ")" ] }, { "cell_type": "code", "execution_count": 12, "id": "c435097c-ee63-4bae-8331-e24b8137d71c", "metadata": {}, "outputs": [], "source": [ "# Ignore certificate checking - for self signed certs\n", "from openai import OpenAI\n", "import httpx\n", "httpx_client = httpx.Client(http2=True, verify=False)\n", "\n", "client = OpenAI(\n", " api_key=mlis_api_key,\n", " base_url= mlis_api_endpoint,\n", " http_client=httpx_client\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "id": "4c55489c-c9db-42b5-9d52-7428bf0dc13f", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "HTTP Request: POST https://do-not-delete--gpt-oss-20b-predictor-hugo-hpe-com-7f9ace25.ai-application.hou-pcai.hpecic.net/v1/chat/completions \"HTTP/1.1 200 OK\"\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Midnight hovered over the old oak room, its silver light painting the dusty floorboards. In the corner, a silver‑eyed cat named Mira stretched into a lazy, curved crescent of fur, her whiskers twitching at the scent of fresh rain. She was the guardian of the attic, so the house’s weary occupants spoke of her as both a muse and a mystery.\n", "\n", "One evening, a wooden box fell from a forgotten shelf, landing with a thud on the floor. Mira leapt, her paws silent, curiosity burning brighter than her whiskers. Inside lay an ornate key, its enamel faded but still gleaming. The key seemed to hum softly, inviting her to ask the attic’s hidden secrets.\n", "\n", "She nudged the key toward a loose plank. With a daring twist, she set the plank open, revealing a narrow, lintel‑raised tunnel that spiraled downwards, lit by a faint phosphorescent glow. Without hesitation, Mira’s green eyes widened, and she dashed into the darkness, her tiny claws echoing against stone.\n", "\n", "The attic breathed around her, remembering the old stories when a quiet, brave cat unlocked a world beyond, turning a simple night into a legendary adventure.\n" ] } ], "source": [ "# stream response with OpenAI API\n", "try:\n", " stream = client.chat.completions.create(\n", " model= mlis_model, \n", " messages=[\n", " {\"role\": \"user\", \"content\": \"Write a short story about cat in 200 words or less\"}\n", " ],\n", " stream=True # Enable streaming\n", " )\n", " for chunk in stream:\n", " if chunk.choices:\n", " content = chunk.choices[0].delta.content\n", " if content is not None:\n", " print(content, end=\"\", flush=True)\n", " print() # Add a newline at the end\n", "except openai.APIError as e:\n", " print(f\"An API error occurred: {e}\")" ] }, { "cell_type": "code", "execution_count": 21, "id": "a741d10f-6d1c-4093-9a19-13b940937512", "metadata": {}, "outputs": [], "source": [ "async def make_openai_request(client, prompt):\n", " \n", " # ignore prompt param and use random prompt\n", " import random\n", " random_integer = random.randint(0, len(PROMPT_LIST)-1)\n", " prompt = PROMPT_LIST[random_integer]\n", " \n", " \"\"\"Makes an asynchronous request to the OpenAI API.\"\"\"\n", " try:\n", " response = await client.completions.create(\n", " model=model_used, # or another suitable model\n", " prompt=prompt,\n", " max_tokens=max_tokens_used\n", " )\n", " if print_response:\n", " print(response)\n", " print(\"###################\")\n", " return response.choices[0].text.strip()\n", " except Exception as e:\n", " print(f\"Error during API call: {e}\")\n", " return None" ] }, { "cell_type": "code", "execution_count": 16, "id": "f4268686-840b-43d0-acb7-55bc1ac83961", "metadata": {}, "outputs": [], "source": [ "async def run_concurrent_requests(num_requests, prompt):\n", " \"\"\"Runs multiple OpenAI API requests concurrently.\"\"\"\n", " # Ignore certificate checking - for self signed certs\n", " from openai import OpenAI\n", " import httpx\n", " httpx_client = httpx.AsyncClient(http2=True, verify=False)\n", " \n", " client = AsyncOpenAI(\n", " api_key= mlis_api_key, \n", " base_url= mlis_api_endpoint,\n", " http_client=httpx_client\n", " )\n", " tasks = [make_openai_request(client, prompt) for _ in range(num_requests)]\n", " \n", " start_time = time.time()\n", " results = await asyncio.gather(*tasks)\n", " end_time = time.time()\n", "\n", " print(f\"Total time for {num_requests} requests: {end_time - start_time:.2f} seconds\")\n", " return results" ] }, { "cell_type": "code", "execution_count": 17, "id": "95ee15b9-c5c7-46d1-8514-859603e6e0ed", "metadata": {}, "outputs": [], "source": [ "async def main():\n", "\n", " \n", " \"\"\"Main function to execute the concurrency test.\"\"\"\n", " num_requests = num_requests_used # Adjust as needed\n", " \n", " prompt = \"Write any random short sentence different from before\"\n", " results = await run_concurrent_requests(num_requests, prompt)\n", "\n", " successful_requests = sum(1 for result in results if result is not None)\n", " print(f\"Successful requests: {successful_requests}/{num_requests}\")" ] }, { "cell_type": "code", "execution_count": 18, "id": "0a03ff13-18ea-49d4-94ba-028045350bfc", "metadata": {}, "outputs": [], "source": [ "#asyncio.run(main())" ] }, { "cell_type": "code", "execution_count": 19, "id": "878dedb9-b681-4874-8d6e-ca6406829c68", "metadata": {}, "outputs": [], "source": [ "# INFO WARNING ERROR CRITICAL\n", "# turn off printing of each HTTP request \n", "import logging\n", "logging.getLogger(\"httpx\").setLevel(logging.WARNING)" ] }, { "cell_type": "code", "execution_count": 26, "id": "e6050607-fa0e-4107-91b7-ab5c3614895b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total time for 10000 requests: 160.92 seconds\n", "Successful requests: 10000/10000\n" ] } ], "source": [ "# existing Asyncio loop\n", "loop = asyncio.get_event_loop()\n", "await loop.create_task(main())" ] }, { "cell_type": "code", "execution_count": null, "id": "a2fef8cf-b600-4b50-b962-a4f25672c5ab", "metadata": {}, "outputs": [], "source": [ "# test Inference Scaling\n", "model_used = \"meta/llama-3.1-8b-instruct\"\n", "max_tokens_used = 50\n", "num_requests_used = 10000\n", "print_response = 0" ] }, { "cell_type": "code", "execution_count": null, "id": "29c57c46-2300-4f37-8af4-e350644c5fb1", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.10" } }, "nbformat": 4, "nbformat_minor": 5 }