{ "cells": [ { "cell_type": "code", "execution_count": 64, "id": "a14125ea-7936-4c3c-a0dd-3f4a92379b6e", "metadata": { "tags": [] }, "outputs": [], "source": [ "# Imports\n", "import ray\n", "from ray.job_submission import JobSubmissionClient, JobStatus\n", "import time" ] }, { "cell_type": "code", "execution_count": 65, "id": "09da1f86-85f2-4593-a9bc-5273f215fb7d", "metadata": { "tags": [] }, "outputs": [], "source": [ "# Ray cluster information for connection\n", "ray_head_ip = \"kuberay-head-svc.kuberay.svc.cluster.local\"\n", "ray_head_port = 8265\n", "ray_address = f\"http://{ray_head_ip}:{ray_head_port}\"\n", "client = JobSubmissionClient(ray_address)" ] }, { "cell_type": "code", "execution_count": null, "id": "392e3e56-35b3-4a37-8686-a20eca18921b", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2024-09-13 13:24:32,153\tINFO dashboard_sdk.py:338 -- Uploading package gcs://_ray_pkg_8d335cfeea7b689e.zip.\n", "2024-09-13 13:24:32,154\tINFO packaging.py:530 -- Creating a file package for local directory './'.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Ray job submitted with job_id: raysubmit_hBHF3VY97pSqTWvk\n", "2024-09-13 06:24:34.492564: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", "2024-09-13 06:24:34.513902: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", "2024-09-13 06:24:34.520338: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", "2024-09-13 06:24:34.536575: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", "2024-09-13 06:24:38.755351: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", "Num GPUs Available: 1\n", "TensorFlow will run on GPU.\n", "2024-09-13 06:24:43.847637: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 6417 MB memory: -> device: 0, name: Tesla P4, pci bus id: 0000:12:00.0, compute capability: 6.1\n", "input: (_Arg): /job:localhost/replica:0/task:0/device:GPU:0\n", "2024-09-13 06:24:43.868656: I tensorflow/core/common_runtime/placer.cc:162] input: (_Arg): /job:localhost/replica:0/task:0/device:GPU:0\n", "_EagerConst: (_EagerConst): /job:localhost/replica:0/task:0/device:GPU:0\n", "2024-09-13 06:24:43.868836: I tensorflow/core/common_runtime/placer.cc:162] _EagerConst: (_EagerConst): /job:localhost/replica:0/task:0/device:GPU:0\n", "output_RetVal: (_Retval): /job:localhost/replica:0/task:0/device:GPU:0\n", "2024-09-13 06:24:43.868938: I tensorflow/core/common_runtime/placer.cc:162] output_RetVal: (_Retval): /job:localhost/replica:0/task:0/device:GPU:0\n", "2024-09-13 06:24:43.928017: I tensorflow/core/common_runtime/eager/execute.cc:1746] Executing op _EagerConst in device /job:localhost/replica:0/task:0/device:GPU:0\n", "x: (_Arg): /job:localhost/replica:0/task:0/device:GPU:0\n", "2024-09-13 06:24:43.929286: I tensorflow/core/common_runtime/placer.cc:162] x: (_Arg): /job:localhost/replica:0/task:0/device:GPU:0\n", "Square: (Square): /job:localhost/replica:0/task:0/device:GPU:0\n", "2024-09-13 06:24:43.929373: I tensorflow/core/common_runtime/placer.cc:162] Square: (Square): /job:localhost/replica:0/task:0/device:GPU:0\n", "y_RetVal: (_Retval): /job:localhost/replica:0/task:0/device:GPU:0\n", "2024-09-13 06:24:43.929416: I tensorflow/core/common_runtime/placer.cc:162] y_RetVal: (_Retval): /job:localhost/replica:0/task:0/device:GPU:0\n", "2024-09-13 06:24:43.929977: I tensorflow/core/common_runtime/eager/execute.cc:1746] Executing op Square in device /job:localhost/replica:0/task:0/device:GPU:0\n", "Original data sample: [ 1 2 3 4 5 6 7 8 9 10]\n", "Squared data sample: [ 1. 4. 9. 16. 25. 36. 49. 64. 81. 100.]\n", "\n" ] } ], "source": [ "# Submit Ray job using JobSubmissionClient\n", "job_id = client.submit_job(\n", " entrypoint=\"python ray-gpu-example.py\",\n", " runtime_env={\n", " \"working_dir\": \"./\"\n", " },\n", " entrypoint_num_gpus = 1,\n", " entrypoint_num_cpus = 1\n", ")\n", "\n", "print(f\"Ray job submitted with job_id: {job_id}\")\n", "\n", "# Waiting for Ray to finish the job and print the result\n", "while True:\n", " status = client.get_job_status(job_id)\n", " if status in [ray.job_submission.JobStatus.RUNNING, ray.job_submission.JobStatus.PENDING]:\n", " time.sleep(5)\n", " else:\n", " break\n", "try:\n", " logs = client.get_job_logs(job_id) \n", " print(logs)\n", "except RuntimeError as e:\n", " print(f\"Failed to get job logs, please check logs on ray dashboard \")" ] }, { "cell_type": "code", "execution_count": null, "id": "4154ccbd-d86e-4517-a697-1393115b29fb", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Ray", "language": "python", "name": "ray" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.6" } }, "nbformat": 4, "nbformat_minor": 5 }