{ "cells": [ { "cell_type": "markdown", "id": "9c079d82", "metadata": {}, "source": [ "## Running Basic Tune Experiments\n", "The most common way to use Tune is also the simplest: as a parallel experiment runner. As a general example, let’s consider executing N independent model training trials using Tune as a simple grid sweep." ] }, { "cell_type": "code", "execution_count": 16, "id": "6f3f53a9-d9d0-46c7-aecd-954d33ac7017", "metadata": {}, "outputs": [], "source": [ "# Imports\n", "import ray\n", "from ray.job_submission import JobSubmissionClient\n", "import time" ] }, { "cell_type": "code", "execution_count": 17, "id": "10bafa3a-1edf-439e-9d2e-39e9b22ff97c", "metadata": { "tags": [] }, "outputs": [], "source": [ "# Ray cluster information for connection\n", "ray_head_ip = \"kuberay-head-svc.kuberay.svc.cluster.local\"\n", "ray_head_port = 8265\n", "ray_address = f\"http://{ray_head_ip}:{ray_head_port}\"\n", "client = JobSubmissionClient(ray_address)" ] }, { "cell_type": "markdown", "id": "7e5dc8a7-c7bc-4d1a-9dc9-e466be0b5ba0", "metadata": { "tags": [] }, "source": [ "##### If job logs are not fetched here, please check it on Ray dashboard." ] }, { "cell_type": "code", "execution_count": 18, "id": "5dfb8001-5083-4973-be01-50fdaf2a6147", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2024-07-02 13:39:57,960\tINFO dashboard_sdk.py:338 -- Uploading package gcs://_ray_pkg_d4b4077d81d2f8d0.zip.\n", "2024-07-02 13:39:57,961\tINFO packaging.py:530 -- Creating a file package for local directory './'.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Ray job submitted with job_id: raysubmit_HhThUqFDcK83c5ym\n", "2024-07-02 06:39:58,006\tINFO job_manager.py:530 -- Runtime env is setting up.\n", "2024-07-02 06:40:28,657\tINFO worker.py:1433 -- Using address kuberay-head-svc.kuberay.svc.cluster.local:6379 set in the environment variable RAY_ADDRESS\n", "2024-07-02 06:40:28,658\tINFO worker.py:1568 -- Connecting to existing Ray cluster at address: kuberay-head-svc.kuberay.svc.cluster.local:6379...\n", "2024-07-02 06:40:28,721\tINFO worker.py:1744 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttp://10.224.1.185:8265 \u001b[39m\u001b[22m\n", "2024-07-02 06:40:29,250\tINFO tune.py:253 -- Initializing Ray automatically. For cluster usage or custom Ray initialization, call `ray.init(...)` before `Tuner(...)`.\n", "╭────────────────────────────────────────────────────────────────────╮\n", "│ Configuration for experiment train_model_2024-07-02_06-40-28 │\n", "├────────────────────────────────────────────────────────────────────┤\n", "│ Search algorithm BasicVariantGenerator │\n", "│ Scheduler FIFOScheduler │\n", "│ Number of trials 5 │\n", "╰────────────────────────────────────────────────────────────────────╯\n", "\n", "View detailed results here: /home/ray/ray_results/train_model_2024-07-02_06-40-28\n", "To visualize your results with TensorBoard, run: `tensorboard --logdir /tmp/ray/session_2024-07-01_07-02-52_634938_8/artifacts/2024-07-02_06-40-29/train_model_2024-07-02_06-40-28/driver_artifacts`\n", "\n", "Trial status: 5 PENDING\n", "Current time: 2024-07-02 06:40:31. Total running time: 0s\n", "Logical resource usage: 0/3 CPUs, 0/0 GPUs\n", "╭─────────────────────────────────────────────────╮\n", "│ Trial name status model_id │\n", "├─────────────────────────────────────────────────┤\n", "│ train_model_a551a_00000 PENDING model_0 │\n", "│ train_model_a551a_00001 PENDING model_1 │\n", "│ train_model_a551a_00002 PENDING model_2 │\n", "│ train_model_a551a_00003 PENDING model_3 │\n", "│ train_model_a551a_00004 PENDING model_4 │\n", "╰─────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00001 started with configuration:\n", "╭──────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00001 config │\n", "├──────────────────────────────────────────────────┤\n", "│ model_id model_1 │\n", "╰──────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00001 completed after 1 iterations at 2024-07-02 06:40:33. Total running time: 2s\n", "╭───────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00001 result │\n", "├───────────────────────────────────────────────────┤\n", "│ checkpoint_dir_name │\n", "│ time_this_iter_s 0.00038 │\n", "│ time_total_s 0.00038 │\n", "│ training_iteration 1 │\n", "│ other_data Ellipsis │\n", "│ score model_1 │\n", "╰───────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00000 started with configuration:\n", "╭──────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00000 config │\n", "├──────────────────────────────────────────────────┤\n", "│ model_id model_0 │\n", "╰──────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00000 completed after 1 iterations at 2024-07-02 06:40:33. Total running time: 2s\n", "╭───────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00000 result │\n", "├───────────────────────────────────────────────────┤\n", "│ checkpoint_dir_name │\n", "│ time_this_iter_s 0.00031 │\n", "│ time_total_s 0.00031 │\n", "│ training_iteration 1 │\n", "│ other_data Ellipsis │\n", "│ score model_0 │\n", "╰───────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00003 started with configuration:\n", "╭──────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00003 config │\n", "├──────────────────────────────────────────────────┤\n", "│ model_id model_3 │\n", "╰──────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00003 completed after 1 iterations at 2024-07-02 06:40:37. Total running time: 6s\n", "╭───────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00003 result │\n", "├───────────────────────────────────────────────────┤\n", "│ checkpoint_dir_name │\n", "│ time_this_iter_s 0.00035 │\n", "│ time_total_s 0.00035 │\n", "│ training_iteration 1 │\n", "│ other_data Ellipsis │\n", "│ score model_3 │\n", "╰───────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00002 started with configuration:\n", "╭──────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00002 config │\n", "├──────────────────────────────────────────────────┤\n", "│ model_id model_2 │\n", "╰──────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00002 completed after 1 iterations at 2024-07-02 06:40:37. Total running time: 6s\n", "╭───────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00002 result │\n", "├───────────────────────────────────────────────────┤\n", "│ checkpoint_dir_name │\n", "│ time_this_iter_s 0.00035 │\n", "│ time_total_s 0.00035 │\n", "│ training_iteration 1 │\n", "│ other_data Ellipsis │\n", "│ score model_2 │\n", "╰───────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00004 started with configuration:\n", "╭──────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00004 config │\n", "├──────────────────────────────────────────────────┤\n", "│ model_id model_4 │\n", "╰──────────────────────────────────────────────────╯\n", "\n", "Trial train_model_a551a_00004 completed after 1 iterations at 2024-07-02 06:40:40. Total running time: 9s\n", "╭───────────────────────────────────────────────────╮\n", "│ Trial train_model_a551a_00004 result │\n", "├───────────────────────────────────────────────────┤\n", "│ checkpoint_dir_name │\n", "│ time_this_iter_s 0.00031 │\n", "│ time_total_s 0.00031 │\n", "│ training_iteration 1 │\n", "│ other_data Ellipsis │\n", "│ score model_4 │\n", "╰───────────────────────────────────────────────────╯\n", "2024-07-02 06:40:40,360\tINFO tune.py:1009 -- Wrote the latest version of all result files and experiment state to '/home/ray/ray_results/train_model_2024-07-02_06-40-28' in 0.0376s.\n", "\n", "Trial status: 5 TERMINATED\n", "Current time: 2024-07-02 06:40:40. Total running time: 9s\n", "Logical resource usage: 1.0/3 CPUs, 0/0 GPUs\n", "╭───────────────────────────────────────────────────────────────────────────────╮\n", "│ Trial name status model_id iter total time (s) │\n", "├───────────────────────────────────────────────────────────────────────────────┤\n", "│ train_model_a551a_00000 TERMINATED model_0 1 0.000306845 │\n", "│ train_model_a551a_00001 TERMINATED model_1 1 0.000383615 │\n", "│ train_model_a551a_00002 TERMINATED model_2 1 0.000350237 │\n", "│ train_model_a551a_00003 TERMINATED model_3 1 0.000349522 │\n", "│ train_model_a551a_00004 TERMINATED model_4 1 0.000305414 │\n", "╰───────────────────────────────────────────────────────────────────────────────╯\n", "\n", "Result(\n", " metrics={'score': 'model_0', 'other_data': Ellipsis},\n", " path='/home/ray/ray_results/train_model_2024-07-02_06-40-28/train_model_a551a_00000_0_model_id=model_0_2024-07-02_06-40-31',\n", " filesystem='local',\n", " checkpoint=None\n", ")\n", "Result(\n", " metrics={'score': 'model_1', 'other_data': Ellipsis},\n", " path='/home/ray/ray_results/train_model_2024-07-02_06-40-28/train_model_a551a_00001_1_model_id=model_1_2024-07-02_06-40-31',\n", " filesystem='local',\n", " checkpoint=None\n", ")\n", "Result(\n", " metrics={'score': 'model_2', 'other_data': Ellipsis},\n", " path='/home/ray/ray_results/train_model_2024-07-02_06-40-28/train_model_a551a_00002_2_model_id=model_2_2024-07-02_06-40-31',\n", " filesystem='local',\n", " checkpoint=None\n", ")\n", "\n" ] } ], "source": [ "# Submit Ray job using JobSubmissionClient\n", "job_id = client.submit_job(\n", " entrypoint=\"python independent-tune-trials-example.py\",\n", " runtime_env={\n", " \"working_dir\": \"./\",\n", " },\n", " entrypoint_num_cpus=1\n", ")\n", "\n", "print(f\"Ray job submitted with job_id: {job_id}\")\n", "\n", "# Waiting for Ray to finish the job and print the result\n", "while True:\n", " status = client.get_job_status(job_id)\n", " if status in [ray.job_submission.JobStatus.RUNNING, ray.job_submission.JobStatus.PENDING]:\n", " time.sleep(5)\n", " else:\n", " break\n", "try:\n", " logs = client.get_job_logs(job_id) \n", " print(logs)\n", "except RuntimeError as e:\n", " print(f\"Failed to get job logs, please check logs on ray dashboard \")" ] }, { "cell_type": "code", "execution_count": null, "id": "04a2aa18-1cad-4f37-926a-57107c1ca807", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Ray", "language": "python", "name": "ray" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.6" } }, "nbformat": 4, "nbformat_minor": 5 }