{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "734317ee-0a0a-4480-b5c3-573633d873ae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import boto3\n",
    "import logging\n",
    "from botocore.exceptions import NoCredentialsError, ClientError\n",
    "\n",
    "# Configure logging\n",
    "logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')\n",
    "\n",
    "def ensure_bucket_exists(s3_client, bucket_name):\n",
    "    \"\"\"\n",
    "    Checks if the S3 bucket exists and creates it if it does not.\n",
    "    This version is simplified for generic S3 compatibility.\n",
    "\n",
    "    Args:\n",
    "        s3_client: The initialized Boto3 S3 client configured with the endpoint URL.\n",
    "        bucket_name (str): The name of the S3 bucket.\n",
    "    \n",
    "    Returns:\n",
    "        bool: True if the bucket exists or was successfully created, False otherwise.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        # Check if the bucket exists using head_bucket\n",
    "        s3_client.head_bucket(Bucket=bucket_name)\n",
    "        logging.info(f\"Bucket '{bucket_name}' already exists.\")\n",
    "        return True\n",
    "    except ClientError as e:\n",
    "        error_code = e.response['Error']['Code']\n",
    "        \n",
    "        # 404 means the bucket does not exist\n",
    "        if error_code == '404':\n",
    "            logging.info(f\"Bucket '{bucket_name}' does not exist. Attempting creation...\")\n",
    "            \n",
    "            try:\n",
    "                # Simplified bucket creation for generic S3 services\n",
    "                s3_client.create_bucket(Bucket=bucket_name)\n",
    "                \n",
    "                logging.info(f\"Bucket '{bucket_name}' successfully created.\")\n",
    "                return True\n",
    "            except ClientError as create_e:\n",
    "                 # Handle specific errors during creation (e.g., name conflict, permission)\n",
    "                logging.error(f\"Failed to create bucket '{bucket_name}': {create_e}\")\n",
    "                return False\n",
    "        \n",
    "        # 403 means Forbidden (Permission Denied) or other access issue\n",
    "        elif error_code in ['403', 'Forbidden']:\n",
    "            logging.error(f\"Access denied. Cannot check or create bucket '{bucket_name}'. Check your keys and permissions.\")\n",
    "            return False\n",
    "        \n",
    "        else:\n",
    "            logging.error(f\"Error checking bucket existence for '{bucket_name}': {e}\")\n",
    "            return False\n",
    "    except Exception as e:\n",
    "        logging.error(f\"An unexpected error occurred during bucket check/creation: {e}\")\n",
    "        return False\n",
    "\n",
    "def upload_directory_to_s3(local_directory, bucket_name, s3_prefix='', endpoint_url=None, aws_access_key_id=None, aws_secret_access_key=None):\n",
    "    \"\"\"\n",
    "    Recursively uploads files from a local directory to an S3-compatible bucket,\n",
    "    maintaining the directory structure, and ensuring the bucket exists first.\n",
    "\n",
    "    Args:\n",
    "        local_directory (str): The path to the local directory to upload.\n",
    "        bucket_name (str): The name of the S3 bucket.\n",
    "        s3_prefix (str): An optional S3 prefix (folder path) to upload files into.\n",
    "        endpoint_url (str, optional): The custom endpoint URL for the S3 provider.\n",
    "        aws_access_key_id (str, optional): The access key ID.\n",
    "        aws_secret_access_key (str, optional): The secret access key.\n",
    "    \"\"\"\n",
    "    # Normalize paths\n",
    "    local_directory = os.path.abspath(local_directory)\n",
    "    if not os.path.isdir(local_directory):\n",
    "        logging.error(f\"Local directory not found: {local_directory}\")\n",
    "        return\n",
    "\n",
    "    # Ensure s3_prefix ends with a slash if it's not empty, unless it's the root.\n",
    "    if s3_prefix and not s3_prefix.endswith('/'):\n",
    "        s3_prefix += '/'\n",
    "\n",
    "    try:\n",
    "        # Initialize S3 client using explicit configuration for generic S3 compatibility\n",
    "        s3_client = boto3.client(\n",
    "            's3', \n",
    "            endpoint_url=endpoint_url,\n",
    "            # Boto3 still expects these parameter names, even for generic S3\n",
    "            aws_access_key_id=aws_access_key_id,\n",
    "            aws_secret_access_key=aws_secret_access_key,\n",
    "            # A region is still required by boto3, but for generic S3, the value often doesn't matter\n",
    "            region_name='us-east-1' \n",
    "        )\n",
    "        \n",
    "        # --- Check and create bucket if necessary ---\n",
    "        if not ensure_bucket_exists(s3_client, bucket_name):\n",
    "            logging.error(\"Failed to ensure bucket exists. Aborting upload.\")\n",
    "            return\n",
    "        \n",
    "        logging.info(f\"Starting upload from '{local_directory}' to s3://{bucket_name}/{s3_prefix}\")\n",
    "        \n",
    "        # Walk through the local directory structure\n",
    "        for root, dirs, files in os.walk(local_directory):\n",
    "            # Calculate the relative path from the root directory passed to the function\n",
    "            relative_path = os.path.relpath(root, local_directory)\n",
    "\n",
    "            for file_name in files:\n",
    "                local_file_path = os.path.join(root, file_name)\n",
    "\n",
    "                # Construct the S3 key\n",
    "                if relative_path == \".\":\n",
    "                    # If we are in the root of the source directory, don't include '.'\n",
    "                    s3_key = s3_prefix + file_name\n",
    "                else:\n",
    "                    # Replace OS path separator (\\ on Windows) with S3 forward slash\n",
    "                    s3_key = s3_prefix + os.path.join(relative_path, file_name).replace(os.path.sep, '/')\n",
    "\n",
    "                try:\n",
    "                    # Upload the file\n",
    "                    # The extra arguments (endpoint_url, etc.) are only passed during client initialization\n",
    "                    s3_client.upload_file(local_file_path, bucket_name, s3_key)\n",
    "                    logging.info(f\"Successfully uploaded: {local_file_path} -> s3://{bucket_name}/{s3_key}\")\n",
    "                \n",
    "                except ClientError as e:\n",
    "                    logging.error(f\"Error uploading {local_file_path}: {e}\")\n",
    "                \n",
    "                except Exception as e:\n",
    "                    logging.error(f\"An unexpected error occurred during file upload: {e}\")\n",
    "\n",
    "        logging.info(\"Upload process completed.\")\n",
    "\n",
    "    except Exception as e:\n",
    "        # Catch initialization errors if keys or endpoint are missing/invalid\n",
    "        logging.error(f\"Failed to initialize S3 client or connect: {e}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "db6c3b83-31de-4ce7-96fb-3b003a49cf0c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Successfully read access key from /etc/secrets/ezua/.auth_token.\n",
      "Bucket 's3models' does not exist. Attempting creation...\n",
      "Bucket 's3models' successfully created.\n",
      "Starting upload from '/mnt/shared/harshal-util/models' to s3://s3models/\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/.agent_harnesses.json -> s3://s3models/.agent_harnesses.json\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/.check_for_skill_update_done -> s3://s3models/.check_for_skill_update_done\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/.check_for_update_done -> s3://s3models/.check_for_update_done\n",
      "Failed to parse headers (url=http://local-s3-service.ezdata-system.svc.cluster.local:30000/s3models/hub/CACHEDIR.TAG): [MissingHeaderBodySeparatorDefect()], unparsed data: 'HTTP/1.1 200 OK\\r\\ncontent-length: 0\\r\\ndate: Tue, 15 Sep 2026 14:45:29 GMT\\r\\netag: \"7610c3f0a61e99809814719d9712f278\"\\r\\nserver: envoy\\r\\nstrict-transport-security: max-age=86400\\r\\nx-amz-id-2: 22111000864ae\\r\\nx-amz-request-id: 22111000864ae\\r\\nx-amz-server-side-encryption: aws:kms\\r\\nx-amz-server-side-encryption-customer-algorithm: AES256\\r\\nx-content-type-options: nosniff\\r\\nx-frame-options: SAMEORIGIN\\r\\nx-envoy-upstream-service-time: 5\\r\\n\\r\\n'\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.11/site-packages/urllib3/connectionpool.py\", line 488, in _make_request\n",
      "    assert_header_parsing(httplib_response.msg)\n",
      "  File \"/opt/conda/lib/python3.11/site-packages/urllib3/util/response.py\", line 91, in assert_header_parsing\n",
      "    raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)\n",
      "urllib3.exceptions.HeaderParsingError: [MissingHeaderBodySeparatorDefect()], unparsed data: 'HTTP/1.1 200 OK\\r\\ncontent-length: 0\\r\\ndate: Tue, 15 Sep 2026 14:45:29 GMT\\r\\netag: \"7610c3f0a61e99809814719d9712f278\"\\r\\nserver: envoy\\r\\nstrict-transport-security: max-age=86400\\r\\nx-amz-id-2: 22111000864ae\\r\\nx-amz-request-id: 22111000864ae\\r\\nx-amz-server-side-encryption: aws:kms\\r\\nx-amz-server-side-encryption-customer-algorithm: AES256\\r\\nx-content-type-options: nosniff\\r\\nx-frame-options: SAMEORIGIN\\r\\nx-envoy-upstream-service-time: 5\\r\\n\\r\\n'\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/CACHEDIR.TAG -> s3://s3models/hub/CACHEDIR.TAG\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/02ee80b6196926a5ad790a004d9efd6ab1ba6542.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/02ee80b6196926a5ad790a004d9efd6ab1ba6542.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/09d433f650646834a83c580877bd60c6d1f88f7755305c12576b5c7058f9af15.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/09d433f650646834a83c580877bd60c6d1f88f7755305c12576b5c7058f9af15.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/0bb6fd75b3ad2fe988565929f329945262c2814e.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/0bb6fd75b3ad2fe988565929f329945262c2814e.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/0fd8120f1c6acddc268ebc2583058efaf699a771.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/0fd8120f1c6acddc268ebc2583058efaf699a771.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/2b1879f356aed350030bb40eb45ad362c89d9891096f79a3ab323d3ba5607668.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/2b1879f356aed350030bb40eb45ad362c89d9891096f79a3ab323d3ba5607668.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/5cc5f00a5b203e90a27a3bd60d1ec393b07971e8.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/5cc5f00a5b203e90a27a3bd60d1ec393b07971e8.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/81ebb55902285e8dd5804ccf423d17ffb2a622ee.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/81ebb55902285e8dd5804ccf423d17ffb2a622ee.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/82e9d31979e92ab929cd544440f129d9ecd797b69e327f80f17e1c50d5551b55.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/82e9d31979e92ab929cd544440f129d9ecd797b69e327f80f17e1c50d5551b55.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/92ecfe1a2414458b4821ac8c13cf8cb70aed66b5eea8dc5ad9eeb4ff309d6d7b.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/92ecfe1a2414458b4821ac8c13cf8cb70aed66b5eea8dc5ad9eeb4ff309d6d7b.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/a6344aac8c09253b3b630fb776ae94478aa0275b.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/a6344aac8c09253b3b630fb776ae94478aa0275b.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/a7c3ca16cee30425ed6ad841a809590f2bcbf290.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/a7c3ca16cee30425ed6ad841a809590f2bcbf290.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/ab33d910f405204e5d388bc3521503584800461dc96808e287821dd451c1edac.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/ab33d910f405204e5d388bc3521503584800461dc96808e287821dd451c1edac.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/bbd5630a05b65c1a8b25141bd11ec44844107d58.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/bbd5630a05b65c1a8b25141bd11ec44844107d58.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/cc7276afd599de091142c6ed3005faf8a74aa257.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/cc7276afd599de091142c6ed3005faf8a74aa257.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/db88166e2bc4c799fd5d1ae643b75e84d03ee70e.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/db88166e2bc4c799fd5d1ae643b75e84d03ee70e.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/f1131204e79d0c09d2bac93f11569a8a655d68ba.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/f1131204e79d0c09d2bac93f11569a8a655d68ba.lock\n",
      "Successfully uploaded: /mnt/shared/harshal-util/models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/fc1cdddd6bfa91128d6e94ee73d0ce62bfcdb7af29e978ddcab30c66ae9ea7fa.lock -> s3://s3models/hub/.locks/models--meta-llama--Llama-3.1-8B-Instruct/fc1cdddd6bfa91128d6e94ee73d0ce62bfcdb7af29e978ddcab30c66ae9ea7fa.lock\n",
      "Failed to parse headers (url=http://local-s3-service.ezdata-system.svc.cluster.local:30000/s3models/hub/models--meta-llama--Llama-3.1-8B-Instruct/blobs/02ee80b6196926a5ad790a004d9efd6ab1ba6542): [MissingHeaderBodySeparatorDefect()], unparsed data: 'HTTP/1.1 200 OK\\r\\ncontent-length: 0\\r\\ndate: Tue, 15 Sep 2026 14:46:30 GMT\\r\\netag: \"ff9bb51206c5b33aa9385a1a371294b6\"\\r\\nserver: envoy\\r\\nstrict-transport-security: max-age=86400\\r\\nx-amz-id-2: 22111000864cf\\r\\nx-amz-request-id: 22111000864cf\\r\\nx-amz-server-side-encryption: aws:kms\\r\\nx-amz-server-side-encryption-customer-algorithm: AES256\\r\\nx-content-type-options: nosniff\\r\\nx-frame-options: SAMEORIGIN\\r\\nx-envoy-upstream-service-time: 5\\r\\n\\r\\n'\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.11/site-packages/urllib3/connectionpool.py\", line 488, in _make_request\n",
      "    assert_header_parsing(httplib_response.msg)\n",
      "  File \"/opt/conda/lib/python3.11/site-packages/urllib3/util/response.py\", line 91, in assert_header_parsing\n",
      "    raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)\n",
      "urllib3.exceptions.HeaderParsingError: [MissingHeaderBodySeparatorDefect()], unparsed data: 'HTTP/1.1 200 OK\\r\\ncontent-length: 0\\r\\ndate: Tue, 15 Sep 2026 14:46:30 GMT\\r\\netag: \"ff9bb51206c5b33aa9385a1a371294b6\"\\r\\nserver: envoy\\r\\nstrict-transport-security: max-age=86400\\r\\nx-amz-id-2: 22111000864cf\\r\\nx-amz-request-id: 22111000864cf\\r\\nx-amz-server-side-encryption: aws:kms\\r\\nx-amz-server-side-encryption-customer-algorithm: AES256\\r\\nx-content-type-options: nosniff\\r\\nx-frame-options: SAMEORIGIN\\r\\nx-envoy-upstream-service-time: 5\\r\\n\\r\\n'\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 59\u001b[0m\n\u001b[1;32m     52\u001b[0m             \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDummy structure created.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     55\u001b[0m         \u001b[38;5;66;03m# Call the main upload function\u001b[39;00m\n\u001b[1;32m     56\u001b[0m         \u001b[38;5;66;03m# WARNING: This will attempt to connect and upload. Ensure your configuration \u001b[39;00m\n\u001b[1;32m     57\u001b[0m         \u001b[38;5;66;03m# (ENDPOINT_URL, KEYS, and TARGET_BUCKET) is correct.\u001b[39;00m\n\u001b[1;32m     58\u001b[0m         \u001b[38;5;66;03m# UNCOMMENT THE LINE BELOW TO EXECUTE THE UPLOAD AND BUCKET CREATION\u001b[39;00m\n\u001b[0;32m---> 59\u001b[0m         \u001b[43mupload_directory_to_s3\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     60\u001b[0m \u001b[43m            \u001b[49m\u001b[43mSOURCE_DIR\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     61\u001b[0m \u001b[43m            \u001b[49m\u001b[43mTARGET_BUCKET\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     62\u001b[0m \u001b[43m            \u001b[49m\u001b[43mTARGET_PREFIX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     63\u001b[0m \u001b[43m            \u001b[49m\u001b[43mENDPOINT_URL\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     64\u001b[0m \u001b[43m            \u001b[49m\u001b[43mAWS_ACCESS_KEY_ID\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     65\u001b[0m \u001b[43m            \u001b[49m\u001b[43mAWS_SECRET_ACCESS_KEY\u001b[49m\n\u001b[1;32m     66\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     68\u001b[0m \u001b[38;5;66;03m#         # For demonstration purposes, just show the paths that would be used:\u001b[39;00m\n\u001b[1;32m     69\u001b[0m \u001b[38;5;66;03m#         print(\"\\n--- Dry Run Path Mapping (Uncomment the function call above to execute) ---\")\u001b[39;00m\n\u001b[1;32m     70\u001b[0m \u001b[38;5;66;03m#         dry_run_directory = SOURCE_DIR\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     88\u001b[0m \u001b[38;5;66;03m#         print(\"If executed, the script will first ensure the bucket exists on your generic S3 provider, then upload the files.\")\u001b[39;00m\n\u001b[1;32m     89\u001b[0m \u001b[38;5;66;03m#         print(\"Be sure to set ENDPOINT_URL, AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY in the configuration block.\")\u001b[39;00m\n",
      "Cell \u001b[0;32mIn[1], line 117\u001b[0m, in \u001b[0;36mupload_directory_to_s3\u001b[0;34m(local_directory, bucket_name, s3_prefix, endpoint_url, aws_access_key_id, aws_secret_access_key)\u001b[0m\n\u001b[1;32m    112\u001b[0m     s3_key \u001b[38;5;241m=\u001b[39m s3_prefix \u001b[38;5;241m+\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(relative_path, file_name)\u001b[38;5;241m.\u001b[39mreplace(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39msep, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m    114\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    115\u001b[0m     \u001b[38;5;66;03m# Upload the file\u001b[39;00m\n\u001b[1;32m    116\u001b[0m     \u001b[38;5;66;03m# The extra arguments (endpoint_url, etc.) are only passed during client initialization\u001b[39;00m\n\u001b[0;32m--> 117\u001b[0m     \u001b[43ms3_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupload_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlocal_file_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbucket_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43ms3_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    118\u001b[0m     logging\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSuccessfully uploaded: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlocal_file_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m -> s3://\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mbucket_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00ms3_key\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    120\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ClientError \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/boto3/s3/inject.py:145\u001b[0m, in \u001b[0;36mupload_file\u001b[0;34m(self, Filename, Bucket, Key, ExtraArgs, Callback, Config)\u001b[0m\n\u001b[1;32m    110\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Upload a file to an S3 object.\u001b[39;00m\n\u001b[1;32m    111\u001b[0m \n\u001b[1;32m    112\u001b[0m \u001b[38;5;124;03mUsage::\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    142\u001b[0m \u001b[38;5;124;03m    transfer.\u001b[39;00m\n\u001b[1;32m    143\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    144\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m S3Transfer(\u001b[38;5;28mself\u001b[39m, Config) \u001b[38;5;28;01mas\u001b[39;00m transfer:\n\u001b[0;32m--> 145\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtransfer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupload_file\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    146\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfilename\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mFilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbucket\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mBucket\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    148\u001b[0m \u001b[43m        \u001b[49m\u001b[43mkey\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mKey\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    149\u001b[0m \u001b[43m        \u001b[49m\u001b[43mextra_args\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mExtraArgs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    150\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcallback\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mCallback\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    151\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/boto3/s3/transfer.py:372\u001b[0m, in \u001b[0;36mS3Transfer.upload_file\u001b[0;34m(self, filename, bucket, key, callback, extra_args)\u001b[0m\n\u001b[1;32m    368\u001b[0m future \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_manager\u001b[38;5;241m.\u001b[39mupload(\n\u001b[1;32m    369\u001b[0m     filename, bucket, key, extra_args, subscribers\n\u001b[1;32m    370\u001b[0m )\n\u001b[1;32m    371\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 372\u001b[0m     \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    373\u001b[0m \u001b[38;5;66;03m# If a client error was raised, add the backwards compatibility layer\u001b[39;00m\n\u001b[1;32m    374\u001b[0m \u001b[38;5;66;03m# that raises a S3UploadFailedError. These specific errors were only\u001b[39;00m\n\u001b[1;32m    375\u001b[0m \u001b[38;5;66;03m# ever thrown for upload_parts but now can be thrown for any related\u001b[39;00m\n\u001b[1;32m    376\u001b[0m \u001b[38;5;66;03m# client error.\u001b[39;00m\n\u001b[1;32m    377\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ClientError \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/s3transfer/futures.py:106\u001b[0m, in \u001b[0;36mTransferFuture.result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    104\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    105\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcancel()\n\u001b[0;32m--> 106\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\n",
      "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/s3transfer/futures.py:103\u001b[0m, in \u001b[0;36mTransferFuture.result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     98\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mresult\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m     99\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    100\u001b[0m         \u001b[38;5;66;03m# Usually the result() method blocks until the transfer is done,\u001b[39;00m\n\u001b[1;32m    101\u001b[0m         \u001b[38;5;66;03m# however if a KeyboardInterrupt is raised we want want to exit\u001b[39;00m\n\u001b[1;32m    102\u001b[0m         \u001b[38;5;66;03m# out of this and propagate the exception.\u001b[39;00m\n\u001b[0;32m--> 103\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_coordinator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    104\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    105\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcancel()\n",
      "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/s3transfer/futures.py:259\u001b[0m, in \u001b[0;36mTransferCoordinator.result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    249\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Waits until TransferFuture is done and returns the result\u001b[39;00m\n\u001b[1;32m    250\u001b[0m \n\u001b[1;32m    251\u001b[0m \u001b[38;5;124;03mIf the TransferFuture succeeded, it will return the result. If the\u001b[39;00m\n\u001b[1;32m    252\u001b[0m \u001b[38;5;124;03mTransferFuture failed, it will raise the exception associated to the\u001b[39;00m\n\u001b[1;32m    253\u001b[0m \u001b[38;5;124;03mfailure.\u001b[39;00m\n\u001b[1;32m    254\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    255\u001b[0m \u001b[38;5;66;03m# Doing a wait() with no timeout cannot be interrupted in python2 but\u001b[39;00m\n\u001b[1;32m    256\u001b[0m \u001b[38;5;66;03m# can be interrupted in python3 so we just wait with the largest\u001b[39;00m\n\u001b[1;32m    257\u001b[0m \u001b[38;5;66;03m# possible value integer value, which is on the scale of billions of\u001b[39;00m\n\u001b[1;32m    258\u001b[0m \u001b[38;5;66;03m# years...\u001b[39;00m\n\u001b[0;32m--> 259\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_done_event\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mMAXINT\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    261\u001b[0m \u001b[38;5;66;03m# Once done waiting, raise an exception if present or return the\u001b[39;00m\n\u001b[1;32m    262\u001b[0m \u001b[38;5;66;03m# final result.\u001b[39;00m\n\u001b[1;32m    263\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.11/threading.py:629\u001b[0m, in \u001b[0;36mEvent.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    627\u001b[0m signaled \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_flag\n\u001b[1;32m    628\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m signaled:\n\u001b[0;32m--> 629\u001b[0m     signaled \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cond\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    630\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m signaled\n",
      "File \u001b[0;32m/opt/conda/lib/python3.11/threading.py:327\u001b[0m, in \u001b[0;36mCondition.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    325\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:    \u001b[38;5;66;03m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[39;00m\n\u001b[1;32m    326\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 327\u001b[0m         \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    328\u001b[0m         gotit \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    329\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    # --- CONFIGURATION (UPDATED FOR GENERIC S3) ---\n",
    "    \n",
    "    # The endpoint URL for your S3 compatible provider (e.g., MinIO, DigitalOcean Spaces, Wasabi)\n",
    "    # The default is now set to the local Kubernetes service address.\n",
    "    ENDPOINT_URL = \"http://local-s3-service.ezdata-system.svc.cluster.local:30000\" \n",
    "\n",
    "    # --- CREDENTIALS CONFIGURATION ---\n",
    "    TOKEN_PATH = \"/etc/secrets/ezua/.auth_token\"\n",
    "    \n",
    "    # 1. Read ACCESS KEY from the specified file\n",
    "    try:\n",
    "        with open(TOKEN_PATH, 'r') as f:\n",
    "            # Read content and strip any leading/trailing whitespace (including newlines)\n",
    "            AWS_ACCESS_KEY_ID = f.read().strip()\n",
    "            logging.info(f\"Successfully read access key from {TOKEN_PATH}.\")\n",
    "    except FileNotFoundError:\n",
    "        AWS_ACCESS_KEY_ID = None\n",
    "        logging.error(f\"Error: Authentication token file not found at {TOKEN_PATH}.\")\n",
    "    except Exception as e:\n",
    "        AWS_ACCESS_KEY_ID = None\n",
    "        logging.error(f\"Error reading authentication token: {e}\")\n",
    "\n",
    "    # 2. Set SECRET KEY to the hardcoded value \"s3\"\n",
    "    AWS_SECRET_ACCESS_KEY = \"s3\"\n",
    "    \n",
    "    # The local folder you want to upload. \n",
    "    # this is root of directory this is not create - but subdirectories are mapped to prefix\n",
    "    SOURCE_DIR = \"./models\" \n",
    "\n",
    "    # The name of your S3 bucket.\n",
    "    TARGET_BUCKET = \"s3models\" \n",
    "    \n",
    "    # Optional: A prefix inside the S3 bucket. \n",
    "    #TARGET_PREFIX = \"test-upload/\" \n",
    "    TARGET_PREFIX=\"\"\n",
    "\n",
    "    # --- EXECUTION ---\n",
    "    \n",
    "    # Check if we have an access key before proceeding to create dummy data or upload\n",
    "    if not AWS_ACCESS_KEY_ID:\n",
    "        logging.error(\"Access key is missing. Cannot proceed with S3 operations.\")\n",
    "    else:\n",
    "        # Create a dummy folder structure for testing if it doesn't exist\n",
    "        if not os.path.exists(SOURCE_DIR):\n",
    "            print(f\"Creating dummy directory structure at '{SOURCE_DIR}' for demonstration.\")\n",
    "            os.makedirs(os.path.join(SOURCE_DIR, 'reports/2023'))\n",
    "            with open(os.path.join(SOURCE_DIR, 'config.ini'), 'w') as f:\n",
    "                f.write(\"[Settings]\\nversion=1.0\")\n",
    "            with open(os.path.join(SOURCE_DIR, 'reports/2023/q4.txt'), 'w') as f:\n",
    "                f.write(\"Q4 Report Data\")\n",
    "            print(\"Dummy structure created.\")\n",
    "\n",
    "\n",
    "        # Call the main upload function\n",
    "        # WARNING: This will attempt to connect and upload. Ensure your configuration \n",
    "        # (ENDPOINT_URL, KEYS, and TARGET_BUCKET) is correct.\n",
    "        # UNCOMMENT THE LINE BELOW TO EXECUTE THE UPLOAD AND BUCKET CREATION\n",
    "        upload_directory_to_s3(\n",
    "            SOURCE_DIR, \n",
    "            TARGET_BUCKET, \n",
    "            TARGET_PREFIX, \n",
    "            ENDPOINT_URL, \n",
    "            AWS_ACCESS_KEY_ID, \n",
    "            AWS_SECRET_ACCESS_KEY\n",
    "        )\n",
    "        \n",
    "#         # For demonstration purposes, just show the paths that would be used:\n",
    "#         print(\"\\n--- Dry Run Path Mapping (Uncomment the function call above to execute) ---\")\n",
    "#         dry_run_directory = SOURCE_DIR\n",
    "#         dry_run_bucket = TARGET_BUCKET\n",
    "#         dry_run_prefix = TARGET_PREFIX\n",
    "        \n",
    "#         for root, dirs, files in os.walk(dry_run_directory):\n",
    "#             relative_path = os.path.relpath(root, dry_run_directory)\n",
    "            \n",
    "#             for file_name in files:\n",
    "#                 local_file_path = os.path.join(root, file_name)\n",
    "                \n",
    "#                 if relative_path == \".\":\n",
    "#                     s3_key = dry_run_prefix + file_name\n",
    "#                 else:\n",
    "#                     s3_key = dry_run_prefix + os.path.join(relative_path, file_name).replace(os.path.sep, '/')\n",
    "                \n",
    "#                 print(f\"Local: {local_file_path.ljust(40)} -> S3 Key: {s3_key}\")\n",
    "                \n",
    "#         print(\"\\n-----------------------------------------------------------------------\")\n",
    "#         print(\"If executed, the script will first ensure the bucket exists on your generic S3 provider, then upload the files.\")\n",
    "#         print(\"Be sure to set ENDPOINT_URL, AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY in the configuration block.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd1d9928-72eb-4cf2-b438-620c004d4a45",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}