Migrate_Storage_Objects.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mansueli/Supa-Migrate/blob/main/Migrate_Storage_Objects.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "![Supabase](https://raw.githubusercontent.com/supabase/supabase/master/packages/common/assets/images/supabase-logo-wordmark--light.svg)\n",
        "\n"
      ],
      "metadata": {
        "id": "ygevkg_zCkmo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setting the Environment variables:"
      ],
      "metadata": {
        "id": "LL4Furs6VliO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "OLD_DB_URL='https://old_project_ref.supabase.co'\n",
        "NEW_DB_URL='https://new_project_ref.supabase.co'\n",
        "OLD_SERVICE_KEY = 'eyJ0000J9.eyJQ.oPyK-LSECRET-aC1I'\n",
        "NEW_SERVICE_KEY = 'eyJ0000J9.eyJpSERVICE_ROLE.d4tffFJoc8iHsk_KEY'"
      ],
      "metadata": {
        "id": "_gS4xrZzVkl3"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Installing dependencies"
      ],
      "metadata": {
        "id": "scZlpumHVbi6"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WqBSjmZxVOn1"
      },
      "outputs": [],
      "source": [
        "!pip install supabase &>log\n",
        "!pip install python-magic &>log\n",
        "!sudo apt install libmagic1 &>log"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Running migration & moving storage objects:"
      ],
      "metadata": {
        "id": "giay7Rs5WkqD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from supabase import create_client\n",
        "import os\n",
        "import re\n",
        "import magic\n",
        "filedata = ''\n",
        "\n",
        "# Function to recursively copy files and directories\n",
        "def copy_files(old_client, new_client, bucket_name, path):\n",
        "    objects = old_client.storage.from_(bucket_name).list(path=path)\n",
        "\n",
        "    for obj in objects:\n",
        "        if obj['metadata'] is None:  # It's a directory\n",
        "            subdirectory = os.path.join(path, obj['name'])\n",
        "            # Create the directory if it doesn't exist\n",
        "            if not os.path.exists(subdirectory):\n",
        "                os.makedirs(subdirectory)\n",
        "            copy_files(old_client, new_client, bucket_name, subdirectory)\n",
        "        else:\n",
        "            try:\n",
        "                print(f\"Downloading {path}/{obj['name']}\")\n",
        "                # Download file\n",
        "                download_path = os.path.join(path, obj['name'])\n",
        "                with open(download_path, 'wb') as f:\n",
        "                    res = old_client.storage.from_(bucket_name).download(f\"{path}/{obj['name']}\")\n",
        "                    f.write(res)\n",
        "\n",
        "                # Get MIME type\n",
        "                mime_type = magic.from_file(os.path.abspath(download_path), mime=True)\n",
        "\n",
        "                # Upload file\n",
        "                print(f\"Uploading file {path}/{obj['name']}\")\n",
        "                with open(download_path, 'rb') as file_object:\n",
        "                    new_client.storage.from_(bucket_name).upload(f\"{path}/{obj['name']}\",\n",
        "                                                                 file_object,\n",
        "                                                                 file_options={\"content-type\": mime_type, \"x-upsert\": 'true'})\n",
        "\n",
        "                # Delete local file after uploading\n",
        "                os.remove(download_path)\n",
        "            except Exception as e:\n",
        "                print(\"Error: \", e)\n",
        "\n",
        "# Main function\n",
        "def main():\n",
        "    # Create clients for the old & new projects\n",
        "    old_supabase_client = create_client(OLD_DB_URL, OLD_SERVICE_KEY)\n",
        "    new_supabase_client = create_client(NEW_DB_URL, NEW_SERVICE_KEY)\n",
        "\n",
        "    # Create all buckets\n",
        "    buckets = old_supabase_client.storage.list_buckets()\n",
        "    for bucket in buckets:\n",
        "        print(\"Copying objects from\", bucket.name)\n",
        "        try:\n",
        "            # Check if the bucket exists before creating it\n",
        "            if not new_supabase_client.storage.get_bucket(bucket.name):\n",
        "                new_supabase_client.storage.create_bucket(bucket.name, options={\"public\": bucket.public})\n",
        "        except Exception as e:\n",
        "            print(\"Unable to create bucket:\", e)\n",
        "\n",
        "        # Copy files recursively\n",
        "        copy_files(old_supabase_client, new_supabase_client, bucket.name, \"\")\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()"
      ],
      "metadata": {
        "id": "NFI8Y6SzF-Aj"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}