landsat_aois_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *\n",
    "import getpass\n",
    "from landsatxplore.earthexplorer import EarthExplorer\n",
    "import tarfile\n",
    "import boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aws_session = rasterio.session.AWSSession(boto3.Session(), requester_pays=True)\n",
    "# aws_session = rasterio.session.AWSSession(boto3.Session())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bbox = read_kml_polygon(\"data/inputs/LANDSAT_8_127111/WA.kml\")[1]\n",
    "bbox = resize_bbox(bbox, 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user = getpass.getpass(\"Enter your username?\")\n",
    "password = getpass.getpass(\"Enter your password?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = \"2023-01-01T00:00:00\"\n",
    "end_date: str = \"2025-01-01T00:00:00\"\n",
    "platform = \"LANDSAT_8\"\n",
    "#   query = get_landsat_search_query(\n",
    "#   [67.45, -72.55, 67.55, -72.45],\n",
    "#   start_date=start_date,\n",
    "#   end_date=end_date,\n",
    "#   platform=platform\n",
    "# ) # Amery ice shelf bed rock\n",
    "# query = get_landsat_search_query(\n",
    "#     [69.2, -68.1, 69.4, -67.9],\n",
    "#     start_date=start_date,\n",
    "#     end_date=end_date,\n",
    "#     platform=platform,\n",
    "#     cloud_cover=50,\n",
    "# )  # Top of Amery ice shelf\n",
    "server_url = \"https://landsatlook.usgs.gov/stac-server/search\"\n",
    "query = get_landsat_search_query(\n",
    "    bbox,\n",
    "    start_date=start_date,\n",
    "    end_date=end_date,\n",
    "    platform=platform,\n",
    ")  # WA sand dunes\n",
    "features = query_stac_server(query, server_url)\n",
    "feat_dict = find_landsat_scenes_dict(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(list(feat_dict.keys())).astype(\"str\")\n",
    "g = [s.split(\"_\")[3][0:6] for s in list(feat_dict.keys())]\n",
    "feat_dict = {id: feat_dict[id] for id in [el[1].iloc[0] for el in list(s.groupby(g))]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aws_reds = [feat_dict[k][\"red\"][1] for k in feat_dict]\n",
    "bounds_dict = {}\n",
    "for r in aws_reds:\n",
    "    id = os.path.basename(r).split(\"_\")[2]\n",
    "    meta = stream_scene_from_aws(r, aws_session, metadata_only=True)[1]\n",
    "    bounds_dict[id] = {\"bounds\": meta[\"bounds\"], \"crs\": meta[\"crs\"].data}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key in bounds_dict:\n",
    "    bounds = bounds_dict[key][\"bounds\"]\n",
    "    crs = bounds_dict[key][\"crs\"]\n",
    "    bottom_utm = UTM(bounds.left, bounds.bottom)\n",
    "    top_utm = UTM(bounds.right, bounds.top)\n",
    "    bottom_lla = UTMtoLLA(bottom_utm, crs)\n",
    "    top_lla = UTMtoLLA(top_utm, crs)\n",
    "    print(key, bottom_lla, top_lla)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_num = \"108074\"\n",
    "required_keys = [k for k in list(feat_dict.keys()) if path_num in k]\n",
    "required_scenes = [feat_dict[k][\"scene_id\"] for k in required_keys]\n",
    "len(required_scenes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scene_index = 13\n",
    "\n",
    "r_url = feat_dict[required_keys[scene_index]][\"red\"][0]\n",
    "g_url = feat_dict[required_keys[scene_index]][\"green\"][0]\n",
    "b_url = feat_dict[required_keys[scene_index]][\"blue\"][0]\n",
    "\n",
    "r_aws = feat_dict[required_keys[scene_index]][\"red\"][1]\n",
    "g_aws = feat_dict[required_keys[scene_index]][\"green\"][1]\n",
    "b_aws = feat_dict[required_keys[scene_index]][\"blue\"][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_img, _ = stream_scene_from_aws(r_aws, aws_session)\n",
    "g_img, _ = stream_scene_from_aws(g_aws, aws_session)\n",
    "b_img, _ = stream_scene_from_aws(b_aws, aws_session)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rgb = apply_gamma(cv.merge([r_img[0, :, :], g_img[0, :, :], b_img[0, :, :]]), 1.0, True)\n",
    "plt.imshow(rgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key in required_keys:\n",
    "    print(f\"Now downloading {key}\")\n",
    "    r_url = feat_dict[key][\"red\"][1]\n",
    "    g_url = feat_dict[key][\"green\"][1]\n",
    "    b_url = feat_dict[key][\"blue\"][1]\n",
    "    output_dir = f\"data/inputs/{platform}_{path_num}/{key}\"\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "    r_output = os.path.join(output_dir, os.path.basename(r_url))\n",
    "    g_output = os.path.join(output_dir, os.path.basename(g_url))\n",
    "    b_output = os.path.join(output_dir, os.path.basename(b_url))\n",
    "    r_img, r_meta = stream_scene_from_aws(r_url, aws_session)\n",
    "    g_img, g_meta = stream_scene_from_aws(g_url, aws_session)\n",
    "    b_img, b_meta = stream_scene_from_aws(b_url, aws_session)\n",
    "\n",
    "    imgs = [r_img, g_img, b_img]\n",
    "    outputs = [r_output, g_output, b_output]\n",
    "    metas = [r_meta, g_meta, b_meta]\n",
    "    for i, img in enumerate(imgs):\n",
    "        with rasterio.open(outputs[i], \"w\", **metas[i][\"profile\"]) as ds:\n",
    "            ds.write(img[0, :, :], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ee = EarthExplorer(user, password)\n",
    "# for scene in required_scenes:\n",
    "#     output_dir = f\"data/inputs/{platform}_{path_num}\"\n",
    "#     os.makedirs(f\"data/inputs/{platform}_{path_num}\", exist_ok=True)\n",
    "#     print(f\"Downloading {scene}\")\n",
    "#     ee.download(scene, output_dir)\n",
    "# ee.logout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tar_files = [\n",
    "#     f for f in glob.glob(f\"data/inputs/{platform}_{path_num}/**\") if f.endswith(\".tar\")\n",
    "# ]\n",
    "# for tar_file in tar_files:\n",
    "#     extraction_dir = os.path.join(\n",
    "#         output_dir, os.path.splitext(os.path.basename(tar_file))[0]\n",
    "#     )\n",
    "#     os.makedirs(extraction_dir, exist_ok=True)\n",
    "#     with tarfile.open(tar_file) as tf:\n",
    "#         tf.extractall(extraction_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "red = f\"data/inputs/LANDSAT_8_132108/LC08_L2SR_132108_20241117_20241126_02_T2_SR/LC08_L2SR_132108_20241117_20241126_02_T2_SR_B4.TIF\"\n",
    "green = f\"data/inputs/LANDSAT_8_132108/LC08_L2SR_132108_20241117_20241126_02_T2_SR/LC08_L2SR_132108_20241117_20241126_02_T2_SR_B3.TIF\"\n",
    "blue = f\"data/inputs/LANDSAT_8_132108/LC08_L2SR_132108_20241117_20241126_02_T2_SR/LC08_L2SR_132108_20241117_20241126_02_T2_SR_B2.TIF\"\n",
    "\n",
    "redf = flip_img(rasterio.open(red).read())[:, :, 0]\n",
    "redf = apply_gamma(redf, 1.0, True)\n",
    "\n",
    "greenf = flip_img(rasterio.open(green).read())[:, :, 0]\n",
    "greenf = apply_gamma(greenf, 1.0, True)\n",
    "\n",
    "bluef = flip_img(rasterio.open(blue).read())[:, :, 0]\n",
    "bluef = apply_gamma(bluef, 1.0, True)\n",
    "\n",
    "\n",
    "img = cv.merge([redf, bluef, greenf])\n",
    "img.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_color_dir = f\"data/inputs/{platform}_{path_num}/true_color\"\n",
    "os.makedirs(true_color_dir, exist_ok=True)\n",
    "scenes_dirs = [\n",
    "    dir for dir in glob.glob(f\"data/inputs/{platform}_{path_num}/**\") if \"_T1\" in dir\n",
    "]\n",
    "for dir in scenes_dirs:\n",
    "    print(f\"Now processing {os.path.basename(dir)}.\")\n",
    "    files = glob.glob(f\"{dir}/**\")\n",
    "    r_band = list(filter(lambda f: f.endswith(\"_B4.TIF\"), files))[0]\n",
    "    g_band = list(filter(lambda f: f.endswith(\"_B3.TIF\"), files))[0]\n",
    "    b_band = list(filter(lambda f: f.endswith(\"_B2.TIF\"), files))[0]\n",
    "    true_bands = [r_band, g_band, b_band]\n",
    "    make_landsat_true_color_scene(\n",
    "        true_bands, f\"{os.path.join(true_color_dir, os.path.basename(dir))}_TC.TIF\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_color_ds_dir = f\"data/inputs/{platform}_{path_num}/true_color_ds\"\n",
    "os.makedirs(true_color_ds_dir, exist_ok=True)\n",
    "tc_files = glob.glob(f\"data/inputs/{platform}_{path_num}/true_color/**\")\n",
    "for tcf in tc_files:\n",
    "    downsample_dataset(tcf, 0.2, os.path.join(true_color_ds_dir, os.path.basename(tcf)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}