HLS_NASA_AWS_update.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask.distributed import LocalCluster, Client\n",
    "import xarray as xr\n",
    "import dask\n",
    "import intake\n",
    "import os\n",
    "#import fsspec, os, netrc, aiohttp,dask\n",
    "from satsearch import Search\n",
    "import hvplot.pandas\n",
    "import hvplot.xarray\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AUTHENTICATION CONFIGURATION\n",
    "from netrc import netrc\n",
    "from subprocess import Popen\n",
    "from getpass import getpass\n",
    "\n",
    "urs = 'urs.earthdata.nasa.gov'    # Earthdata URL to call for authentication\n",
    "prompts = ['Enter NASA Earthdata Login Username \\n(or create an account at urs.earthdata.nasa.gov): ',\n",
    "           'Enter NASA Earthdata Login Password: ']\n",
    "\n",
    "# Determine if netrc file exists, and if so, if it includes NASA Earthdata Login Credentials\n",
    "try:\n",
    "    netrcDir = os.path.expanduser(\"~/.netrc\")\n",
    "    netrc(netrcDir).authenticators(urs)[0]\n",
    "    del netrcDir\n",
    "\n",
    "# Below, create a netrc file and prompt user for NASA Earthdata Login Username and Password\n",
    "except FileNotFoundError:\n",
    "    homeDir = os.path.expanduser(\"~\")\n",
    "    Popen('touch {0}.netrc | chmod og-rw {0}.netrc | echo machine {1} >> {0}.netrc'.format(homeDir + os.sep, urs), shell=True)\n",
    "    Popen('echo login {} >> {}.netrc'.format(getpass(prompt=prompts[0]), homeDir + os.sep), shell=True)\n",
    "    Popen('echo password {} >> {}.netrc'.format(getpass(prompt=prompts[1]), homeDir + os.sep), shell=True)\n",
    "    del homeDir\n",
    "\n",
    "# Determine OS and edit netrc file if it exists but is not set up for NASA Earthdata Login\n",
    "except TypeError:\n",
    "    homeDir = os.path.expanduser(\"~\")\n",
    "    Popen('echo machine {1} >> {0}.netrc'.format(homeDir + os.sep, urs), shell=True)\n",
    "    Popen('echo login {} >> {}.netrc'.format(getpass(prompt=prompts[0]), homeDir + os.sep), shell=True)\n",
    "    Popen('echo password {} >> {}.netrc'.format(getpass(prompt=prompts[1]), homeDir + os.sep), shell=True)\n",
    "    del homeDir\n",
    "del urs, prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setup GDAL Env for optimum performance\n",
    "env = dict(GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR', \n",
    "           AWS_NO_SIGN_REQUEST='YES',\n",
    "           GDAL_MAX_RAW_BLOCK_CACHE_SIZE='200000000',\n",
    "           GDAL_SWATH_SIZE='200000000',\n",
    "           VSI_CURL_CACHE_SIZE='200000000',\n",
    "           CPL_VSIL_CURL_ALLOWED_EXTENSIONS='TIF',\n",
    "           GDAL_HTTP_UNSAFESSL='YES',\n",
    "           GDAL_HTTP_COOKIEFILE=os.path.expanduser('~/cookies.txt'),\n",
    "           GDAL_HTTP_COOKIEJAR=os.path.expanduser('~/cookies.txt'))\n",
    "os.environ.update(env)\n",
    "\n",
    "#from osgeo import gdal\n",
    "#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN','EMPTY_DIR')\n",
    "#gdal.SetConfigOption('AWS_NO_SIGN_REQUEST', 'YES')\n",
    "#gdal.SetConfigOption('GDAL_MAX_RAW_BLOCK_CACHE_SIZE', '200000000')\n",
    "#gdal.SetConfigOption('GDAL_SWATH_SIZE', '200000000')\n",
    "#gdal.SetConfigOption('VSI_CURL_CACHE_SIZE', '200000000')\n",
    "#gdal.SetConfigOption('CPL_VSIL_CURL_ALLOWED_EXTENSIONS', 'TIF')\n",
    "#gdal.SetConfigOption('GDAL_HTTP_UNSAFESSL', 'YES')\n",
    "#gdal.SetConfigOption('GDAL_HTTP_COOKIEFILE', '~/cookies.txt')\n",
    "#gdal.SetConfigOption('GDAL_HTTP_COOKIEJAR', '~/cookies.txt')\n",
    "\n",
    "#gdal.SetConfigOption('VSI_CACHE', 'FALSE')\n",
    "#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'YES')\n",
    "#gdal.SetConfigOption('GDAL_HTTP_COOKIEFILE', os.path.expanduser('~\\\\cookies.txt'))\n",
    "#gdal.SetConfigOption('GDAL_HTTP_COOKIEJAR', os.path.expanduser('~\\\\cookies.txt'))\n",
    "#gdal.SetConfigOption(\"GDAL_HTTP_UNSAFESSL\", \"YES\")\n",
    "#gdal.SetConfigOption('CPL_VSIL_CURL_ALLOWED_EXTENSIONS','.tif')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dask.config.set({'distributed.dashboard.link':'http://localhost:8888/proxy/8787/status'})#'https://localhost:8787/status'})\n",
    "cluster = LocalCluster(threads_per_worker=2)\n",
    "client = Client(cluster)\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_STAC_items(url, collection, dates, bbox):\n",
    "    results = Search.search(url=url,\n",
    "                        collections=[collection], \n",
    "                        datetime=dates,\n",
    "                        bounding_box=bbox,    \n",
    "                        sortby=['-properties.datetime'])\n",
    "\n",
    "    items = results.items()\n",
    "    print(f'Found {len(items)} Items')\n",
    "    return intake.open_stac_item_collection(items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://cmr.earthdata.nasa.gov/stac/LPCLOUD' \n",
    "collection = 'C1711924822-LPCLOUD' #HLS\n",
    "bbox=[-104.79107047,   40.78311181, -104.67687336,   40.87008987]\n",
    "dates = '2013-01-01/2021-12-31'\n",
    "\n",
    "cat = get_STAC_items(url,collection,dates,','.join(map(str, bbox)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat.to_geopandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@dask.delayed\n",
    "def lazy_open(href,chunks=dict(band=1, x=3660, y=3660)):\n",
    "    return xr.open_rasterio(href, chunks=chunks)\n",
    "\n",
    "band = 'B01'\n",
    "dataArrays = dask.compute(*[lazy_open(cat[item][band].urlpath) for item in list(cat)], retries=3)\n",
    "da = xr.concat(dataArrays, dim='band', join='override', combine_attrs='drop').rename(band='time')\n",
    "da['time'] = [cat[item].metadata['datetime'] for item in list(cat)]\n",
    "da"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "da = da.persist()\n",
    "da"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}