notebooks/hubmap-kidney-products-download.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "970f755b-0387-41ce-bd2f-ce247979746e",
   "metadata": {},
   "source": [
    "# Download Kidney Data Products from HuBMAP"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2b32639-94f2-40e4-8842-d620e986d10a",
   "metadata": {},
   "source": [
    "## Imports / functions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6db30373-1d2a-4b3b-b354-343512779aa8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import requests\n",
    "import warnings\n",
    "from urllib.request import urlretrieve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "75f16c0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_file(url, filename, dir=\".\"):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir, exist_ok = True) # unlike os.mkdir, os.makedirs creates directories recursively\n",
    "    urlretrieve(url, dir + \"/\" + filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5c4821da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 Kidney Data Products\n"
     ]
    }
   ],
   "source": [
    "DATA_PRODUCTS_API=\"https://data-products.cmu.hubmapconsortium.org/api/\"\n",
    "def get_kidney_data_products():\n",
    "    products = json.loads(requests.get(DATA_PRODUCTS_API).content)\n",
    "    codes = set([\"LK\", \"RK\"])\n",
    "    kidney_products = [ p for p in products if p['tissue']['tissuecode'] in codes ]\n",
    "    return kidney_products\n",
    "\n",
    "products = get_kidney_data_products()\n",
    "print(len(products), \"Kidney Data Products\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b4d5e9f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://hubmap-data-products.s3.amazonaws.com/d8397c5a-cdad-4d29-93ef-cf62702ed61a/LK_processed.h5ad to hubmap-LK-processed.h5ad\n",
      "Downloading https://hubmap-data-products.s3.amazonaws.com/31a34af6-ab40-47e8-937d-71929a9d1c6c/RK_processed.h5ad to hubmap-RK-processed.h5ad\n"
     ]
    }
   ],
   "source": [
    "for product in products:\n",
    "    code = product['tissue']['tissuecode']\n",
    "    url = product['download']\n",
    "    file = f\"hubmap-{code}-processed.h5ad\"\n",
    "    print(f\"Downloading {url} to {file}\")\n",
    "    download_file(url, file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}