From 38b2bb53a5dab57f0f9ec057d326c6d082be80cb Mon Sep 17 00:00:00 2001 From: Tyler Erickson Date: Thu, 30 May 2024 10:44:26 -0700 Subject: [PATCH] Minor update to README example --- README.md | 4 +- index.ipynb | 467 +++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 448 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index b835b1a..1bd96f5 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,9 @@ def get_country_stats(country_name): ).get('DEM').getInfo(), } -# Create and submit jobs to among the workers. +# Create and submit jobs among the workers. submitted_jobs = [ - client.submit(get_country_stats, country) + client.submit(get_country_stats, country, retries=5) for country in country_list ] diff --git a/index.ipynb b/index.ipynb index 9db0ed1..677804c 100644 --- a/index.ipynb +++ b/index.ipynb @@ -1,14 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#| hide\n" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -61,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -88,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -112,9 +103,94 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "17aa0dba96ba442892e6445f9721c007", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
╭──────────────────────────────── Package Info ────────────────────────────────╮\n",
+       "│                    ╷                                                         │\n",
+       "│   Package           Note                                                    │\n",
+       "│ ╶──────────────────┼───────────────────────────────────────────────────────╴ │\n",
+       "│   earthengine-dask │ Wheel built from                                        │\n",
+       "│                    │ ~/Documents/GitHub/VorGeo/earthengine-dask              │\n",
+       "│                    ╵                                                         │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────╯\n",
+       "
\n" + ], + "text/plain": [ + "╭──────────────────────────────── \u001b[1;32mPackage Info\u001b[0m ────────────────────────────────╮\n", + "│ ╷ │\n", + "│ \u001b[1m \u001b[0m\u001b[1mPackage \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mNote \u001b[0m\u001b[1m \u001b[0m │\n", + "│ ╶──────────────────┼───────────────────────────────────────────────────────╴ │\n", + "│ earthengine-dask │ Wheel built from │\n", + "│ │ ~/Documents/GitHub/VorGeo/earthengine-dask │\n", + "│ ╵ │\n", + "╰──────────────────────────────────────────────────────────────────────────────╯\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cd050b60dacf42a1973daaf94c721af2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Google Application Default Credentials have been written to a file on your Coiled VM(s).\n",
+      "These credentials will potentially be valid until explicitly revoked by running\n",
+      "gcloud auth application-default revoke\n"
+     ]
+    }
+   ],
    "source": [
     "cluster = ClusterGEE(\n",
     "    name='test-class-cluster',\n",
@@ -133,9 +209,226 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
\n", + "
\n", + "
\n", + "

Client

\n", + "

Client-6d2ffaee-1eab-11ef-acda-fe11494405b6

\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
Connection method: Cluster objectCluster type: earthengine_dask.ClusterGEE
\n", + " Dashboard: https://cluster-zoqmf.dask.host/mnc9NHlG1-Dp_CBQ/status\n", + "
\n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "

Cluster Info

\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

ClusterGEE

\n", + "

test-class-cluster

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " Dashboard: https://cluster-zoqmf.dask.host/mnc9NHlG1-Dp_CBQ/status\n", + " \n", + " Workers: 2\n", + "
\n", + " Total threads: 16\n", + " \n", + " Total memory: 61.17 GiB\n", + "
\n", + "\n", + "
\n", + " \n", + "

Scheduler Info

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

Scheduler

\n", + "

Scheduler-81ff2eac-d070-45e0-8601-8eff06d7d72c

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " Comm: tls://10.2.0.48:8786\n", + " \n", + " Workers: 2\n", + "
\n", + " Dashboard: http://10.2.0.48:8787/status\n", + " \n", + " Total threads: 16\n", + "
\n", + " Started: Just now\n", + " \n", + " Total memory: 61.17 GiB\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "

Workers

\n", + "
\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: test-class-cluster-worker-5e0fe43b89

\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tls://10.2.0.47:42155\n", + " \n", + " Total threads: 8\n", + "
\n", + " Dashboard: http://10.2.0.47:8787/status\n", + " \n", + " Memory: 30.58 GiB\n", + "
\n", + " Nanny: tls://10.2.0.47:45609\n", + "
\n", + " Local directory: /scratch/dask-scratch-space/worker-rk_7aw52\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: test-class-cluster-worker-b75971f916

\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tls://10.2.0.46:39499\n", + " \n", + " Total threads: 8\n", + "
\n", + " Dashboard: http://10.2.0.46:8787/status\n", + " \n", + " Memory: 30.58 GiB\n", + "
\n", + " Nanny: tls://10.2.0.46:39223\n", + "
\n", + " Local directory: /scratch/dask-scratch-space/worker-d0zam3_g\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "client = cluster.get_client()\n", "client" @@ -156,9 +449,129 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countryarea_km2mean_elev
0Abyei Area10460402.592190
1Afghanistan6420931809.717311
2Akrotiri12760.796081
3Aksai Chin304485324.949965
4Albania28638689.740847
............
279West Bank5813341.715816
280Western Sahara269689253.175100
281Yemen454682932.051351
282Zambia7540321118.995950
283Zimbabwe391916973.295555
\n", + "

284 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " country area_km2 mean_elev\n", + "0 Abyei Area 10460 402.592190\n", + "1 Afghanistan 642093 1809.717311\n", + "2 Akrotiri 127 60.796081\n", + "3 Aksai Chin 30448 5324.949965\n", + "4 Albania 28638 689.740847\n", + ".. ... ... ...\n", + "279 West Bank 5813 341.715816\n", + "280 Western Sahara 269689 253.175100\n", + "281 Yemen 454682 932.051351\n", + "282 Zambia 754032 1118.995950\n", + "283 Zimbabwe 391916 973.295555\n", + "\n", + "[284 rows x 3 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Get a list of countries to analyze.\n", "country_fc = ee.FeatureCollection('USDOS/LSIB_SIMPLE/2017')\n", @@ -177,9 +590,9 @@ " ).get('DEM').getInfo(),\n", " }\n", "\n", - "# Create and submit jobs to among the workers.\n", + "# Create and submit jobs among the workers.\n", "submitted_jobs = [\n", - " client.submit(get_country_stats, country)\n", + " client.submit(get_country_stats, country, retries=5)\n", " for country in country_list\n", "]\n", "\n", @@ -198,7 +611,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -208,9 +621,21 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4,