fix(taskmanager): add early guard for over-failed tasks

talkiq · Jul 5, 2018 · 9941564 · 9941564
1 parent 3c7878b
commit 9941564
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/gcloud/rest/taskqueue/manager.py b/gcloud/rest/taskqueue/manager.py
@@ -117,8 +117,17 @@ def find_and_process_work(self):
         leasers = []
         payloads = []
         for task in tasks:
-            payloads.append(
-                json.loads(decode(task['pullMessage']['payload']).decode()))
+            payload = json.loads(
+                decode(task['pullMessage']['payload']).decode())
+            retries = int(task['status']['attemptDispatchCount'])
+            if self.retry_limit is not None and retries > 2 * self.retry_limit:
+                log.warning('force failing task %s with %d/%d retries',
+                            task['name'], retries, self.retry_limit)
+                self.fail_task(payload, 'force failed after too many attempts')
+                self.tq.delete(task['name'])
+                continue
+
+            payloads.append(payload)
 
             try:
                 data = self.manager.dict()

diff --git a/setup.py b/setup.py
@@ -13,7 +13,7 @@
 
 setuptools.setup(
     name='gcloud-rest',
-    version='1.4.0',
+    version='1.5.0',
     description='RESTful Python Client for Google Cloud',
     long_description=README,
     namespace_packages=[