Add retry logic for all management commands

- use tenacity library
- log at INFO level
- Remove some exception handling in linksearchtotal_collect command to enable retry

Bug: T403209

Author: jsnshrmn

extlinks/common/management/commands/__init__.py

@@ -4,21 +4,46 @@
 import logging
 from os import remove
 from os.path import basename
+import sys
+from tenacity import Retrying, stop_after_attempt, wait_exponential
+
+logger = logging.getLogger("django")
 
 
 class BaseCommand(DjangoBaseCommand):
     """
-    Django BaseCommand wrapper that adds file locks to management commands
+    Django BaseCommand wrapper that adds
+        - file locks
+        - up to 5 retries with exponential backoff
     """
 
+    def __init__(self, *args, **options):
+        super().__init__(*args, **options)
+        self.name = basename(inspect.getfile(self.__class__))
+        self.status = 0
+
+    def retry_log(self, retry_state):
+        logger.warning(f"{self.name} attempt {retry_state.attempt_number} failed")
+
     def handle(self, *args, **options):
-        lockname = basename(inspect.getfile(self.__class__))
         # Use a lockfile to prevent overruns.
-        lockfile = "/tmp/{}.lock".format(lockname)
+        logger.info(f"Executing {self.name}")
+        lockfile = f"/tmp/{self.name}.lock"
         lock = FileLock(lockfile)
         lock.acquire()
         try:
-            self._handle(*args, **options)
-        finally:
-            lock.release()
-            remove(lockfile)
+            for attempt in Retrying(
+                after=self.retry_log,
+                reraise=True,
+                stop=stop_after_attempt(5),
+                wait=wait_exponential(multiplier=1, min=60, max=300),
+            ):
+                with attempt:
+                    self._handle(*args, **options)
+        except Exception as e:
+            logger.warning(f"Retries exhausted for {self.name}")
+            logger.error(e)
+            self.status = 1
+        lock.release()
+        remove(lockfile)
+        sys.exit(self.status)

extlinks/links/management/commands/linksearchtotal_collect.py

@@ -32,16 +32,12 @@ def _handle(self, *args, **options):
         total_links_dictionary = {}
         for i, language in enumerate(wiki_list_data):
             logger.info(f"connecting to db {language}")
-            try:
-                db = MySQLdb.connect(
-                    host=f"{language}wiki.analytics.db.svc.wikimedia.cloud",
-                    user=os.environ["REPLICA_DB_USER"],
-                    passwd=os.environ["REPLICA_DB_PASSWORD"],
-                    db=f"{language}wiki_p",
-                )
-            except MySQLdb.OperationalError as e:
-                logger.error(str(e))
-                sys.exit(1)
+            db = MySQLdb.connect(
+                host=f"{language}wiki.analytics.db.svc.wikimedia.cloud",
+                user=os.environ["REPLICA_DB_USER"],
+                passwd=os.environ["REPLICA_DB_PASSWORD"],
+                db=f"{language}wiki_p",
+            )
 
             cur = db.cursor()
 

extlinks/settings/logging.py

@@ -25,7 +25,7 @@
         },
         "handlers": {
             "nodebug_console": {
-                "level": "WARNING",
+                "level": "INFO",
                 "filters": ["require_debug_false"],
                 "class": "logging.StreamHandler",
                 "formatter": "django.server",

requirements/django.txt

@@ -8,6 +8,7 @@ pymemcache==4.0.0
 python-dotenv==1.1.0
 sentry-sdk==2.10.0
 sseclient==0.0.27
+tenacity==9.1.2
 time_machine==2.14.2
 python-swiftclient>=4.6.0,<5.0.0
 keystoneauth1>=5.9.2,<6.0.0