stamina: abort on repeated database error

diff --git a/nonstop/management/commands/stamina.py b/nonstop/management/commands/stamina.py
index 1122263c591849989bc5d25449809b229efb48b9..6cf3ae4cd2f8414346afcf9e83e851782943de4b 100644 (file)
--- a/nonstop/management/commands/stamina.py
+++ b/nonstop/management/commands/stamina.py
@@ -10,6 +10,7 @@ import time
 import requests
 
 from django.conf import settings
+import django.db
 from django.core.management.base import BaseCommand
 
 from emissions.models import Nonstop
@@ -49,7 +50,7 @@ class Command(BaseCommand):
                 asyncio.run(self.main(), debug=settings.DEBUG)
             except KeyboardInterrupt:
                 break
-            except Exception:
+            except Exception as e:
                 timestamp = time.time()
                 if (timestamp - latest_exception_timestamp) > 300:
                     # if latest exception was a "long" time ago, assume
@@ -69,6 +70,12 @@ class Command(BaseCommand):
                     latest_alert_timestamp = timestamp
                     alert_index += 1
 
+                if alert_index and isinstance(e, django.db.InterfaceError):
+                    # most likely "connection already closed", because postgresql
+                    # is been restarted; log then get out to be restarted.
+                    logger.error('Aborting on repeated database error')
+                    break
+
                 time.sleep(2)  # retry after a bit
                 latest_exception_timestamp = timestamp
                 continue