- try:
- asyncio.run(self.main(), debug=settings.DEBUG)
- except KeyboardInterrupt:
- pass
+ alert_index = 0
+ latest_alert_timestamp = 0
+ latest_exception_timestamp = 0
+
+ def exception_alert_thresholds():
+ yield 0
+ duration = 3
+ while duration < 3600:
+ yield duration
+ duration *= 5
+ duration = 3600
+ while True:
+ yield duration
+ duration += 3600
+
+ while True:
+ try:
+ asyncio.run(self.main(), debug=settings.DEBUG)
+ except KeyboardInterrupt:
+ break
+ except Exception:
+ timestamp = time.time()
+ if (timestamp - latest_exception_timestamp) > 300:
+ # if latest exception was a "long" time ago, assume
+ # things went smooth for a while and reset things
+ alert_index = 0
+ latest_alert_timestamp = 0
+ latest_exception_timestamp = 0
+
+ alert_threshold = 0
+ for i, threshold in enumerate(exception_alert_thresholds()):
+ if i == alert_index:
+ alert_threshold = threshold
+ break
+
+ if (timestamp - latest_alert_timestamp) > alert_threshold:
+ logger.exception('General exception (alert index: %s)', alert_index)
+ latest_alert_timestamp = timestamp
+ alert_index += 1
+
+ time.sleep(2) # retry after a bit
+ latest_exception_timestamp = timestamp
+ continue
+ break