From 4f6f1d9f66bbcfcfee59f249e48aa63b6b1888d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C4=93teris=20Caune?= Date: Fri, 7 Feb 2020 10:36:45 +0200 Subject: [PATCH] Fix sendalerts crash loop when encountering a bad cron schedule --- CHANGELOG.md | 2 +- hc/api/management/commands/sendalerts.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2626b478..21f0e176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ All notable changes to this project will be documented in this file. - Increase the allowable length of Matrix room alias to 100 (#320) - Make sure Check.last_ping and Ping.created timestamps match exactly - Don't trigger "down" notifications when changing schedule interactively in web UI - +- Fix sendalerts crash loop when encountering a bad cron schedule ## v1.12.0 - 2020-01-02 diff --git a/hc/api/management/commands/sendalerts.py b/hc/api/management/commands/sendalerts.py index 3407708c..be267600 100644 --- a/hc/api/management/commands/sendalerts.py +++ b/hc/api/management/commands/sendalerts.py @@ -1,3 +1,4 @@ +from datetime import timedelta as td import time from threading import Thread @@ -103,7 +104,16 @@ class Command(BaseCommand): old_status = check.status q = Check.objects.filter(id=check.id, status=old_status) - if check.get_status(with_started=False) != "down": + try: + status = check.get_status(with_started=False) + except Exception as e: + # Make sure we don't trip on this check again for an hour: + # Otherwise sendalerts may end up in a crash loop. + q.update(alert_after=now + td(hours=1)) + # Then re-raise the exception: + raise e + + if status != "down": # It is not down yet. Update alert_after q.update(alert_after=check.going_down_after()) return True