diff --git a/.gitignore b/.gitignore
index cead003d..ca72aab3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
__pycache__/
*.pyc
.coverage
-local_settings.py
+hc.sqlite
+hc/local_settings.py
static-collected
\ No newline at end of file
diff --git a/hc/api/management/commands/sendalerts.py b/hc/api/management/commands/sendalerts.py
index c5ed4dec..f81044f8 100644
--- a/hc/api/management/commands/sendalerts.py
+++ b/hc/api/management/commands/sendalerts.py
@@ -1,17 +1,63 @@
+import logging
import sys
import time
from django.core.management.base import BaseCommand
+from django.db.models import Q
from django.utils import timezone
from hc.api.models import Check
+logger = logging.getLogger(__name__)
-def _log(message):
+
+def _stdout(message):
sys.stdout.write(message)
sys.stdout.flush()
+def handle_one():
+ """ Send an alert for a single check.
+
+ Return True if an appropriate check was selected and processed.
+ Return False if no checks need to be processed.
+
+ """
+
+ query = Check.objects.filter(user__isnull=False)
+
+ now = timezone.now()
+ going_down = Q(alert_after__lt=now, status="up")
+ going_up = Q(alert_after__gt=now, status="down")
+ query = query.filter(going_down | going_up)
+
+ try:
+ check = query[0]
+ except IndexError:
+ return False
+
+ check.status = check.get_status()
+
+ tmpl = "\nSending alert, status=%s, code=%s\n"
+ _stdout(tmpl % (check.status, check.code))
+
+ try:
+ check.send_alert()
+ except:
+ # Catch EVERYTHING. If we crash here, what can happen is:
+ # - the sendalerts command will crash
+ # - supervisor will respawn sendalerts command
+ # - sendalerts will try same thing again, resulting in infinite loop
+ # So instead we catch and log all exceptions, and mark
+ # the checks as paused so they are not retried.
+ logger.error("Could not alert %s" % check.code, exc_info=True)
+ check.status = "paused"
+ finally:
+ check.save()
+
+ return True
+
+
class Command(BaseCommand):
help = 'Sends UP/DOWN email alerts'
@@ -19,36 +65,12 @@ class Command(BaseCommand):
ticks = 0
while True:
- # Gone down?
- query = Check.objects
- query = query.filter(alert_after__lt=timezone.now())
- query = query.filter(user__isnull=False)
- query = query.filter(status="up")
- for check in query:
- check.status = "down"
-
- _log("\nSending notification(s) about going down for %s\n" % check.code)
- check.send_alert()
- ticks = 0
-
- # Save status after the notification is sent
- check.save()
-
- # Gone up?
- query = Check.objects
- query = query.filter(alert_after__gt=timezone.now())
- query = query.filter(user__isnull=False)
- query = query.filter(status="down")
- for check in query:
- check.status = "up"
-
- _log("\nSending notification(s) about going up for %s\n" % check.code)
- check.send_alert()
- ticks = 0
-
- # Save status after the notification is sent
- check.save()
+ success = True
+ while success:
+ success = handle_one()
+ ticks = 0 if success else ticks + 1
time.sleep(1)
- ticks = (ticks + 1) % 80
- _log("." + ("\n" if ticks == 0 else ""))
+ _stdout(".")
+ if ticks % 60 == 0:
+ _stdout("\n")
diff --git a/hc/api/migrations/0016_auto_20151030_1107.py b/hc/api/migrations/0016_auto_20151030_1107.py
new file mode 100644
index 00000000..5f527eec
--- /dev/null
+++ b/hc/api/migrations/0016_auto_20151030_1107.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('api', '0015_auto_20151022_1008'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='check',
+ name='status',
+ field=models.CharField(default='new', max_length=6, choices=[('up', 'Up'), ('down', 'Down'), ('new', 'New'), ('paused', 'Paused')]),
+ ),
+ ]
diff --git a/hc/api/models.py b/hc/api/models.py
index 1f6e8281..8fb48733 100644
--- a/hc/api/models.py
+++ b/hc/api/models.py
@@ -15,8 +15,12 @@ import requests
from hc.lib import emails
-
-STATUSES = (("up", "Up"), ("down", "Down"), ("new", "New"))
+STATUSES = (
+ ("up", "Up"),
+ ("down", "Down"),
+ ("new", "New"),
+ ("paused", "Paused")
+)
DEFAULT_TIMEOUT = td(days=1)
DEFAULT_GRACE = td(hours=1)
CHANNEL_KINDS = (("email", "Email"), ("webhook", "Webhook"),
@@ -60,8 +64,8 @@ class Check(models.Model):
channel.notify(self)
def get_status(self):
- if self.status == "new":
- return "new"
+ if self.status in ("new", "paused"):
+ return self.status
now = timezone.now()
diff --git a/static/css/base.css b/static/css/base.css
index 798f2681..0bc73002 100644
--- a/static/css/base.css
+++ b/static/css/base.css
@@ -42,7 +42,7 @@ body {
font-size: small;
}
-.glyphicon.up, .glyphicon.new, .glyphicon.grace, .glyphicon.down {
+.glyphicon.up, .glyphicon.new, .glyphicon.paused, .glyphicon.grace, .glyphicon.down {
font-size: 22px;
}
@@ -50,7 +50,7 @@ body {
color: #5cb85c;
}
-.glyphicon.new {
+.glyphicon.new, .glyphicon.paused {
color: #AAA;
}
diff --git a/templates/emails/alert-body-html.html b/templates/emails/alert-body-html.html
index 81f3997b..cc5c60ed 100644
--- a/templates/emails/alert-body-html.html
+++ b/templates/emails/alert-body-html.html
@@ -19,6 +19,7 @@
}
.new { background: #AAA; }
+ .paused { background: #AAA; }
.up { background: #5cb85c; }
.grace { background: #f0ad4e; }
.down { background: #d9534f; }
@@ -55,6 +56,8 @@
LATE
{% elif check.get_status == "down" %}
DOWN
+ {% elif check.get_status == "paused" %}
+ PAUSED
{% endif %}
|
diff --git a/templates/front/my_checks_desktop.html b/templates/front/my_checks_desktop.html
index 5e7715af..1d07442f 100644
--- a/templates/front/my_checks_desktop.html
+++ b/templates/front/my_checks_desktop.html
@@ -26,6 +26,8 @@
{% elif check.get_status == "down" %}
+ {% elif check.get_status == "paused" %}
+
{% endif %}
|
diff --git a/templates/front/my_checks_mobile.html b/templates/front/my_checks_mobile.html
index e9cad751..1d44e4bb 100644
--- a/templates/front/my_checks_mobile.html
+++ b/templates/front/my_checks_mobile.html
@@ -31,6 +31,8 @@
LATE
{% elif check.get_status == "down" %}
DOWN
+ {% elif check.get_status == "paused" %}
+ PAUSED
{% endif %}
|