From de203275c6e74708334950f8cc0d774874ee499a Mon Sep 17 00:00:00 2001
From: Di Wu <dw323@cornell.edu>
Date: Fri, 30 Oct 2015 23:35:24 -0700
Subject: [PATCH 1/2] sendalerts concurrently

---
 hc/api/management/__init__.py            |  0
 hc/api/management/commands/__init__.py   |  0
 hc/api/management/commands/sendalerts.py | 42 ++++++++++++++----------
 requirements.txt                         |  7 ++--
 4 files changed, 29 insertions(+), 20 deletions(-)
 create mode 100644 hc/api/management/__init__.py
 create mode 100644 hc/api/management/commands/__init__.py

diff --git a/hc/api/management/__init__.py b/hc/api/management/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/hc/api/management/commands/__init__.py b/hc/api/management/commands/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/hc/api/management/commands/sendalerts.py b/hc/api/management/commands/sendalerts.py
index f81044f8..080fcfd6 100644
--- a/hc/api/management/commands/sendalerts.py
+++ b/hc/api/management/commands/sendalerts.py
@@ -2,12 +2,14 @@ import logging
 import sys
 import time
 
+from concurrent.futures import ThreadPoolExecutor
 from django.core.management.base import BaseCommand
+from django.db import connection
 from django.db.models import Q
 from django.utils import timezone
-
 from hc.api.models import Check
 
+executor = ThreadPoolExecutor(max_workers=10)
 logger = logging.getLogger(__name__)
 
 
@@ -16,26 +18,31 @@ def _stdout(message):
     sys.stdout.flush()
 
 
-def handle_one():
-    """ Send an alert for a single check.
-
-    Return True if an appropriate check was selected and processed.
-    Return False if no checks need to be processed.
-
-    """
-
+def handle_many():
+    """ Send alerts for many checks simultaneously. """
     query = Check.objects.filter(user__isnull=False)
 
     now = timezone.now()
     going_down = Q(alert_after__lt=now, status="up")
     going_up = Q(alert_after__gt=now, status="down")
     query = query.filter(going_down | going_up)
-
-    try:
-        check = query[0]
-    except IndexError:
+    checks = list(query.iterator())
+    if not checks:
         return False
 
+    for future in [executor.submit(handle_one, check) for check in checks]:
+        future.result()
+
+    return True
+
+
+def handle_one(check):
+    """ Send an alert for a single check.
+
+    Return True if an appropriate check was selected and processed.
+    Return False if no checks need to be processed.
+
+    """
     check.status = check.get_status()
 
     tmpl = "\nSending alert, status=%s, code=%s\n"
@@ -54,6 +61,7 @@ def handle_one():
         check.status = "paused"
     finally:
         check.save()
+        connection.close()
 
     return True
 
@@ -65,10 +73,10 @@ class Command(BaseCommand):
 
         ticks = 0
         while True:
-            success = True
-            while success:
-                success = handle_one()
-                ticks = 0 if success else ticks + 1
+            if handle_many():
+                ticks = 0
+            else:
+                ticks += 1
 
             time.sleep(1)
             _stdout(".")
diff --git a/requirements.txt b/requirements.txt
index e236d131..8cbfbf76 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,10 @@
-Django==1.8.2
-django_compressor
 django-appconf
 django-ses-backend
-psycopg2==2.6
+Django==1.8.2
+django_compressor
 djmail
+futures
 premailer
+psycopg2==2.6
 pygments
 requests

From 93c71ad10ee485a675fde536d41d9721bd55b3c2 Mon Sep 17 00:00:00 2001
From: Di Wu <dw323@cornell.edu>
Date: Fri, 30 Oct 2015 23:47:07 -0700
Subject: [PATCH 2/2] timeout on all the notifications

---
 hc/api/models.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hc/api/models.py b/hc/api/models.py
index 8fb48733..7e35b055 100644
--- a/hc/api/models.py
+++ b/hc/api/models.py
@@ -143,7 +143,7 @@ class Channel(models.Model):
                 "icon_url": "https://healthchecks.io/static/img/logo@2x.png"
             }
 
-            r = requests.post(self.value, json=payload)
+            r = requests.post(self.value, json=payload, timeout=5)
 
             n.status = r.status_code
             n.save()
@@ -155,7 +155,7 @@ class Channel(models.Model):
                 "color": "green" if check.status == "up" else "red",
             }
 
-            r = requests.post(self.value, json=payload)
+            r = requests.post(self.value, json=payload, timeout=5)
 
             n.status = r.status_code
             n.save()
@@ -179,7 +179,7 @@ class Channel(models.Model):
             }
 
             url = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
-            r = requests.post(url, data=json.dumps(payload))
+            r = requests.post(url, data=json.dumps(payload), timeout=5)
 
             n.status = r.status_code
             n.save()