diff --git a/hc/api/management/commands/fillnpings.py b/hc/api/management/commands/fillnpings.py index 037f292f..27802ae3 100644 --- a/hc/api/management/commands/fillnpings.py +++ b/hc/api/management/commands/fillnpings.py @@ -1,13 +1,70 @@ +""" + +Populate api_check.n_pings and api_ping.n fields. + + - api_ping.n stores ping's serial number, counted separately for + each check. For example, if a particular check has received 100 pings, + its first ping will have a n=1, and the 100th ping will have a n=100. + + - api_check.n_pings stores the last serial number assigned to a ping. + It also is the total number of pings the check has ever received. + +This command works by "replaying" stored pings in their primary +key order, and counting up their serial numbers. At the very end, +api_check.n_pings fields are updated as well. + +Depending on the size of api_ping table, this command can potentially +take a long time to complete. + +Note on ping pruning: when the prunepings command is run, some of the +pings with the lowest serial numbers get removed. This doesn't affect +the "n" field for remaining pings, or the "n_pings" value of checks. +The serial numbers keep going up. + +""" + +import gc +from collections import Counter + from django.core.management.base import BaseCommand +from django.db import connection, transaction from hc.api.models import Check, Ping class Command(BaseCommand): - help = 'Fill check.n_pings field' + help = 'Fill check.n_pings field and ping.n field' def handle(self, *args, **options): - for check in Check.objects.all(): - check.n_pings = Ping.objects.filter(owner=check).count() - check.save(update_fields=("n_pings", )) + connection.use_debug_cursor = False + chunksize = 2000 + + # Reset all n_pings fields to zero + Check.objects.update(n_pings=0) + + counts = Counter() + + pk = 0 + last_pk = Ping.objects.order_by('-pk')[0].pk + queryset = Ping.objects.order_by('pk') + + transaction.set_autocommit(False) + while pk < last_pk: + for ping in queryset.filter(pk__gt=pk)[:chunksize]: + pk = ping.pk + counts[ping.owner_id] += 1 + + ping.n = counts[ping.owner_id] + ping.save(update_fields=("n", )) + + gc.collect() + progress = 100 * pk / last_pk + self.stdout.write("Processed ping id %d (%.2f%%)" % (pk, progress)) + + transaction.commit() + transaction.set_autocommit(True) + + self.stdout.write("Updating check.n_pings") + for check_id, n_pings in counts.items(): + Check.objects.filter(pk=check_id).update(n_pings=n_pings) return "Done!" diff --git a/hc/api/management/commands/prunepings.py b/hc/api/management/commands/prunepings.py index 96a4af5e..0050d6e8 100644 --- a/hc/api/management/commands/prunepings.py +++ b/hc/api/management/commands/prunepings.py @@ -2,30 +2,21 @@ from django.db.models import F from django.contrib.auth.models import User from django.core.management.base import BaseCommand from hc.accounts.models import Profile -from hc.api.models import Check +from hc.api.models import Ping class Command(BaseCommand): help = 'Prune pings based on limits in user profiles' def handle(self, *args, **options): - # Create any missing user profiles for user in User.objects.filter(profile=None): Profile.objects.for_user(user) - # Select checks having n_ping greater than the limit in user profile - checks = Check.objects - checks = checks.annotate(limit=F("user__profile__ping_log_limit")) - checks = checks.filter(n_pings__gt=F("limit")) - - total = 0 - for check in checks: - n = check.prune_pings(check.limit) - total += n - self.stdout.write("---") - self.stdout.write("User: %s" % check.user.email) - self.stdout.write("Check: %s" % check.name) - self.stdout.write("Pruned: %d" % n) + q = Ping.objects + q = q.annotate(limit=F("owner__user__profile__ping_log_limit")) + q = q.filter(n__lt=F("owner__n_pings") - F("limit")) + q = q.filter(n__gt=0) + n_pruned, _ = q.delete() - return "Done! Pruned %d pings." % total + return "Done! Pruned %d pings" % n_pruned diff --git a/hc/api/management/commands/prunepingsslow.py b/hc/api/management/commands/prunepingsslow.py new file mode 100644 index 00000000..87c1c5e1 --- /dev/null +++ b/hc/api/management/commands/prunepingsslow.py @@ -0,0 +1,35 @@ +from django.db.models import F +from django.contrib.auth.models import User +from django.core.management.base import BaseCommand +from hc.accounts.models import Profile +from hc.api.models import Check, Ping + + +class Command(BaseCommand): + help = """Prune pings based on limits in user profiles. + + This command prunes each check individually. So it does the work + in small chunks instead of a few big SQL queries like the `prunepings` + command. It is appropriate for initial pruning of the potentially + huge api_ping table. + + """ + + def handle(self, *args, **options): + # Create any missing user profiles + for user in User.objects.filter(profile=None): + Profile.objects.for_user(user) + + checks = Check.objects.annotate( + limit=F("user__profile__ping_log_limit")) + + for check in checks: + q = Ping.objects.filter(owner_id=check.id) + q = q.filter(n__lt=check.n_pings - check.limit) + q = q.filter(n__gt=0) + n_pruned, _ = q.delete() + + self.stdout.write("Pruned %d pings for check %s (%s)" % + (n_pruned, check.id, check.name)) + + return "Done!" diff --git a/hc/api/migrations/0021_ping_n.py b/hc/api/migrations/0021_ping_n.py new file mode 100644 index 00000000..c40ee518 --- /dev/null +++ b/hc/api/migrations/0021_ping_n.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-01-03 09:26 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0020_check_n_pings'), + ] + + operations = [ + migrations.AddField( + model_name='ping', + name='n', + field=models.IntegerField(null=True), + ), + ] diff --git a/hc/api/models.py b/hc/api/models.py index 5191508a..34673d68 100644 --- a/hc/api/models.py +++ b/hc/api/models.py @@ -94,35 +94,9 @@ class Check(models.Model): def tags_list(self): return self.tags.split(" ") - def prune_pings(self, keep_limit): - """ Prune pings for this check. - - If the check has more than `keep_limit` ping objects, prune the - oldest ones. Return the number of pruned pings. - - `keep_limit` specifies how many ping objects to keep. - - """ - - pings = Ping.objects.filter(owner=self).order_by("-id") - cutoff = pings[keep_limit:keep_limit+1] - - # If cutoff is empty slice then the check has less than `keep_limit` - # pings and there's nothing to prune yet. - if len(cutoff) == 0: - return 0 - - cutoff_id = cutoff[0].id - q = Ping.objects.filter(owner=self, id__lte=cutoff_id) - n_pruned, _ = q.delete() - - self.n_pings = keep_limit - self.save(update_fields=("n_pings", )) - - return n_pruned - class Ping(models.Model): + n = models.IntegerField(null=True) owner = models.ForeignKey(Check) created = models.DateTimeField(auto_now_add=True) scheme = models.CharField(max_length=10, default="http") diff --git a/hc/api/tests/test_check_model.py b/hc/api/tests/test_check_model.py deleted file mode 100644 index 2d37ed29..00000000 --- a/hc/api/tests/test_check_model.py +++ /dev/null @@ -1,21 +0,0 @@ -from django.test import TestCase -from hc.api.models import Check, Ping - - -class CheckModelTestCase(TestCase): - - def test_prune_pings(self): - check = Check() - check.save() - - for i in range(0, 6): - p = Ping(owner=check, ua="UA%d" % i) - p.save() - - check.prune_pings(keep_limit=3) - - self.assertEqual(check.n_pings, 3) - - ua_set = set(Ping.objects.values_list("ua", flat=True)) - # UA0, UA1, UA2 should have been pruned-- - self.assertEqual(ua_set, set(["UA3", "UA4", "UA5"])) diff --git a/hc/api/views.py b/hc/api/views.py index bf729902..372129bb 100644 --- a/hc/api/views.py +++ b/hc/api/views.py @@ -23,9 +23,11 @@ def ping(request, code): check.status = "up" check.save() + check.refresh_from_db() ping = Ping(owner=check) headers = request.META + ping.n = check.n_pings ping.remote_addr = headers.get("HTTP_X_REAL_IP", headers["REMOTE_ADDR"]) ping.scheme = headers.get("HTTP_X_SCHEME", "http") ping.method = headers["REQUEST_METHOD"] diff --git a/hc/front/views.py b/hc/front/views.py index dcd007cf..c2bacc06 100644 --- a/hc/front/views.py +++ b/hc/front/views.py @@ -194,8 +194,12 @@ def log(request, code): profile = Profile.objects.for_user(request.user) limit = profile.ping_log_limit - pings = Ping.objects.filter(owner=check).order_by("created")[:limit] + pings = Ping.objects.filter(owner=check).order_by("-id")[:limit] + pings = list(pings) + # oldest-to-newest order will be more convenient for adding + # "not received" placeholders: + pings.reverse() # Add a dummy ping object at the end. We iterate over *pairs* of pings # and don't want to handle a special case of a check with a single ping. diff --git a/static/css/log.css b/static/css/log.css index 9327a0ca..b3c72cf7 100644 --- a/static/css/log.css +++ b/static/css/log.css @@ -44,3 +44,11 @@ background: #fff3f2; } +#log .n-cell { + text-align: center; + font-family: monospace; +} + +#log .hash { + color: #aaa; +} \ No newline at end of file diff --git a/templates/front/log.html b/templates/front/log.html index e5afd369..ea89c295 100644 --- a/templates/front/log.html +++ b/templates/front/log.html @@ -35,8 +35,10 @@ {% for record in pings %} {% if record.ping %} - - + + {% if record.ping.n %} + #{{ record.ping.n }} + {% endif %}
@@ -65,7 +67,7 @@ {% endif %} {% if record.placeholder_date %} - +