show user count in main instance list

apiv1/serializers.py

@@ -1,4 +1,5 @@
 from rest_framework import serializers
+from collections import OrderedDict
 from scraper.models import Instance, InstanceStats
 
 
@@ -11,7 +12,15 @@ class InstanceStatsSerializer(serializers.ModelSerializer):
 class InstanceListSerializer(serializers.ModelSerializer):
     class Meta:
         model = Instance
-        fields = ('name', )
+        fields = ('name', 'user_count')
+
+    def to_representation(self, instance):
+        """
+        Object instance -> Dict of primitive datatypes.
+        """
+        ret = super(InstanceListSerializer, self).to_representation(instance)
+        ret = OrderedDict(list(filter(lambda x: x[1], ret.items())))
+        return ret
 
 
 class InstanceDetailSerializer(serializers.ModelSerializer):

scraper/management/commands/scrape.py

@@ -26,7 +26,7 @@ from scraper.management.commands._util import require_lock, InvalidResponseError
 
 # TODO: use the /api/v1/server/followers and /api/v1/server/following endpoints in peertube instances
 
-SEED = 'geekly.social'
+SEED = 'mastodon.social'
 TIMEOUT = 20
 
 
@@ -76,14 +76,21 @@ class Command(BaseCommand):
     @require_lock(Instance, 'ACCESS EXCLUSIVE')
     def save_data(self, data):
         """Save data"""
-        instance, _ = Instance.objects.get_or_create(name=get_key(data, ['instance']))
+        user_count = get_key(data, ['info', 'stats', 'user_count'])
+        if user_count:
+            instance, _ = Instance.objects.update_or_create(
+                name=get_key(data, ['instance']),
+                defaults={'user_count': user_count},
+            )
+        else:
+            instance, _ = Instance.objects.get_or_create(name=get_key(data, ['instance']))
         if data['status'] == 'success':
             # Save stats
             stats = InstanceStats(
                 instance=instance,
-                num_peers=get_key(data, ['info', 'stats', 'domain_count']),
-                num_statuses=get_key(data, ['info', 'stats', 'status_count']),
-                num_users=get_key(data, ['info', 'stats', 'user_count']),
+                domain_count=get_key(data, ['info', 'stats', 'domain_count']),
+                status_count=get_key(data, ['info', 'stats', 'status_count']),
+                user_count=get_key(data, ['info', 'stats', 'user_count']),
                 version=get_key(data, ['info', 'version']),
                 status=get_key(data, ['status']),
             )

scraper/migrations/0003_auto_20180828_2206.py

@@ -0,0 +1,33 @@
+# Generated by Django 2.1 on 2018-08-28 22:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scraper', '0002_auto_20180826_2201'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='instancestats',
+            old_name='num_peers',
+            new_name='domain_count',
+        ),
+        migrations.RenameField(
+            model_name='instancestats',
+            old_name='num_statuses',
+            new_name='status_count',
+        ),
+        migrations.RenameField(
+            model_name='instancestats',
+            old_name='num_users',
+            new_name='user_count',
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='user_count',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

scraper/models.py

@@ -4,17 +4,19 @@ from django.db import models
 class Instance(models.Model):
     name = models.CharField(max_length=200, primary_key=True)
     peers = models.ManyToManyField('self', symmetrical=False)
+    user_count = models.IntegerField(blank=True, null=True)
 
 
 class InstanceStats(models.Model):
+    # TODO: collect everything the API exposes
     timestamp = models.DateTimeField(auto_now_add=True)
     instance = models.ForeignKey(
         Instance,
         on_delete=models.CASCADE,
         related_name='stats',
     )
-    num_peers = models.IntegerField(blank=True, null=True)
-    num_statuses = models.IntegerField(blank=True, null=True)
-    num_users = models.IntegerField(blank=True, null=True)
+    domain_count = models.IntegerField(blank=True, null=True)
+    status_count = models.IntegerField(blank=True, null=True)
+    user_count = models.IntegerField(blank=True, null=True)
     version = models.CharField(max_length=1000, blank=True)
     status = models.CharField(max_length=100)