From e52f581067e6f518b89a5ad41886bb93eb28d33d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tao=20Bror=20Bojl=C3=A9n?= Date: Thu, 28 Feb 2019 18:06:16 +0000 Subject: [PATCH] bump personal instance threshold to 10 --- backend/scraper/management/commands/scrape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/scraper/management/commands/scrape.py b/backend/scraper/management/commands/scrape.py index 373afdd..ddd0fea 100644 --- a/backend/scraper/management/commands/scrape.py +++ b/backend/scraper/management/commands/scrape.py @@ -24,7 +24,7 @@ from scraper.management.commands._util import require_lock, InvalidResponseExcep SEED = 'mastodon.social' TIMEOUT = 20 # seconds NUM_THREADS = 16 # roughly 40MB each -PERSONAL_INSTANCE_THRESHOLD = 5 # instances with < this many users won't be crawled +PERSONAL_INSTANCE_THRESHOLD = 10 # instances with < this many users won't be crawled MAX_STATUSES_PER_PAGE = 100 STATUS_SCRAPE_LIMIT = 5000 INSTANCE_SCRAPE_LIMIT = 50 # note: this does not include newly discovered instances! they will always be crawled.