diff --git a/backend/scraper/management/commands/scrape.py b/backend/scraper/management/commands/scrape.py index 373afdd..ddd0fea 100644 --- a/backend/scraper/management/commands/scrape.py +++ b/backend/scraper/management/commands/scrape.py @@ -24,7 +24,7 @@ from scraper.management.commands._util import require_lock, InvalidResponseExcep SEED = 'mastodon.social' TIMEOUT = 20 # seconds NUM_THREADS = 16 # roughly 40MB each -PERSONAL_INSTANCE_THRESHOLD = 5 # instances with < this many users won't be crawled +PERSONAL_INSTANCE_THRESHOLD = 10 # instances with < this many users won't be crawled MAX_STATUSES_PER_PAGE = 100 STATUS_SCRAPE_LIMIT = 5000 INSTANCE_SCRAPE_LIMIT = 50 # note: this does not include newly discovered instances! they will always be crawled.