Merge pull request 'feature(backups): a task to autorefresh cache. Redis expiry abolished' (#58) from recache-task into master

Reviewed-on: https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api/pulls/58
Reviewed-by: Inex Code <inex.code@selfprivacy.org>

selfprivacy_api/backup/storage.py

@@ -16,9 +16,6 @@ from selfprivacy_api.utils.redis_model_storage import (
 from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
 from selfprivacy_api.backup.providers import get_kind
 
-# a hack to store file path.
-REDIS_SNAPSHOT_CACHE_EXPIRE_SECONDS = 24 * 60 * 60  # one day
-
 REDIS_SNAPSHOTS_PREFIX = "backups:snapshots:"
 REDIS_LAST_BACKUP_PREFIX = "backups:last-backed-up:"
 REDIS_INITTED_CACHE = "backups:repo_initted"
@@ -26,7 +23,6 @@ REDIS_INITTED_CACHE = "backups:repo_initted"
 REDIS_PROVIDER_KEY = "backups:provider"
 REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period"
 
-
 redis = RedisPool().get_connection()
 
 
@@ -89,7 +85,6 @@ class Storage:
         """Stores snapshot metadata in redis for caching purposes"""
         snapshot_key = Storage.__snapshot_key(snapshot)
         store_model_as_hash(redis, snapshot_key, snapshot)
-        redis.expire(snapshot_key, REDIS_SNAPSHOT_CACHE_EXPIRE_SECONDS)
 
     @staticmethod
     def delete_cached_snapshot(snapshot: Snapshot) -> None:

selfprivacy_api/backup/tasks.py

@@ -7,13 +7,17 @@ from selfprivacy_api.graphql.common_types.backup import RestoreStrategy
 
 from selfprivacy_api.models.backup.snapshot import Snapshot
 from selfprivacy_api.utils.huey import huey
+from huey import crontab
 from selfprivacy_api.services.service import Service
 from selfprivacy_api.backup import Backups
 
+SNAPSHOT_CACHE_TTL_HOURS = 6
+
 
 def validate_datetime(dt: datetime) -> bool:
     """
-    Validates that the datetime passed in is timezone-aware.
+    Validates that it is time to back up.
+    Also ensures that the timezone-aware time is used.
     """
     if dt.tzinfo is None:
         return Backups.is_time_to_backup(dt.replace(tzinfo=timezone.utc))
@@ -50,3 +54,8 @@ def automatic_backup():
     time = datetime.utcnow().replace(tzinfo=timezone.utc)
     for service in Backups.services_to_back_up(time):
         start_backup(service)
+
+
+@huey.periodic_task(crontab(hour=SNAPSHOT_CACHE_TTL_HOURS))
+def reload_snapshot_cache():
+    Backups.force_snapshot_cache_reload()

tests/test_graphql/test_backup.py

@@ -32,7 +32,11 @@ from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
 from selfprivacy_api.backup.jobs import add_backup_job, add_restore_job
 
 
-from selfprivacy_api.backup.tasks import start_backup, restore_snapshot
+from selfprivacy_api.backup.tasks import (
+    start_backup,
+    restore_snapshot,
+    reload_snapshot_cache,
+)
 from selfprivacy_api.backup.storage import Storage
 from selfprivacy_api.backup.jobs import get_backup_job
 
@@ -806,3 +810,16 @@ def test_tempfile():
     with tempfile.TemporaryDirectory() as temp:
         assert path.exists(temp)
     assert not path.exists(temp)
+
+
+# Storage
+def test_cache_invalidaton_task(backups, dummy_service):
+    Backups.back_up(dummy_service)
+    assert len(Storage.get_cached_snapshots()) == 1
+
+    # Does not trigger resync
+    Storage.invalidate_snapshot_storage()
+    assert Storage.get_cached_snapshots() == []
+
+    reload_snapshot_cache()
+    assert len(Storage.get_cached_snapshots()) == 1