selfprivacy-rest-api/tests/test_graphql/test_backup.py

import pytest
import os
import os.path as path
from os import makedirs
from os import remove
from os import listdir
from os import urandom
from datetime import datetime, timedelta, timezone, date, time
from subprocess import Popen
from copy import copy

import secrets


import tempfile

from selfprivacy_api.utils.huey import huey

import selfprivacy_api.services as services
from selfprivacy_api.services import Service, get_all_services
from selfprivacy_api.services.service import ServiceStatus

from selfprivacy_api.services import get_service_by_id
from selfprivacy_api.services.test_service import DummyService
from selfprivacy_api.graphql.queries.providers import BackupProvider
from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason
from selfprivacy_api.jobs import Jobs, JobStatus

from selfprivacy_api.models.backup.snapshot import Snapshot

from selfprivacy_api.graphql.common_types.backup import AutobackupQuotas

from selfprivacy_api.backup import Backups, BACKUP_PROVIDER_ENVS
import selfprivacy_api.backup.providers as providers
from selfprivacy_api.backup.providers import AbstractBackupProvider
from selfprivacy_api.backup.providers.backblaze import Backblaze
from selfprivacy_api.backup.providers.none import NoBackups
from selfprivacy_api.backup.util import sync
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
from selfprivacy_api.backup.jobs import add_backup_job, add_restore_job


from selfprivacy_api.backup.tasks import (
    start_backup,
    restore_snapshot,
    reload_snapshot_cache,
)
from selfprivacy_api.backup.storage import Storage
from selfprivacy_api.backup.jobs import get_backup_job


TESTFILE_BODY = "testytest!"
TESTFILE_2_BODY = "testissimo!"
REPO_NAME = "test_backup"


def prepare_localfile_backups(temp_dir):
    test_repo_path = path.join(temp_dir, "totallyunrelated")
    assert not path.exists(test_repo_path)
    Backups.set_localfile_repo(test_repo_path)


@pytest.fixture(scope="function")
def backups_local(tmpdir):
    Backups.reset()
    prepare_localfile_backups(tmpdir)
    Jobs.reset()
    Backups.init_repo()


@pytest.fixture(scope="function")
def backups(tmpdir):
    # for those tests that are supposed to pass with any repo
    Backups.reset()
    if BACKUP_PROVIDER_ENVS["kind"] in os.environ.keys():
        Backups.set_provider_from_envs()
    else:
        prepare_localfile_backups(tmpdir)
    Jobs.reset()
    # assert not repo_path

    Backups.init_repo()
    yield
    Backups.erase_repo()


@pytest.fixture()
def backups_backblaze(generic_userdata):
    Backups.reset(reset_json=False)


@pytest.fixture()
def raw_dummy_service(tmpdir):
    dirnames = ["test_service", "also_test_service"]
    service_dirs = []
    for d in dirnames:
        service_dir = path.join(tmpdir, d)
        makedirs(service_dir)
        service_dirs.append(service_dir)

    testfile_path_1 = path.join(service_dirs[0], "testfile.txt")
    with open(testfile_path_1, "w") as file:
        file.write(TESTFILE_BODY)

    testfile_path_2 = path.join(service_dirs[1], "testfile2.txt")
    with open(testfile_path_2, "w") as file:
        file.write(TESTFILE_2_BODY)

    # we need this to not change get_folders() much
    class TestDummyService(DummyService, folders=service_dirs):
        pass

    service = TestDummyService()
    return service


@pytest.fixture()
def dummy_service(tmpdir, backups, raw_dummy_service) -> Service:
    service = raw_dummy_service

    # register our service
    services.services.append(service)

    # make sure we are in immediate mode because this thing is non pickleable to store on queue.
    huey.immediate = True
    assert huey.immediate is True

    assert get_service_by_id(service.get_id()) is not None
    yield service

    # cleanup because apparently it matters wrt tasks
    services.services.remove(service)


@pytest.fixture()
def memory_backup() -> AbstractBackupProvider:
    ProviderClass = providers.get_provider(BackupProvider.MEMORY)
    assert ProviderClass is not None
    memory_provider = ProviderClass(login="", key="")
    assert memory_provider is not None
    return memory_provider


@pytest.fixture()
def file_backup(tmpdir) -> AbstractBackupProvider:
    test_repo_path = path.join(tmpdir, "test_repo")
    ProviderClass = providers.get_provider(BackupProvider.FILE)
    assert ProviderClass is not None
    provider = ProviderClass(location=test_repo_path)
    assert provider is not None
    return provider


def test_config_load(generic_userdata):
    Backups.reset(reset_json=False)
    provider = Backups.provider()

    assert provider is not None
    assert isinstance(provider, Backblaze)
    assert provider.login == "ID"
    assert provider.key == "KEY"
    assert provider.location == "selfprivacy"

    assert provider.backupper.account == "ID"
    assert provider.backupper.key == "KEY"


def test_reset_sets_to_none1():
    Backups.reset()
    provider = Backups.provider()
    assert provider is not None
    assert isinstance(provider, NoBackups)


def test_reset_sets_to_none2(backups):
    # now with something set up first^^^
    Backups.reset()
    provider = Backups.provider()
    assert provider is not None
    assert isinstance(provider, NoBackups)


def test_setting_from_envs(tmpdir):
    Backups.reset()
    environment_stash = {}
    if BACKUP_PROVIDER_ENVS["kind"] in os.environ.keys():
        # we are running under special envs, stash them before rewriting them
        for key in BACKUP_PROVIDER_ENVS.values():
            environment_stash[key] = os.environ[key]

    os.environ[BACKUP_PROVIDER_ENVS["kind"]] = "BACKBLAZE"
    os.environ[BACKUP_PROVIDER_ENVS["login"]] = "ID"
    os.environ[BACKUP_PROVIDER_ENVS["key"]] = "KEY"
    os.environ[BACKUP_PROVIDER_ENVS["location"]] = "selfprivacy"
    Backups.set_provider_from_envs()
    provider = Backups.provider()

    assert provider is not None
    assert isinstance(provider, Backblaze)
    assert provider.login == "ID"
    assert provider.key == "KEY"
    assert provider.location == "selfprivacy"

    assert provider.backupper.account == "ID"
    assert provider.backupper.key == "KEY"

    if environment_stash != {}:
        for key in BACKUP_PROVIDER_ENVS.values():
            os.environ[key] = environment_stash[key]
    else:
        for key in BACKUP_PROVIDER_ENVS.values():
            del os.environ[key]


def test_json_reset(generic_userdata):
    Backups.reset(reset_json=False)
    provider = Backups.provider()
    assert provider is not None
    assert isinstance(provider, Backblaze)
    assert provider.login == "ID"
    assert provider.key == "KEY"
    assert provider.location == "selfprivacy"

    Backups.reset()
    provider = Backups.provider()
    assert provider is not None
    assert isinstance(provider, AbstractBackupProvider)
    assert provider.login == ""
    assert provider.key == ""
    assert provider.location == ""
    assert provider.repo_id == ""


def test_select_backend():
    provider = providers.get_provider(BackupProvider.BACKBLAZE)
    assert provider is not None
    assert provider == Backblaze


def test_file_backend_init(file_backup):
    file_backup.backupper.init()


def test_reinit_after_purge(backups):
    assert Backups.is_initted() is True

    Backups.erase_repo()
    assert Backups.is_initted() is False
    with pytest.raises(ValueError):
        Backups.get_all_snapshots()

    Backups.init_repo()
    assert Backups.is_initted() is True
    assert len(Backups.get_all_snapshots()) == 0


def test_backup_simple_file(raw_dummy_service, file_backup):
    # temporarily incomplete
    service = raw_dummy_service
    assert service is not None
    assert file_backup is not None

    name = service.get_id()
    file_backup.backupper.init()


def test_backup_service(dummy_service, backups):
    id = dummy_service.get_id()
    assert_job_finished(f"services.{id}.backup", count=0)
    assert Backups.get_last_backed_up(dummy_service) is None

    Backups.back_up(dummy_service)

    now = datetime.now(timezone.utc)
    date = Backups.get_last_backed_up(dummy_service)
    assert date is not None
    assert now > date
    assert now - date < timedelta(minutes=1)

    assert_job_finished(f"services.{id}.backup", count=1)


def test_no_repo(memory_backup):
    with pytest.raises(ValueError):
        assert memory_backup.backupper.get_snapshots() == []


def test_one_snapshot(backups, dummy_service):
    Backups.back_up(dummy_service)

    snaps = Backups.get_snapshots(dummy_service)
    assert len(snaps) == 1
    snap = snaps[0]
    assert snap.service_name == dummy_service.get_id()


def test_backup_returns_snapshot(backups, dummy_service):
    service_folders = dummy_service.get_folders()
    provider = Backups.provider()
    name = dummy_service.get_id()
    snapshot = provider.backupper.start_backup(service_folders, name)

    assert snapshot.id is not None
    assert len(snapshot.id) == len(Backups.get_all_snapshots()[0].id)
    assert Backups.get_snapshot_by_id(snapshot.id) is not None
    assert snapshot.service_name == name
    assert snapshot.created_at is not None
    assert snapshot.reason == BackupReason.EXPLICIT


def test_backup_reasons(backups, dummy_service):
    snap = Backups.back_up(dummy_service, BackupReason.AUTO)
    assert snap.reason == BackupReason.AUTO

    Backups.force_snapshot_cache_reload()
    snaps = Backups.get_snapshots(dummy_service)
    assert snaps[0].reason == BackupReason.AUTO


unlimited_quotas = AutobackupQuotas(
    last=-1,
    daily=-1,
    weekly=-1,
    monthly=-1,
    yearly=-1,
)

zero_quotas = AutobackupQuotas(
    last=0,
    daily=0,
    weekly=0,
    monthly=0,
    yearly=0,
)


def test_get_empty_quotas(backups):
    quotas = Backups.autobackup_quotas()
    assert quotas is not None
    assert quotas == unlimited_quotas


def test_set_quotas(backups):
    quotas = AutobackupQuotas(
        last=3,
        daily=2343,
        weekly=343,
        monthly=0,
        yearly=-34556,
    )
    Backups.set_autobackup_quotas(quotas)
    assert Backups.autobackup_quotas() == AutobackupQuotas(
        last=3,
        daily=2343,
        weekly=343,
        monthly=0,
        yearly=-1,
    )


def test_set_zero_quotas(backups):
    quotas = AutobackupQuotas(
        last=0,
        daily=0,
        weekly=0,
        monthly=0,
        yearly=0,
    )
    Backups.set_autobackup_quotas(quotas)
    assert Backups.autobackup_quotas() == zero_quotas


def test_set_unlimited_quotas(backups):
    quotas = AutobackupQuotas(
        last=-1,
        daily=-1,
        weekly=-1,
        monthly=-1,
        yearly=-1,
    )
    Backups.set_autobackup_quotas(quotas)
    assert Backups.autobackup_quotas() == unlimited_quotas


def test_set_zero_quotas_after_unlimited(backups):
    quotas = AutobackupQuotas(
        last=-1,
        daily=-1,
        weekly=-1,
        monthly=-1,
        yearly=-1,
    )
    Backups.set_autobackup_quotas(quotas)
    assert Backups.autobackup_quotas() == unlimited_quotas

    quotas = AutobackupQuotas(
        last=0,
        daily=0,
        weekly=0,
        monthly=0,
        yearly=0,
    )
    Backups.set_autobackup_quotas(quotas)
    assert Backups.autobackup_quotas() == zero_quotas


def dummy_snapshot(date: datetime):
    return Snapshot(
        id=str(hash(date)),
        service_name="someservice",
        created_at=date,
        reason=BackupReason.EXPLICIT,
    )


def test_autobackup_snapshots_pruning(backups):
    # Wednesday, fourth week
    now = datetime(year=2023, month=1, day=25, hour=10)

    snaps = [
        dummy_snapshot(now),
        dummy_snapshot(now - timedelta(minutes=5)),
        dummy_snapshot(now - timedelta(hours=2)),
        dummy_snapshot(now - timedelta(hours=5)),
        dummy_snapshot(now - timedelta(days=1)),
        dummy_snapshot(now - timedelta(days=1, hours=2)),
        dummy_snapshot(now - timedelta(days=1, hours=3)),
        dummy_snapshot(now - timedelta(days=2)),
        dummy_snapshot(now - timedelta(days=7)),
        dummy_snapshot(now - timedelta(days=12)),
        dummy_snapshot(now - timedelta(days=23)),
        dummy_snapshot(now - timedelta(days=28)),
        dummy_snapshot(now - timedelta(days=32)),
        dummy_snapshot(now - timedelta(days=47)),
        dummy_snapshot(now - timedelta(days=64)),
        dummy_snapshot(now - timedelta(days=84)),
        dummy_snapshot(now - timedelta(days=104)),
        dummy_snapshot(now - timedelta(days=365 * 2)),
    ]
    old_len = len(snaps)

    quotas = copy(unlimited_quotas)
    Backups.set_autobackup_quotas(quotas)
    assert Backups._prune_snaps_with_quotas(snaps) == snaps

    quotas = copy(zero_quotas)
    quotas.last = 2
    quotas.daily = 2
    Backups.set_autobackup_quotas(quotas)

    snaps_to_keep = Backups._prune_snaps_with_quotas(snaps)
    assert snaps_to_keep == [
        dummy_snapshot(now),
        dummy_snapshot(now - timedelta(minutes=5)),
        # dummy_snapshot(now - timedelta(hours=2)),
        # dummy_snapshot(now - timedelta(hours=5)),
        dummy_snapshot(now - timedelta(days=1)),
        # dummy_snapshot(now - timedelta(days=1, hours=2)),
        # dummy_snapshot(now - timedelta(days=1, hours=3)),
        # dummy_snapshot(now - timedelta(days=2)),
        # dummy_snapshot(now - timedelta(days=7)),
        # dummy_snapshot(now - timedelta(days=12)),
        # dummy_snapshot(now - timedelta(days=23)),
        # dummy_snapshot(now - timedelta(days=28)),
        # dummy_snapshot(now - timedelta(days=32)),
        # dummy_snapshot(now - timedelta(days=47)),
        # dummy_snapshot(now - timedelta(days=64)),
        # dummy_snapshot(now - timedelta(days=84)),
        # dummy_snapshot(now - timedelta(days=104)),
        # dummy_snapshot(now - timedelta(days=365 * 2)),
    ]

    # checking that this function does not mutate the argument
    assert snaps != snaps_to_keep
    assert len(snaps) == old_len

    quotas = copy(zero_quotas)
    quotas.weekly = 4
    Backups.set_autobackup_quotas(quotas)

    snaps_to_keep = Backups._prune_snaps_with_quotas(snaps)
    assert snaps_to_keep == [
        dummy_snapshot(now),
        # dummy_snapshot(now - timedelta(minutes=5)),
        # dummy_snapshot(now - timedelta(hours=2)),
        # dummy_snapshot(now - timedelta(hours=5)),
        # dummy_snapshot(now - timedelta(days=1)),
        # dummy_snapshot(now - timedelta(days=1, hours=2)),
        # dummy_snapshot(now - timedelta(days=1, hours=3)),
        # dummy_snapshot(now - timedelta(days=2)),
        dummy_snapshot(now - timedelta(days=7)),
        dummy_snapshot(now - timedelta(days=12)),
        dummy_snapshot(now - timedelta(days=23)),
        # dummy_snapshot(now - timedelta(days=28)),
        # dummy_snapshot(now - timedelta(days=32)),
        # dummy_snapshot(now - timedelta(days=47)),
        # dummy_snapshot(now - timedelta(days=64)),
        # dummy_snapshot(now - timedelta(days=84)),
        # dummy_snapshot(now - timedelta(days=104)),
        # dummy_snapshot(now - timedelta(days=365 * 2)),
    ]

    quotas = copy(zero_quotas)
    quotas.monthly = 7
    Backups.set_autobackup_quotas(quotas)

    snaps_to_keep = Backups._prune_snaps_with_quotas(snaps)
    assert snaps_to_keep == [
        dummy_snapshot(now),
        # dummy_snapshot(now - timedelta(minutes=5)),
        # dummy_snapshot(now - timedelta(hours=2)),
        # dummy_snapshot(now - timedelta(hours=5)),
        # dummy_snapshot(now - timedelta(days=1)),
        # dummy_snapshot(now - timedelta(days=1, hours=2)),
        # dummy_snapshot(now - timedelta(days=1, hours=3)),
        # dummy_snapshot(now - timedelta(days=2)),
        # dummy_snapshot(now - timedelta(days=7)),
        # dummy_snapshot(now - timedelta(days=12)),
        # dummy_snapshot(now - timedelta(days=23)),
        dummy_snapshot(now - timedelta(days=28)),
        # dummy_snapshot(now - timedelta(days=32)),
        # dummy_snapshot(now - timedelta(days=47)),
        dummy_snapshot(now - timedelta(days=64)),
        # dummy_snapshot(now - timedelta(days=84)),
        dummy_snapshot(now - timedelta(days=104)),
        dummy_snapshot(now - timedelta(days=365 * 2)),
    ]


def test_autobackup_snapshots_pruning_yearly(backups):
    snaps = [
        dummy_snapshot(datetime(year=2055, month=3, day=1)),
        dummy_snapshot(datetime(year=2055, month=2, day=1)),
        dummy_snapshot(datetime(year=2023, month=4, day=1)),
        dummy_snapshot(datetime(year=2023, month=3, day=1)),
        dummy_snapshot(datetime(year=2023, month=2, day=1)),
        dummy_snapshot(datetime(year=2021, month=2, day=1)),
    ]
    quotas = copy(zero_quotas)
    quotas.yearly = 2
    Backups.set_autobackup_quotas(quotas)

    snaps_to_keep = Backups._prune_snaps_with_quotas(snaps)
    assert snaps_to_keep == [
        dummy_snapshot(datetime(year=2055, month=3, day=1)),
        dummy_snapshot(datetime(year=2023, month=4, day=1)),
    ]


def test_autobackup_snapshots_pruning_bottleneck(backups):
    now = datetime(year=2023, month=1, day=25, hour=10)
    snaps = [
        dummy_snapshot(now),
        dummy_snapshot(now - timedelta(minutes=5)),
        dummy_snapshot(now - timedelta(hours=2)),
        dummy_snapshot(now - timedelta(hours=3)),
        dummy_snapshot(now - timedelta(hours=4)),
    ]

    yearly_quota = copy(zero_quotas)
    yearly_quota.yearly = 2

    monthly_quota = copy(zero_quotas)
    monthly_quota.monthly = 2

    weekly_quota = copy(zero_quotas)
    weekly_quota.weekly = 2

    daily_quota = copy(zero_quotas)
    daily_quota.daily = 2

    last_quota = copy(zero_quotas)
    last_quota.last = 1
    last_quota.yearly = 2

    for quota in [last_quota, yearly_quota, monthly_quota, weekly_quota, daily_quota]:
        print(quota)
        Backups.set_autobackup_quotas(quota)
        snaps_to_keep = Backups._prune_snaps_with_quotas(snaps)
        assert snaps_to_keep == [
            dummy_snapshot(now),
            # If there is a vacant quota, we should keep the last snapshot even if it doesn't fit
            dummy_snapshot(now - timedelta(hours=4)),
        ]


def test_autobackup_snapshots_pruning_edgeweek(backups):
    # jan 1 2023 is Sunday
    snaps = [
        dummy_snapshot(datetime(year=2023, month=1, day=6)),
        dummy_snapshot(datetime(year=2023, month=1, day=1)),
        dummy_snapshot(datetime(year=2022, month=12, day=31)),
        dummy_snapshot(datetime(year=2022, month=12, day=30)),
    ]
    quotas = copy(zero_quotas)
    quotas.weekly = 2
    Backups.set_autobackup_quotas(quotas)

    snaps_to_keep = Backups._prune_snaps_with_quotas(snaps)
    assert snaps_to_keep == [
        dummy_snapshot(datetime(year=2023, month=1, day=6)),
        dummy_snapshot(datetime(year=2023, month=1, day=1)),
    ]


def test_autobackup_snapshots_pruning_big_gap(backups):
    snaps = [
        dummy_snapshot(datetime(year=2023, month=1, day=6)),
        dummy_snapshot(datetime(year=2023, month=1, day=2)),
        dummy_snapshot(datetime(year=2022, month=10, day=31)),
        dummy_snapshot(datetime(year=2022, month=10, day=30)),
    ]
    quotas = copy(zero_quotas)
    quotas.weekly = 2
    Backups.set_autobackup_quotas(quotas)

    snaps_to_keep = Backups._prune_snaps_with_quotas(snaps)
    assert snaps_to_keep == [
        dummy_snapshot(datetime(year=2023, month=1, day=6)),
        dummy_snapshot(datetime(year=2022, month=10, day=31)),
    ]


def test_too_many_auto(backups, dummy_service):
    assert Backups.autobackup_quotas()
    quota = copy(zero_quotas)
    quota.last = 2
    Backups.set_autobackup_quotas(quota)
    assert Backups.autobackup_quotas().last == 2

    snap = Backups.back_up(dummy_service, BackupReason.AUTO)
    assert len(Backups.get_snapshots(dummy_service)) == 1
    snap2 = Backups.back_up(dummy_service, BackupReason.AUTO)
    assert len(Backups.get_snapshots(dummy_service)) == 2
    snap3 = Backups.back_up(dummy_service, BackupReason.AUTO)
    assert len(Backups.get_snapshots(dummy_service)) == 2

    snaps = Backups.get_snapshots(dummy_service)
    assert snap2 in snaps
    assert snap3 in snaps
    assert snap not in snaps

    quota.last = -1
    Backups.set_autobackup_quotas(quota)
    snap4 = Backups.back_up(dummy_service, BackupReason.AUTO)

    snaps = Backups.get_snapshots(dummy_service)
    assert len(snaps) == 3
    assert snap4 in snaps

    # Retroactivity
    quota.last = 1
    Backups.set_autobackup_quotas(quota)
    Backups.prune_all_autosnaps()
    snaps = Backups.get_snapshots(dummy_service)
    assert len(snaps) == 1

    snap5 = Backups.back_up(dummy_service, BackupReason.AUTO)
    snaps = Backups.get_snapshots(dummy_service)
    assert len(snaps) == 1
    assert snap5 in snaps

    # Explicit snaps are not affected
    snap6 = Backups.back_up(dummy_service, BackupReason.EXPLICIT)

    snaps = Backups.get_snapshots(dummy_service)
    assert len(snaps) == 2
    assert snap5 in snaps
    assert snap6 in snaps


def folder_files(folder):
    return [
        path.join(folder, filename)
        for filename in listdir(folder)
        if filename is not None
    ]


def service_files(service):
    result = []
    for service_folder in service.get_folders():
        result.extend(folder_files(service_folder))
    return result


def test_restore(backups, dummy_service):
    paths_to_nuke = service_files(dummy_service)
    contents = []

    for service_file in paths_to_nuke:
        with open(service_file, "r") as file:
            contents.append(file.read())

    Backups.back_up(dummy_service)
    snap = Backups.get_snapshots(dummy_service)[0]
    assert snap is not None

    for p in paths_to_nuke:
        assert path.exists(p)
        remove(p)
        assert not path.exists(p)

    Backups._restore_service_from_snapshot(dummy_service, snap.id)
    for p, content in zip(paths_to_nuke, contents):
        assert path.exists(p)
        with open(p, "r") as file:
            assert file.read() == content


def test_sizing(backups, dummy_service):
    Backups.back_up(dummy_service)
    snap = Backups.get_snapshots(dummy_service)[0]
    size = Backups.snapshot_restored_size(snap.id)
    assert size is not None
    assert size > 0


def test_init_tracking(backups, tmpdir):
    assert Backups.is_initted() is True
    Backups.reset()
    assert Backups.is_initted() is False
    separate_dir = tmpdir / "out_of_the_way"
    prepare_localfile_backups(separate_dir)
    Backups.init_repo()

    assert Backups.is_initted() is True


def finished_jobs():
    return [job for job in Jobs.get_jobs() if job.status is JobStatus.FINISHED]


def assert_job_finished(job_type, count):
    finished_types = [job.type_id for job in finished_jobs()]
    assert finished_types.count(job_type) == count


def assert_job_has_run(job_type):
    job = [job for job in finished_jobs() if job.type_id == job_type][0]
    assert JobStatus.RUNNING in Jobs.status_updates(job)


def job_progress_updates(job_type):
    job = [job for job in finished_jobs() if job.type_id == job_type][0]
    return Jobs.progress_updates(job)


def assert_job_had_progress(job_type):
    assert len(job_progress_updates(job_type)) > 0


def make_large_file(path: str, bytes: int):
    with open(path, "wb") as file:
        file.write(urandom(bytes))


def test_snapshots_by_id(backups, dummy_service):
    snap1 = Backups.back_up(dummy_service)
    snap2 = Backups.back_up(dummy_service)
    snap3 = Backups.back_up(dummy_service)

    assert snap2.id is not None
    assert snap2.id != ""

    assert len(Backups.get_snapshots(dummy_service)) == 3
    assert Backups.get_snapshot_by_id(snap2.id).id == snap2.id


@pytest.fixture(params=["instant_server_stop", "delayed_server_stop"])
def simulated_service_stopping_delay(request) -> float:
    if request.param == "instant_server_stop":
        return 0.0
    else:
        return 0.3


def test_backup_service_task(backups, dummy_service, simulated_service_stopping_delay):
    dummy_service.set_delay(simulated_service_stopping_delay)

    handle = start_backup(dummy_service)
    handle(blocking=True)

    snaps = Backups.get_snapshots(dummy_service)
    assert len(snaps) == 1

    id = dummy_service.get_id()
    job_type_id = f"services.{id}.backup"
    assert_job_finished(job_type_id, count=1)
    assert_job_has_run(job_type_id)
    assert_job_had_progress(job_type_id)


def test_forget_snapshot(backups, dummy_service):
    snap1 = Backups.back_up(dummy_service)
    snap2 = Backups.back_up(dummy_service)
    assert len(Backups.get_snapshots(dummy_service)) == 2

    Backups.forget_snapshot(snap2)
    assert len(Backups.get_snapshots(dummy_service)) == 1
    Backups.force_snapshot_cache_reload()
    assert len(Backups.get_snapshots(dummy_service)) == 1

    assert Backups.get_snapshots(dummy_service)[0].id == snap1.id

    Backups.forget_snapshot(snap1)
    assert len(Backups.get_snapshots(dummy_service)) == 0


def test_forget_nonexistent_snapshot(backups, dummy_service):
    bogus = Snapshot(
        id="gibberjibber",
        service_name="nohoho",
        created_at=datetime.now(timezone.utc),
        reason=BackupReason.EXPLICIT,
    )
    with pytest.raises(ValueError):
        Backups.forget_snapshot(bogus)


def test_backup_larger_file(backups, dummy_service):
    dir = path.join(dummy_service.get_folders()[0], "LARGEFILE")
    mega = 2**20
    make_large_file(dir, 100 * mega)

    handle = start_backup(dummy_service)
    handle(blocking=True)

    # results will be slightly different on different machines. if someone has troubles with it on their machine, consider dropping this test.
    id = dummy_service.get_id()
    job_type_id = f"services.{id}.backup"
    assert_job_finished(job_type_id, count=1)
    assert_job_has_run(job_type_id)
    updates = job_progress_updates(job_type_id)
    assert len(updates) > 3
    assert updates[int((len(updates) - 1) / 2.0)] > 10
    # clean up a bit
    remove(dir)


@pytest.fixture(params=["verify", "inplace"])
def restore_strategy(request) -> RestoreStrategy:
    if request.param == "verify":
        return RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE
    else:
        return RestoreStrategy.INPLACE


@pytest.fixture(params=["failed", "healthy"])
def failed(request) -> bool:
    if request.param == "failed":
        return True
    return False


def test_restore_snapshot_task(
    backups, dummy_service, restore_strategy, simulated_service_stopping_delay, failed
):
    dummy_service.set_delay(simulated_service_stopping_delay)
    if failed:
        dummy_service.set_status(ServiceStatus.FAILED)

    Backups.back_up(dummy_service)
    snaps = Backups.get_snapshots(dummy_service)
    assert len(snaps) == 1

    paths_to_nuke = service_files(dummy_service)
    contents = []

    for service_file in paths_to_nuke:
        with open(service_file, "r") as file:
            contents.append(file.read())

    for p in paths_to_nuke:
        remove(p)

    handle = restore_snapshot(snaps[0], restore_strategy)
    handle(blocking=True)

    for p, content in zip(paths_to_nuke, contents):
        assert path.exists(p)
        with open(p, "r") as file:
            assert file.read() == content

    snaps = Backups.get_snapshots(dummy_service)
    if restore_strategy == RestoreStrategy.INPLACE:
        assert len(snaps) == 2
        reasons = [snap.reason for snap in snaps]
        assert BackupReason.PRE_RESTORE in reasons
    else:
        assert len(snaps) == 1


def test_set_autobackup_period(backups):
    assert Backups.autobackup_period_minutes() is None

    Backups.set_autobackup_period_minutes(2)
    assert Backups.autobackup_period_minutes() == 2

    Backups.disable_all_autobackup()
    assert Backups.autobackup_period_minutes() is None

    Backups.set_autobackup_period_minutes(3)
    assert Backups.autobackup_period_minutes() == 3

    Backups.set_autobackup_period_minutes(0)
    assert Backups.autobackup_period_minutes() is None

    Backups.set_autobackup_period_minutes(3)
    assert Backups.autobackup_period_minutes() == 3

    Backups.set_autobackup_period_minutes(-1)
    assert Backups.autobackup_period_minutes() is None


def test_no_default_autobackup(backups, dummy_service):
    now = datetime.now(timezone.utc)
    assert not Backups.is_time_to_backup_service(dummy_service, now)
    assert not Backups.is_time_to_backup(now)


def backuppable_services() -> list[Service]:
    return [service for service in get_all_services() if service.can_be_backed_up()]


def test_services_to_back_up(backups, dummy_service):
    backup_period = 13  # minutes
    now = datetime.now(timezone.utc)

    dummy_service.set_backuppable(False)
    services = Backups.services_to_back_up(now)
    assert len(services) == 0

    dummy_service.set_backuppable(True)

    services = Backups.services_to_back_up(now)
    assert len(services) == 0

    Backups.set_autobackup_period_minutes(backup_period)

    services = Backups.services_to_back_up(now)
    assert len(services) == len(backuppable_services())
    assert dummy_service.get_id() in [
        service.get_id() for service in backuppable_services()
    ]


def test_autobackup_timer_periods(backups, dummy_service):
    now = datetime.now(timezone.utc)
    backup_period = 13  # minutes

    assert not Backups.is_time_to_backup_service(dummy_service, now)
    assert not Backups.is_time_to_backup(now)

    Backups.set_autobackup_period_minutes(backup_period)
    assert Backups.is_time_to_backup_service(dummy_service, now)
    assert Backups.is_time_to_backup(now)

    Backups.set_autobackup_period_minutes(0)
    assert not Backups.is_time_to_backup_service(dummy_service, now)
    assert not Backups.is_time_to_backup(now)


def test_autobackup_timer_enabling(backups, dummy_service):
    now = datetime.now(timezone.utc)
    backup_period = 13  # minutes
    dummy_service.set_backuppable(False)

    Backups.set_autobackup_period_minutes(backup_period)
    assert Backups.is_time_to_backup(
        now
    )  # there are other services too, not just our dummy

    # not backuppable service is not backuppable even if period is set
    assert not Backups.is_time_to_backup_service(dummy_service, now)

    dummy_service.set_backuppable(True)
    assert dummy_service.can_be_backed_up()
    assert Backups.is_time_to_backup_service(dummy_service, now)

    Backups.disable_all_autobackup()
    assert not Backups.is_time_to_backup_service(dummy_service, now)
    assert not Backups.is_time_to_backup(now)


def test_autobackup_timing(backups, dummy_service):
    backup_period = 13  # minutes
    now = datetime.now(timezone.utc)

    Backups.set_autobackup_period_minutes(backup_period)
    assert Backups.is_time_to_backup_service(dummy_service, now)
    assert Backups.is_time_to_backup(now)

    Backups.back_up(dummy_service)

    now = datetime.now(timezone.utc)
    assert not Backups.is_time_to_backup_service(dummy_service, now)

    past = datetime.now(timezone.utc) - timedelta(minutes=1)
    assert not Backups.is_time_to_backup_service(dummy_service, past)

    future = datetime.now(timezone.utc) + timedelta(minutes=backup_period + 2)
    assert Backups.is_time_to_backup_service(dummy_service, future)


def test_backup_unbackuppable(backups, dummy_service):
    dummy_service.set_backuppable(False)
    assert dummy_service.can_be_backed_up() is False
    with pytest.raises(ValueError):
        Backups.back_up(dummy_service)


def test_failed_autoback_prevents_more_autobackup(backups, dummy_service):
    backup_period = 13  # minutes
    now = datetime.now(timezone.utc)

    Backups.set_autobackup_period_minutes(backup_period)
    assert Backups.is_time_to_backup_service(dummy_service, now)

    # artificially making an errored out backup job
    dummy_service.set_backuppable(False)
    with pytest.raises(ValueError):
        Backups.back_up(dummy_service)
    dummy_service.set_backuppable(True)

    assert Backups.get_last_backed_up(dummy_service) is None
    assert Backups.get_last_backup_error_time(dummy_service) is not None

    assert Backups.is_time_to_backup_service(dummy_service, now) is False


# Storage
def test_snapshots_caching(backups, dummy_service):
    Backups.back_up(dummy_service)

    # we test indirectly that we do redis calls instead of shell calls
    start = datetime.now()
    for i in range(10):
        snapshots = Backups.get_snapshots(dummy_service)
        assert len(snapshots) == 1
    assert datetime.now() - start < timedelta(seconds=0.5)

    cached_snapshots = Storage.get_cached_snapshots()
    assert len(cached_snapshots) == 1

    Storage.delete_cached_snapshot(cached_snapshots[0])
    cached_snapshots = Storage.get_cached_snapshots()
    assert len(cached_snapshots) == 0

    snapshots = Backups.get_snapshots(dummy_service)
    assert len(snapshots) == 1
    cached_snapshots = Storage.get_cached_snapshots()
    assert len(cached_snapshots) == 1


def lowlevel_forget(snapshot_id):
    Backups.provider().backupper.forget_snapshot(snapshot_id)


# Storage
def test_snapshots_cache_invalidation(backups, dummy_service):
    Backups.back_up(dummy_service)
    cached_snapshots = Storage.get_cached_snapshots()
    assert len(cached_snapshots) == 1

    Storage.invalidate_snapshot_storage()
    cached_snapshots = Storage.get_cached_snapshots()
    assert len(cached_snapshots) == 0

    Backups.force_snapshot_cache_reload()
    cached_snapshots = Storage.get_cached_snapshots()
    assert len(cached_snapshots) == 1
    snap = cached_snapshots[0]

    lowlevel_forget(snap.id)
    cached_snapshots = Storage.get_cached_snapshots()
    assert len(cached_snapshots) == 1

    Backups.force_snapshot_cache_reload()
    cached_snapshots = Storage.get_cached_snapshots()
    assert len(cached_snapshots) == 0


# Storage
def test_init_tracking_caching(backups, raw_dummy_service):
    assert Storage.has_init_mark() is True
    Backups.reset()
    assert Storage.has_init_mark() is False

    Storage.mark_as_init()

    assert Storage.has_init_mark() is True
    assert Backups.is_initted() is True


# Storage
def test_init_tracking_caching2(backups, tmpdir):
    assert Storage.has_init_mark() is True
    Backups.reset()
    assert Storage.has_init_mark() is False
    separate_dir = tmpdir / "out_of_the_way"
    prepare_localfile_backups(separate_dir)
    assert Storage.has_init_mark() is False

    Backups.init_repo()

    assert Storage.has_init_mark() is True


# Storage
def test_provider_storage(backups_backblaze):
    provider = Backups.provider()

    assert provider is not None

    assert isinstance(provider, Backblaze)
    assert provider.login == "ID"
    assert provider.key == "KEY"

    Storage.store_provider(provider)
    restored_provider = Backups._load_provider_redis()
    assert isinstance(restored_provider, Backblaze)
    assert restored_provider.login == "ID"
    assert restored_provider.key == "KEY"


def test_sync(dummy_service):
    src = dummy_service.get_folders()[0]
    dst = dummy_service.get_folders()[1]
    old_files_src = set(listdir(src))
    old_files_dst = set(listdir(dst))
    assert old_files_src != old_files_dst

    sync(src, dst)
    new_files_src = set(listdir(src))
    new_files_dst = set(listdir(dst))
    assert new_files_src == old_files_src
    assert new_files_dst == new_files_src


def test_sync_nonexistent_src(dummy_service):
    src = "/var/lib/nonexistentFluffyBunniesOfUnix"
    dst = dummy_service.get_folders()[1]

    with pytest.raises(ValueError):
        sync(src, dst)


def test_move_blocks_backups(backups, dummy_service, restore_strategy):
    snap = Backups.back_up(dummy_service)
    job = Jobs.add(
        type_id=f"services.{dummy_service.get_id()}.move",
        name="Move Dummy",
        description=f"Moving Dummy data to the Rainbow Land",
        status=JobStatus.RUNNING,
    )

    with pytest.raises(ValueError):
        Backups.back_up(dummy_service)

    with pytest.raises(ValueError):
        Backups.restore_snapshot(snap, restore_strategy)


def test_double_lock_unlock(backups, dummy_service):
    # notice that introducing stale locks is only safe for other tests if we erase repo in between
    # which we do at the time of writing this test

    Backups.provider().backupper.lock()
    with pytest.raises(ValueError):
        Backups.provider().backupper.lock()

    Backups.provider().backupper.unlock()
    Backups.provider().backupper.lock()

    Backups.provider().backupper.unlock()
    Backups.provider().backupper.unlock()


def test_operations_while_locked(backups, dummy_service):
    # Stale lock prevention test

    # consider making it fully at the level of backupper?
    # because this is where prevention lives?
    # Backups singleton is here only so that we can run this against B2, S3 and whatever
    # But maybe it is not necessary (if restic treats them uniformly enough)

    Backups.provider().backupper.lock()
    snap = Backups.back_up(dummy_service)
    assert snap is not None

    Backups.provider().backupper.lock()
    # using lowlevel to make sure no caching interferes
    assert Backups.provider().backupper.is_initted() is True

    Backups.provider().backupper.lock()
    assert Backups.snapshot_restored_size(snap.id) > 0

    Backups.provider().backupper.lock()
    Backups.restore_snapshot(snap)

    Backups.provider().backupper.lock()
    Backups.forget_snapshot(snap)

    Backups.provider().backupper.lock()
    assert Backups.provider().backupper.get_snapshots() == []

    # check that no locks were left
    Backups.provider().backupper.lock()
    Backups.provider().backupper.unlock()


# a paranoid check to weed out problems with tempdirs that are not dependent on us
def test_tempfile():
    with tempfile.TemporaryDirectory() as temp:
        assert path.exists(temp)
    assert not path.exists(temp)


# Storage
def test_cache_invalidaton_task(backups, dummy_service):
    Backups.back_up(dummy_service)
    assert len(Storage.get_cached_snapshots()) == 1

    # Does not trigger resync
    Storage.invalidate_snapshot_storage()
    assert Storage.get_cached_snapshots() == []

    reload_snapshot_cache()
    assert len(Storage.get_cached_snapshots()) == 1