selfprivacy-rest-api/selfprivacy_api/jobs/__init__.py

299 lines
8.3 KiB
Python
Raw Normal View History

2022-07-30 15:24:21 +00:00
"""
Jobs controller. It handles the jobs that are created by the user.
This is a singleton class holding the jobs list.
Jobs can be added and removed.
A single job can be updated.
A job is a dictionary with the following keys:
- id: unique identifier of the job
- name: name of the job
- description: description of the job
- status: status of the job
- created_at: date of creation of the job
- updated_at: date of last update of the job
- finished_at: date of finish of the job
- error: error message if the job failed
- result: result of the job
"""
import typing
import datetime
from uuid import UUID
2022-07-30 15:24:21 +00:00
import uuid
from enum import Enum
from pydantic import BaseModel
2022-11-16 13:54:54 +00:00
from selfprivacy_api.utils.redis_pool import RedisPool
JOB_EXPIRATION_SECONDS = 10 * 24 * 60 * 60 # ten days
STATUS_LOGS_PREFIX = "jobs_logs:status:"
PROGRESS_LOGS_PREFIX = "jobs_logs:progress:"
2022-07-30 15:24:21 +00:00
class JobStatus(str, Enum):
2022-07-30 15:24:21 +00:00
"""
Status of a job.
"""
CREATED = "CREATED"
RUNNING = "RUNNING"
FINISHED = "FINISHED"
ERROR = "ERROR"
class Job(BaseModel):
2022-07-30 15:24:21 +00:00
"""
Job class.
"""
2022-09-09 14:42:40 +00:00
uid: UUID
type_id: str
name: str
description: str
status: JobStatus
status_text: typing.Optional[str]
progress: typing.Optional[int]
created_at: datetime.datetime
updated_at: datetime.datetime
finished_at: typing.Optional[datetime.datetime]
error: typing.Optional[str]
result: typing.Optional[str]
2022-07-30 15:24:21 +00:00
class Jobs:
"""
Jobs class.
"""
@staticmethod
def reset() -> None:
"""
Reset the jobs list.
"""
2022-11-16 13:54:54 +00:00
jobs = Jobs.get_jobs()
for job in jobs:
Jobs.remove(job)
Jobs.reset_logs()
@staticmethod
2022-07-30 15:24:21 +00:00
def add(
name: str,
type_id: str,
description: str,
status: JobStatus = JobStatus.CREATED,
status_text: str = "",
progress: int = 0,
2022-07-30 15:24:21 +00:00
) -> Job:
"""
Add a job to the jobs list.
"""
job = Job(
2022-09-09 14:42:40 +00:00
uid=uuid.uuid4(),
2022-07-30 15:24:21 +00:00
name=name,
type_id=type_id,
2022-07-30 15:24:21 +00:00
description=description,
status=status,
status_text=status_text,
progress=progress,
2022-07-30 15:24:21 +00:00
created_at=datetime.datetime.now(),
updated_at=datetime.datetime.now(),
finished_at=None,
error=None,
result=None,
)
2022-12-30 18:06:16 +00:00
redis = RedisPool().get_connection()
_store_job_as_hash(redis, _redis_key_from_uuid(job.uid), job)
2022-07-30 15:24:21 +00:00
return job
@staticmethod
def remove(job: Job) -> None:
2022-07-30 15:24:21 +00:00
"""
Remove a job from the jobs list.
"""
Jobs.remove_by_uid(str(job.uid))
@staticmethod
def remove_by_uid(job_uuid: str) -> bool:
"""
Remove a job from the jobs list.
"""
2022-12-30 18:06:16 +00:00
redis = RedisPool().get_connection()
key = _redis_key_from_uuid(job_uuid)
2022-12-30 18:06:16 +00:00
if redis.exists(key):
redis.delete(key)
return True
return False
2022-07-30 15:24:21 +00:00
@staticmethod
2023-07-20 15:24:26 +00:00
def reset_logs() -> None:
redis = RedisPool().get_connection()
for key in redis.keys(STATUS_LOGS_PREFIX + "*"):
redis.delete(key)
@staticmethod
2023-07-20 15:24:26 +00:00
def log_status_update(job: Job, status: JobStatus) -> None:
redis = RedisPool().get_connection()
2023-05-17 18:36:39 +00:00
key = _status_log_key_from_uuid(job.uid)
redis.lpush(key, status.value)
redis.expire(key, 10)
2023-05-17 18:36:39 +00:00
@staticmethod
2023-07-20 15:24:26 +00:00
def log_progress_update(job: Job, progress: int) -> None:
2023-05-17 18:36:39 +00:00
redis = RedisPool().get_connection()
key = _progress_log_key_from_uuid(job.uid)
redis.lpush(key, progress)
redis.expire(key, 10)
@staticmethod
2023-07-20 15:24:26 +00:00
def status_updates(job: Job) -> list[JobStatus]:
result: list[JobStatus] = []
redis = RedisPool().get_connection()
2023-05-17 18:36:39 +00:00
key = _status_log_key_from_uuid(job.uid)
if not redis.exists(key):
return []
2023-07-20 15:24:26 +00:00
status_strings: list[str] = redis.lrange(key, 0, -1) # type: ignore
for status in status_strings:
try:
result.append(JobStatus[status])
2023-07-20 15:24:26 +00:00
except KeyError as error:
raise ValueError("impossible job status: " + status) from error
return result
2023-05-17 18:36:39 +00:00
@staticmethod
2023-07-20 15:24:26 +00:00
def progress_updates(job: Job) -> list[int]:
result: list[int] = []
2023-05-17 18:36:39 +00:00
redis = RedisPool().get_connection()
key = _progress_log_key_from_uuid(job.uid)
if not redis.exists(key):
return []
2023-07-20 15:24:26 +00:00
progress_strings: list[str] = redis.lrange(key, 0, -1) # type: ignore
2023-05-17 18:36:39 +00:00
for progress in progress_strings:
try:
result.append(int(progress))
2023-07-20 15:24:26 +00:00
except KeyError as error:
raise ValueError("impossible job progress: " + progress) from error
2023-05-17 18:36:39 +00:00
return result
@staticmethod
2022-07-30 15:24:21 +00:00
def update(
job: Job,
status: JobStatus,
status_text: typing.Optional[str] = None,
progress: typing.Optional[int] = None,
name: typing.Optional[str] = None,
description: typing.Optional[str] = None,
error: typing.Optional[str] = None,
result: typing.Optional[str] = None,
2022-07-30 15:24:21 +00:00
) -> Job:
"""
Update a job in the jobs list.
"""
if name is not None:
job.name = name
if description is not None:
job.description = description
if status_text is not None:
job.status_text = status_text
if status == JobStatus.FINISHED:
job.progress = 100
if progress is not None:
# explicitly provided progress has priority
job.progress = progress
2023-05-17 18:36:39 +00:00
Jobs.log_progress_update(job, progress)
2022-07-30 15:24:21 +00:00
job.status = status
Jobs.log_status_update(job, status)
2022-07-30 15:24:21 +00:00
job.updated_at = datetime.datetime.now()
job.error = error
job.result = result
if status in (JobStatus.FINISHED, JobStatus.ERROR):
job.finished_at = datetime.datetime.now()
2022-12-30 18:06:16 +00:00
redis = RedisPool().get_connection()
key = _redis_key_from_uuid(job.uid)
2022-12-30 18:06:16 +00:00
if redis.exists(key):
_store_job_as_hash(redis, key, job)
if status in (JobStatus.FINISHED, JobStatus.ERROR):
2022-12-30 18:06:16 +00:00
redis.expire(key, JOB_EXPIRATION_SECONDS)
2022-07-30 15:24:21 +00:00
return job
@staticmethod
def get_job(uid: str) -> typing.Optional[Job]:
2022-07-30 15:24:21 +00:00
"""
Get a job from the jobs list.
"""
2022-12-30 18:06:16 +00:00
redis = RedisPool().get_connection()
key = _redis_key_from_uuid(uid)
2022-12-30 18:06:16 +00:00
if redis.exists(key):
return _job_from_hash(redis, key)
2022-07-30 15:24:21 +00:00
return None
@staticmethod
def get_jobs() -> typing.List[Job]:
2022-07-30 15:24:21 +00:00
"""
Get the jobs list.
"""
2022-12-30 18:06:16 +00:00
redis = RedisPool().get_connection()
job_keys = redis.keys("jobs:*")
jobs = []
for job_key in job_keys:
job = _job_from_hash(redis, job_key)
if job is not None:
jobs.append(job)
return jobs
@staticmethod
def is_busy() -> bool:
"""
Check if there is a job running.
"""
2022-11-16 13:54:54 +00:00
for job in Jobs.get_jobs():
if job.status == JobStatus.RUNNING:
2022-11-16 13:54:54 +00:00
return True
return False
2022-11-16 13:54:54 +00:00
2023-07-20 15:24:26 +00:00
def _redis_key_from_uuid(uuid_string) -> str:
2022-12-30 18:06:16 +00:00
return "jobs:" + str(uuid_string)
2022-11-16 13:54:54 +00:00
2023-07-20 15:24:26 +00:00
def _status_log_key_from_uuid(uuid_string) -> str:
return STATUS_LOGS_PREFIX + str(uuid_string)
2023-07-20 15:24:26 +00:00
def _progress_log_key_from_uuid(uuid_string) -> str:
2023-05-17 18:36:39 +00:00
return PROGRESS_LOGS_PREFIX + str(uuid_string)
2023-07-20 15:24:26 +00:00
def _store_job_as_hash(redis, redis_key, model) -> None:
2022-11-16 13:54:54 +00:00
for key, value in model.dict().items():
if isinstance(value, uuid.UUID):
value = str(value)
if isinstance(value, datetime.datetime):
value = value.isoformat()
if isinstance(value, JobStatus):
value = value.value
2022-12-30 18:06:16 +00:00
redis.hset(redis_key, key, str(value))
2022-11-16 13:54:54 +00:00
2023-07-20 15:24:26 +00:00
def _job_from_hash(redis, redis_key) -> typing.Optional[Job]:
2022-12-30 18:06:16 +00:00
if redis.exists(redis_key):
job_dict = redis.hgetall(redis_key)
2022-11-16 13:54:54 +00:00
for date in [
"created_at",
"updated_at",
"finished_at",
]:
if job_dict[date] != "None":
job_dict[date] = datetime.datetime.fromisoformat(job_dict[date])
for key in job_dict.keys():
if job_dict[key] == "None":
job_dict[key] = None
return Job(**job_dict)
return None