now saving the results from disk and retrieving

the at server startup
This commit is contained in:
Karma Riuk
2025-05-20 12:18:42 +02:00
parent 346b3fc40a
commit 1cd2fb0752
3 changed files with 54 additions and 14 deletions

View File

@ -68,10 +68,8 @@
<button id="info-results-btn" class='info-btn'><i class="fa fa-info"></i></button> <button id="info-results-btn" class='info-btn'><i class="fa fa-info"></i></button>
</legend> </legend>
<div style="display: flex; align-items: center; gap: 0.5em"> <div style="display: flex; align-items: center; gap: 0.5em">
<label for="uuid">UUID:</label> <label for="uuid">Process id:</label>
<input type="text" id="uuid" <input type="text" id="uuid" required />
pattern="[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89ABab][0-9a-fA-F]{3}-[0-9a-fA-F]{12}"
title="Must be a valid UUID, e.g. 123e4567-e89b-12d3-a456-426614174000" required />
</div> </div>
<br /><br /> <br /><br />
<button id="request-status">Request status</button> <button id="request-status">Request status</button>

View File

@ -5,7 +5,7 @@ from utils.errors import InvalidJsonFormatError
from utils.process_data import evaluate_comments, evaluate_refinement from utils.process_data import evaluate_comments, evaluate_refinement
from utils.observer import SocketObserver, Status, Subject from utils.observer import SocketObserver, Status, Subject
import functools import functools
import json, uuid import json
from utils.queue_manager import QueueManager from utils.queue_manager import QueueManager
@ -62,8 +62,8 @@ def handler(type_: str, validate_json: Callable, evaluate_submission: Callable):
except InvalidJsonFormatError as e: except InvalidJsonFormatError as e:
return jsonify({'error': 'Invalid JSON format', 'message': str(e)}), 400 return jsonify({'error': 'Invalid JSON format', 'message': str(e)}), 400
process_id = str(uuid.uuid4()) subject = Subject(type_, evaluate_submission)
subject = Subject(process_id, type_, evaluate_submission) process_id = subject.id
Subject.uuid2subject[process_id] = subject Subject.uuid2subject[process_id] = subject
QUEUE_MANAGER.submit(subject, validated) QUEUE_MANAGER.submit(subject, validated)

View File

@ -1,7 +1,12 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from enum import Enum from enum import Enum
from threading import Thread import os
from typing import Callable, Optional, Set, Any import tempfile
from typing import Callable, Optional, Set
from flask import json
RESULTS_DIR = os.getenv("RESULTS_DIR", "submission_results")
class Status(Enum): class Status(Enum):
@ -49,14 +54,47 @@ class Subject:
obs2subject: dict[Observer, "Subject"] = {} obs2subject: dict[Observer, "Subject"] = {}
uuid2subject: dict[str, "Subject"] = {} uuid2subject: dict[str, "Subject"] = {}
def __init__(self, id: str, type_: str, task: Callable) -> None: @classmethod
self.id = id def setup(cls):
if not os.path.exists(RESULTS_DIR):
os.mkdir(RESULTS_DIR)
for file in os.listdir(RESULTS_DIR):
file_path = os.path.join(RESULTS_DIR, file)
if os.path.getsize(file_path) == 0:
# submission was still being processed before the server stopped
# the file was created but never written to, therefore must be removed
os.remove(file_path)
continue
_, type_, _ = file.split("_")
with open(file_path, "r") as f:
cls.uuid2subject[file] = Subject(
type_, lambda: None, id=file, status=Status.COMPLETE, results=json.load(f)
)
def __init__(
self,
type_: str,
task: Callable,
id: Optional[str] = None,
status: Status = Status.CREATED,
results: Optional[dict] = None,
) -> None:
self.type = type_ self.type = type_
self.observers: Set[Observer] = set() self.observers: Set[Observer] = set()
self.status: Status = Status.CREATED self.status: Status = status
self.results: Optional[dict] = None self.results: Optional[dict] = results
self.task = task self.task = task
self.percent: float = -1 self.percent: float = -1
if id is None:
_, self.full_path = tempfile.mkstemp(
prefix=f"crab_{type_}_", dir=RESULTS_DIR, text=True
)
self.id = os.path.basename(self.full_path)
else:
self.full_path = os.path.abspath(os.path.join(RESULTS_DIR, id))
self.id = id
def registerObserver(self, observer: Observer) -> None: def registerObserver(self, observer: Observer) -> None:
self.observers.add(observer) self.observers.add(observer)
@ -83,4 +121,8 @@ class Subject:
Subject.obs2subject.pop(observer) Subject.obs2subject.pop(observer)
self.observers.clear() self.observers.clear()
self.results = results self.results = results
# TODO: maybe save results to disk here? with open(self.full_path, "w") as f:
json.dump(results, f)
Subject.setup()