diff --git a/src/routes/answers.py b/src/routes/answers.py index 2a54079..b1f8af7 100644 --- a/src/routes/answers.py +++ b/src/routes/answers.py @@ -1,7 +1,7 @@ # routes/answers.py from flask import Blueprint, request, jsonify, current_app from utils.errors import InvalidJsonFormatError -from utils.process_data import evaluate_comments +from utils.process_data import evaluate_comments, evaluate_refinement import json router = Blueprint('answers', __name__, url_prefix='/answers') @@ -84,4 +84,8 @@ def submit_refinement(): socketio.emit('successful-upload', room=sid) socketio.emit('started-processing', room=sid) - return jsonify({'message': 'Answer submitted successfully'}) + results = evaluate_refinement( + validated, lambda p: socketio.emit('progress', {'percent': p}, room=sid) + ) + + return jsonify(results) diff --git a/src/utils/process_data.py b/src/utils/process_data.py index 78c0f7d..4726d37 100644 --- a/src/utils/process_data.py +++ b/src/utils/process_data.py @@ -1,5 +1,4 @@ -import sys - +import sys, docker from utils.handlers import get_build_handler from .paths import get_project_path from sacrebleu import sentence_bleu as bleu @@ -11,6 +10,8 @@ REFERENCE_MAP = Dataset.from_json( ARCHIVES_ROOT = str(get_project_path('../data/archives')) +DOCKER_CLIENT = docker.from_env() + def evaluate_comments(answers: dict[str, str], percent_cb): total = len(answers) @@ -37,10 +38,51 @@ def evaluate_comments(answers: dict[str, str], percent_cb): return results -def evaluate_refinement(answers: dict[str, str], percent_cb): +def evaluate_refinement(answers: dict[str, dict[str, str]], percent_cb): + print("Processing refinement...") total = len(answers) - for i, (key, value) in enumerate(answers.items(), 1): - print(f"Processing {key}: {value}...") - # time.sleep(1) + results = {} + for i, (id, value) in enumerate(answers.items(), 1): + print(f"Processing {id}...") + if id not in REFERENCE_MAP: + print(f"[WARNING] skipping {id} since it is not present in dataset", file=sys.stderr) + continue + entry = REFERENCE_MAP[id] + try: + build_handler = get_build_handler( + ARCHIVES_ROOT, entry.metadata.archive_name(ArchiveState.MERGED) + ) + build_handler.set_client(DOCKER_CLIENT) + except Exception as e: + print( + f"[ERROR] {id} ({entry.metadata.repo} #PR {entry.metadata.pr_number}) {type(e)}: {e}", + file=sys.stderr, + ) + continue + + results[id] = {} + with build_handler: + steps = [ + ("compilation", build_handler.compile_repo), + ("test", build_handler.test_repo), + ] + for task, action in steps: + try: + print(f"Executing {task}...") + action() + print( + f"{task} executed successfully on {id} ({entry.metadata.repo} #PR {entry.metadata.pr_number})" + ) + results[id][task] = True + except Exception as e: + results[id][task] = False + print( + f"[ERROR] {id} ({entry.metadata.repo} #PR {entry.metadata.pr_number}) {type(e)}: {e}", + file=sys.stderr, + ) + break + print("Done") percent_cb(int(i / total * 100)) + + return results