From 9655fd7a6e42f259646450868f66e403f4e31f00 Mon Sep 17 00:00:00 2001
From: Karma Riuk <riukkarma@gmail.com>
Date: Wed, 7 May 2025 23:16:03 +0200
Subject: [PATCH] added bleu score evaluation for the comment generation

---
 package.json              |  1 +
 src/utils/process_data.js | 41 +++++++++++++++++++++++++++++++++++----
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/package.json b/package.json
index 93f6a76..9ffbfd1 100644
--- a/package.json
+++ b/package.json
@@ -9,6 +9,7 @@
         "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js"
     },
     "dependencies": {
+        "bleu-score": "^1.0.4",
         "cors": "^2.8.5",
         "dotenv": "^16.3.1",
         "express": "^4.18.2",
diff --git a/src/utils/process_data.js b/src/utils/process_data.js
index 9b5fef8..d3d67fa 100644
--- a/src/utils/process_data.js
+++ b/src/utils/process_data.js
@@ -1,12 +1,45 @@
+import fs from "fs";
+import { getProjectPath } from "../utils/paths.js";
+import { bleu } from "bleu-score";
+
+const DATASET_PATH = getProjectPath("data/dataset.json");
+
 export const evaluate_comments = async (answers, percent_cb) => {
+    console.log(`Reading dataset...`);
+    const raw = fs.readFileSync(DATASET_PATH);
+    const dataset = JSON.parse(raw);
+
+    console.log(`Building reference map...`);
+    const referenceMap = {};
+    for (const entry of dataset.entries) {
+        const id = entry.metadata.id;
+        const comments = entry.comments;
+        referenceMap[id] = comments.map((c) => c.body);
+    }
+
     const total = Object.keys(answers).length;
     let i = 0;
-    for (const [key, value] of Object.entries(answers)) {
-        console.log(`Processing ${key}: ${value}...`);
-        await new Promise((res) => setTimeout(res, 1000));
-        console.log("Done");
+    for (const [id, generated_comment] of Object.entries(answers)) {
+        console.log(`Processing ${id}...`);
+        if (!(id in referenceMap)) {
+            // throw new Error(`id: "${id}" is not present in the dataset`);
+            console.error(`id: "${id}" is not present in the dataset`);
+            continue;
+        }
+        const references = referenceMap[id];
+
+        let maxScore = 0;
+        for (let reference of references) {
+            const score = bleu(reference, generated_comment, 4); // TODO: ask prof what number show be here
+            maxScore = Math.max(score, maxScore);
+            console.log(`bleu score: ${score}`);
+        }
+        console.log(`Max bleu score: ${maxScore}`);
+
+        console.log(`Done with ${id}`);
         percent_cb(Math.floor((++i / total) * 100));
     }
+    console.log("Done processing comments!");
 };
 
 export const evaluate_refinement = async (answers, percent_cb) => {