diff --git a/errors.py b/errors.py
index 37da78c..dd30cb8 100644
--- a/errors.py
+++ b/errors.py
@@ -17,6 +17,12 @@ class NoLinesForCommentError(SetupException):
     reason_for_failure = "There are no line reference for the comment"
 
 
+class CommentedFileNotInOriginalChanges(SetupException):
+    reason_for_failure = (
+        "Commented file is not part of the original PR (most like due to a merge of another branch)"
+    )
+
+
 class CantCloneRepoError(SetupException):
     reason_for_failure = "Couldn't clone the repository"
 
diff --git a/pull_requests.py b/pull_requests.py
index 2fbeaab..e03ef69 100644
--- a/pull_requests.py
+++ b/pull_requests.py
@@ -23,6 +23,7 @@ from errors import (
     CantCheckoutCommitError,
     CantEnsureFullHistoryError,
     CantFetchPRError,
+    CommentedFileNotInOriginalChanges,
     MultipleFilesError,
     NoDiffsAfterError,
     NoDiffsBeforeError,
@@ -185,7 +186,10 @@ def get_files(pr: PullRequest, repo: Repository, repo_path: str) -> dict[str, Fi
 
 def get_comments(pr: PullRequest) -> list[Comment]:
     ret = []
+    filenames = {file.filename for file in pr.get_files()}
     for comment in pr.get_review_comments():
+        if comment.path not in filenames:
+            raise CommentedFileNotInOriginalChanges(f"File {comment.path} not in {filenames}")
         if (
             comment.start_line is None
             and comment.original_start_line is None