From 0d8b81054d232b75570c032f7ae47ef79da48b22 Mon Sep 17 00:00:00 2001 From: Karma Riuk Date: Wed, 26 Mar 2025 13:05:05 +0100 Subject: [PATCH] formatted code --- dataset.py | 33 ++++++----- pull_requests.py | 143 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 135 insertions(+), 41 deletions(-) diff --git a/dataset.py b/dataset.py index 3c52a6f..cd4536a 100644 --- a/dataset.py +++ b/dataset.py @@ -2,31 +2,37 @@ from dataclasses import dataclass, field from typing import Dict, List import json + +# fmt: off @dataclass class FileData: path: str - content: str = "" # Not sure about this, maybe we should just keep the path and extract the contents dynamically (boh) + content: str = "" # Not sure about this, maybe we should just keep the path and extract the contents dynamically (boh) + @dataclass class Metadata: - repo: str # the name of the repo, with style XXX/YYY + repo: str # the name of the repo, with style XXX/YYY pr_number: int - merge_commit_sha: str # to checkout for the tests - commented_files : Dict[str, str] # comment -> filename - commented_files_coverages: Dict[str, Dict[str, float]] = field(default_factory=dict) # filename -> jacoco-report -> coverage + merge_commit_sha: str # to checkout for the tests + commented_files: Dict[str, str] # comment -> filename + commented_files_coverages: Dict[str, Dict[str, float]] = field(default_factory=dict) # filename -> jacoco-report -> coverage successful: bool = True build_system: str = "" reason_for_failure: str = "" last_cmd_error_msg: str = "" + @dataclass class DatasetEntry: metadata: Metadata - files: Dict[str, FileData] # filename -> file data, files before the PR (before the first PR commits) - diffs_before: Dict[str, str] # filename -> diff, diffs between the opening of the PR and the comment + files: Dict[str, FileData] # filename -> file data, files before the PR (before the first PR commits) + diffs_before: Dict[str, str] # filename -> diff, diffs between the opening of the PR and the comment comments: List[str] - diffs_after: Dict[str, str] # filename -> diff, changes after the comment + diffs_after: Dict[str, str] # filename -> diff, changes after the comment + +# fmt: on @dataclass class Dataset: entries: List[DatasetEntry] = field(default_factory=list) @@ -48,19 +54,20 @@ class Dataset: for entry_data in data["entries"]: metadata_data = entry_data["metadata"] metadata = Metadata(**metadata_data) - if not keep_still_in_progress and metadata.reason_for_failure == "Was still being processed": + if ( + not keep_still_in_progress + and metadata.reason_for_failure == "Was still being processed" + ): continue - files = { - fname: FileData(**fdata) for fname, fdata in entry_data["files"].items() - } + files = {fname: FileData(**fdata) for fname, fdata in entry_data["files"].items()} entry = DatasetEntry( metadata=metadata, files=files, diffs_before=entry_data["diffs_before"], comments=entry_data["comments"], - diffs_after=entry_data["diffs_after"] + diffs_after=entry_data["diffs_after"], ) entries.append(entry) diff --git a/pull_requests.py b/pull_requests.py index d9e66e1..86118b4 100644 --- a/pull_requests.py +++ b/pull_requests.py @@ -17,7 +17,7 @@ def get_good_projects(csv_file: str) -> pd.DataFrame: """ Extracts the good (the ones that compile and test successfully, and that have at least one test) from the given file. - + Parameters: csv_file (str): The csv file containing the projects. @@ -27,13 +27,17 @@ def get_good_projects(csv_file: str) -> pd.DataFrame: df = pd.read_csv(csv_file) return df.loc[(df['good_repo_for_crab'] == True) & (df['n_tests'] > 0)] + def is_pull_good(pull: PullRequest, verbose: bool = False): return ( has_only_1_comment(pull.get_commits(), pull.get_review_comments(), verbose=verbose) and pull.user.type != "Bot" ) -def get_good_prs(repo: Repository, cache: dict[str, dict[int, DatasetEntry]] = {}) -> list[PullRequest]: + +def get_good_prs( + repo: Repository, cache: dict[str, dict[int, DatasetEntry]] = {} +) -> list[PullRequest]: good_prs = [] potenially_good_prs = repo.get_pulls(state="closed") @@ -42,7 +46,11 @@ def get_good_prs(repo: Repository, cache: dict[str, dict[int, DatasetEntry]] = { if number_of_prs == 0: return [] - with tqdm(total=number_of_prs, desc=f"Extracting good PRs from {repo.full_name}", leave=False) as pbar: + with tqdm( + total=number_of_prs, + desc=f"Extracting good PRs from {repo.full_name}", + leave=False, + ) as pbar: for pr in potenially_good_prs: pbar.set_postfix({"new good found": len(good_prs), "pr_number": pr.number}) if pr.merged_at is None or pr.number in cache.get(repo.full_name, set()): @@ -54,8 +62,15 @@ def get_good_prs(repo: Repository, cache: dict[str, dict[int, DatasetEntry]] = { return good_prs + def run_git_cmd(cmd: list[str], repo_path: str) -> subprocess.CompletedProcess: - return subprocess.run(["git", "-C", repo_path] + cmd, check=True, capture_output=True, text=True) + return subprocess.run( + ["git", "-C", repo_path] + cmd, + check=True, + capture_output=True, + text=True, + ) + def ensure_full_history(repo_path: str) -> None: result = run_git_cmd(["rev-parse", "--is-shallow-repository"], repo_path) @@ -63,11 +78,19 @@ def ensure_full_history(repo_path: str) -> None: if result.stdout.strip() == "true": run_git_cmd(["fetch", "--unshallow"], repo_path) + def reset_repo_to_latest_commit(repo_path: str) -> None: current_branch = run_git_cmd(["rev-parse", "--abbrev-ref", "HEAD"], repo_path).stdout.strip() run_git_cmd(["reset", "--hard", current_branch], repo_path) -def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir: str, cache: dict[str, dict[int, DatasetEntry]] = {}): + +def process_pull( + repo: Repository, + pr: PullRequest, + dataset: Dataset, + repos_dir: str, + cache: dict[str, dict[int, DatasetEntry]] = {}, +): if pr.number in cache.get(repo.full_name, set()): dataset.entries.append(cache[repo.full_name][pr.number]) return @@ -80,7 +103,9 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir: last_commit = commits[-1] try: - diffs_before = {file.filename: file.patch for file in repo.compare(pr.base.sha, first_commit.sha).files} + diffs_before = { + file.filename: file.patch for file in repo.compare(pr.base.sha, first_commit.sha).files + } except GithubException as e: return @@ -91,12 +116,21 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir: commented_file_path = comment.path try: - diffs_after = {file.filename: file.patch for file in repo.compare(first_commit.sha, last_commit.sha).files} + diffs_after = { + file.filename: file.patch + for file in repo.compare(first_commit.sha, last_commit.sha).files + } except GithubException as e: return entry = DatasetEntry( - metadata=Metadata(repo.full_name, pr.number, pr.merge_commit_sha, {comment_text: commented_file_path}, reason_for_failure="Was still being processed"), + metadata=Metadata( + repo.full_name, + pr.number, + pr.merge_commit_sha, + {comment_text: commented_file_path}, + reason_for_failure="Was still being processed", + ), files={file.filename: FileData(file.filename) for file in pr.get_files()}, diffs_before=diffs_before, comments=[comment_text], @@ -126,16 +160,25 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir: # raise e return entry.metadata.successful - if not _try_cmd(lambda: ensure_full_history(repo_path), "Couldn't ensure the full history of the repo (fetch --unshallow)"): + if not _try_cmd( + lambda: ensure_full_history(repo_path), + "Couldn't ensure the full history of the repo (fetch --unshallow)", + ): return try: run_git_cmd(["checkout", pr.merge_commit_sha], repo_path) except subprocess.CalledProcessError: - if not _try_cmd(lambda: run_git_cmd(["fetch", "origin", f"pull/{pr.number}/merge"], repo_path), "Couldn't fetch the PR's merge commit"): + if not _try_cmd( + lambda: run_git_cmd(["fetch", "origin", f"pull/{pr.number}/merge"], repo_path), + "Couldn't fetch the PR's merge commit", + ): return - - if not _try_cmd(lambda: run_git_cmd(["checkout", pr.merge_commit_sha], repo_path), "Coudln't checkout the PR's merge commit (even after fetching the pull//merge)"): + + if not _try_cmd( + lambda: run_git_cmd(["checkout", pr.merge_commit_sha], repo_path), + "Coudln't checkout the PR's merge commit (even after fetching the pull//merge)", + ): return build_handler = get_build_handler(repos_dir, repo.full_name, updates) @@ -146,7 +189,7 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir: return entry.metadata.build_system = build_handler.get_type() build_handler.set_client(docker_client) - + def _check_coverages(): for coverage_file, coverage in build_handler.check_coverage(commented_file_path): entry.metadata.commented_files_coverages[commented_file_path][coverage_file] = coverage @@ -162,7 +205,12 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir: with build_handler, tqdm(total=len(steps), desc="Processing PR", leave=False) as pbar: try: for message, action in steps: - pbar.set_postfix({"doing": message, "started at": datetime.now().strftime("%d/%m, %H:%M:%S")}) + pbar.set_postfix( + { + "doing": message, + "started at": datetime.now().strftime("%d/%m, %H:%M:%S"), + } + ) action() pbar.update(1) except HandlerException as e: @@ -174,10 +222,16 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir: reset_repo_to_latest_commit(repo_path) if entry.metadata.successful: - entry.metadata.reason_for_failure = "" # was set to 'still processing', since it's done being processed and was successful, there are no reasons for failure + entry.metadata.reason_for_failure = "" # was set to 'still processing', since it's done being processed and was successful, there are no reasons for failure dataset.to_json(args.output) -def process_repo(repo_name: str, dataset: Dataset, repos_dir: str, cache: dict[str, dict[int, DatasetEntry]] = {}): + +def process_repo( + repo_name: str, + dataset: Dataset, + repos_dir: str, + cache: dict[str, dict[int, DatasetEntry]] = {}, +): repo = g.get_repo(repo_name) if repo.full_name in cache: dataset.entries.extend(cache[repo.full_name].values()) @@ -191,7 +245,13 @@ def process_repo(repo_name: str, dataset: Dataset, repos_dir: str, cache: dict[s pbar.set_postfix({"pr": pr.number}) process_pull(repo, pr, dataset, repos_dir, cache) -def process_repos(df: pd.DataFrame, dataset: Dataset, repos_dir: str, cache: dict[str, dict[int, DatasetEntry]] = {}): + +def process_repos( + df: pd.DataFrame, + dataset: Dataset, + repos_dir: str, + cache: dict[str, dict[int, DatasetEntry]] = {}, +): """ Processes the repos in the given csv file, extracting the good ones and creating the "triplets" for the dataset. @@ -206,23 +266,50 @@ def process_repos(df: pd.DataFrame, dataset: Dataset, repos_dir: str, cache: dic for _, row in df.iterrows(): repo_name = row["name"] assert isinstance(repo_name, str) - pbar.set_postfix({ - "repo": repo_name, - "started at": datetime.now().strftime("%d/%m, %H:%M:%S"), - "# triplets": f"{len(dataset)}/{len(dataset.entries)} ({len(dataset)/len(dataset.entries) if len(dataset.entries) > 0 else 0:.2%})" - }) + pbar.set_postfix( + { + "repo": repo_name, + "started at": datetime.now().strftime("%d/%m, %H:%M:%S"), + "# triplets": f"{len(dataset)}/{len(dataset.entries)} ({len(dataset)/len(dataset.entries) if len(dataset.entries) > 0 else 0:.2%})", + } + ) process_repo(repo_name, dataset, repos_dir, cache) pbar.update(1) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Creates the triplets for the CRAB dataset.') - parser.add_argument('csv_file', type=str, help='The csv file containing the projects (the results from clone_repos.py).') - parser.add_argument('-o', '--output', type=str, default="./dataset.json", help='The file in which the dataset will be contained. Default is "./dataset.json"') - parser.add_argument('-r', '--repos', type=str, default="./results/", help='The directory in which the repos were cloned (will be cloned if they aren\'t there already). Default: "./results/"') - parser.add_argument('-c', '--cache', type=str, help="The name of the output file from another run of this script. This is for when the script unexpectedly got interrupted and you want to resume from where you left off.") + parser.add_argument( + 'csv_file', + type=str, + help='The csv file containing the projects (the results from clone_repos.py).', + ) + parser.add_argument( + '-o', + '--output', + type=str, + default="./dataset.json", + help='The file in which the dataset will be contained. Default is "./dataset.json"', + ) + parser.add_argument( + '-r', + '--repos', + type=str, + default="./results/", + help='The directory in which the repos were cloned (will be cloned if they aren\'t there already). Default: "./results/"', + ) + parser.add_argument( + '-c', + '--cache', + type=str, + help="The name of the output file from another run of this script. This is for when the script unexpectedly got interrupted and you want to resume from where you left off.", + ) # parser.add_argument('-v', '--verbose', action='store_true', help='Prints the number of good projects.') - parser.add_argument("--only-repo", type=str, help="If this argument is not provided, all the repos in the '--repos' csv will be processed. If instead you want to run the script on a single repo (for testing purposes mainly) provide a string of form 'XXX/YYY' to this argument, where XXX is the owner of the repo and YYY is the name of the repo") + parser.add_argument( + "--only-repo", + type=str, + help="If this argument is not provided, all the repos in the '--repos' csv will be processed. If instead you want to run the script on a single repo (for testing purposes mainly) provide a string of form 'XXX/YYY' to this argument, where XXX is the owner of the repo and YYY is the name of the repo", + ) args = parser.parse_args() g = Github(os.environ["GITHUB_AUTH_TOKEN_CRAB"]) @@ -239,7 +326,7 @@ if __name__ == "__main__": cache_dataset = Dataset.from_json(args.cache) for cache_entry in cache_dataset.entries: cache[cache_entry.metadata.repo][cache_entry.metadata.pr_number] = cache_entry - + dataset = Dataset() try: # try and finally to save, regardless of an error occuring or the program finished correctly