mirror of
https://github.com/karma-riuk/crab.git
synced 2025-07-05 13:38:12 +02:00
formatted code
This commit is contained in:
Karma Riuk
143
pull_requests.py
143
pull_requests.py
@ -17,7 +17,7 @@ def get_good_projects(csv_file: str) -> pd.DataFrame:
|
||||
"""
|
||||
Extracts the good (the ones that compile and test successfully, and that
|
||||
have at least one test) from the given file.
|
||||
|
||||
|
||||
Parameters:
|
||||
csv_file (str): The csv file containing the projects.
|
||||
|
||||
@ -27,13 +27,17 @@ def get_good_projects(csv_file: str) -> pd.DataFrame:
|
||||
df = pd.read_csv(csv_file)
|
||||
return df.loc[(df['good_repo_for_crab'] == True) & (df['n_tests'] > 0)]
|
||||
|
||||
|
||||
def is_pull_good(pull: PullRequest, verbose: bool = False):
|
||||
return (
|
||||
has_only_1_comment(pull.get_commits(), pull.get_review_comments(), verbose=verbose)
|
||||
and pull.user.type != "Bot"
|
||||
)
|
||||
|
||||
def get_good_prs(repo: Repository, cache: dict[str, dict[int, DatasetEntry]] = {}) -> list[PullRequest]:
|
||||
|
||||
def get_good_prs(
|
||||
repo: Repository, cache: dict[str, dict[int, DatasetEntry]] = {}
|
||||
) -> list[PullRequest]:
|
||||
good_prs = []
|
||||
|
||||
potenially_good_prs = repo.get_pulls(state="closed")
|
||||
@ -42,7 +46,11 @@ def get_good_prs(repo: Repository, cache: dict[str, dict[int, DatasetEntry]] = {
|
||||
if number_of_prs == 0:
|
||||
return []
|
||||
|
||||
with tqdm(total=number_of_prs, desc=f"Extracting good PRs from {repo.full_name}", leave=False) as pbar:
|
||||
with tqdm(
|
||||
total=number_of_prs,
|
||||
desc=f"Extracting good PRs from {repo.full_name}",
|
||||
leave=False,
|
||||
) as pbar:
|
||||
for pr in potenially_good_prs:
|
||||
pbar.set_postfix({"new good found": len(good_prs), "pr_number": pr.number})
|
||||
if pr.merged_at is None or pr.number in cache.get(repo.full_name, set()):
|
||||
@ -54,8 +62,15 @@ def get_good_prs(repo: Repository, cache: dict[str, dict[int, DatasetEntry]] = {
|
||||
|
||||
return good_prs
|
||||
|
||||
|
||||
def run_git_cmd(cmd: list[str], repo_path: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["git", "-C", repo_path] + cmd, check=True, capture_output=True, text=True)
|
||||
return subprocess.run(
|
||||
["git", "-C", repo_path] + cmd,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def ensure_full_history(repo_path: str) -> None:
|
||||
result = run_git_cmd(["rev-parse", "--is-shallow-repository"], repo_path)
|
||||
@ -63,11 +78,19 @@ def ensure_full_history(repo_path: str) -> None:
|
||||
if result.stdout.strip() == "true":
|
||||
run_git_cmd(["fetch", "--unshallow"], repo_path)
|
||||
|
||||
|
||||
def reset_repo_to_latest_commit(repo_path: str) -> None:
|
||||
current_branch = run_git_cmd(["rev-parse", "--abbrev-ref", "HEAD"], repo_path).stdout.strip()
|
||||
run_git_cmd(["reset", "--hard", current_branch], repo_path)
|
||||
|
||||
def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir: str, cache: dict[str, dict[int, DatasetEntry]] = {}):
|
||||
|
||||
def process_pull(
|
||||
repo: Repository,
|
||||
pr: PullRequest,
|
||||
dataset: Dataset,
|
||||
repos_dir: str,
|
||||
cache: dict[str, dict[int, DatasetEntry]] = {},
|
||||
):
|
||||
if pr.number in cache.get(repo.full_name, set()):
|
||||
dataset.entries.append(cache[repo.full_name][pr.number])
|
||||
return
|
||||
@ -80,7 +103,9 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir:
|
||||
last_commit = commits[-1]
|
||||
|
||||
try:
|
||||
diffs_before = {file.filename: file.patch for file in repo.compare(pr.base.sha, first_commit.sha).files}
|
||||
diffs_before = {
|
||||
file.filename: file.patch for file in repo.compare(pr.base.sha, first_commit.sha).files
|
||||
}
|
||||
except GithubException as e:
|
||||
return
|
||||
|
||||
@ -91,12 +116,21 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir:
|
||||
commented_file_path = comment.path
|
||||
|
||||
try:
|
||||
diffs_after = {file.filename: file.patch for file in repo.compare(first_commit.sha, last_commit.sha).files}
|
||||
diffs_after = {
|
||||
file.filename: file.patch
|
||||
for file in repo.compare(first_commit.sha, last_commit.sha).files
|
||||
}
|
||||
except GithubException as e:
|
||||
return
|
||||
|
||||
entry = DatasetEntry(
|
||||
metadata=Metadata(repo.full_name, pr.number, pr.merge_commit_sha, {comment_text: commented_file_path}, reason_for_failure="Was still being processed"),
|
||||
metadata=Metadata(
|
||||
repo.full_name,
|
||||
pr.number,
|
||||
pr.merge_commit_sha,
|
||||
{comment_text: commented_file_path},
|
||||
reason_for_failure="Was still being processed",
|
||||
),
|
||||
files={file.filename: FileData(file.filename) for file in pr.get_files()},
|
||||
diffs_before=diffs_before,
|
||||
comments=[comment_text],
|
||||
@ -126,16 +160,25 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir:
|
||||
# raise e
|
||||
return entry.metadata.successful
|
||||
|
||||
if not _try_cmd(lambda: ensure_full_history(repo_path), "Couldn't ensure the full history of the repo (fetch --unshallow)"):
|
||||
if not _try_cmd(
|
||||
lambda: ensure_full_history(repo_path),
|
||||
"Couldn't ensure the full history of the repo (fetch --unshallow)",
|
||||
):
|
||||
return
|
||||
|
||||
try:
|
||||
run_git_cmd(["checkout", pr.merge_commit_sha], repo_path)
|
||||
except subprocess.CalledProcessError:
|
||||
if not _try_cmd(lambda: run_git_cmd(["fetch", "origin", f"pull/{pr.number}/merge"], repo_path), "Couldn't fetch the PR's merge commit"):
|
||||
if not _try_cmd(
|
||||
lambda: run_git_cmd(["fetch", "origin", f"pull/{pr.number}/merge"], repo_path),
|
||||
"Couldn't fetch the PR's merge commit",
|
||||
):
|
||||
return
|
||||
|
||||
if not _try_cmd(lambda: run_git_cmd(["checkout", pr.merge_commit_sha], repo_path), "Coudln't checkout the PR's merge commit (even after fetching the pull/<pr_number>/merge)"):
|
||||
|
||||
if not _try_cmd(
|
||||
lambda: run_git_cmd(["checkout", pr.merge_commit_sha], repo_path),
|
||||
"Coudln't checkout the PR's merge commit (even after fetching the pull/<pr_number>/merge)",
|
||||
):
|
||||
return
|
||||
|
||||
build_handler = get_build_handler(repos_dir, repo.full_name, updates)
|
||||
@ -146,7 +189,7 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir:
|
||||
return
|
||||
entry.metadata.build_system = build_handler.get_type()
|
||||
build_handler.set_client(docker_client)
|
||||
|
||||
|
||||
def _check_coverages():
|
||||
for coverage_file, coverage in build_handler.check_coverage(commented_file_path):
|
||||
entry.metadata.commented_files_coverages[commented_file_path][coverage_file] = coverage
|
||||
@ -162,7 +205,12 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir:
|
||||
with build_handler, tqdm(total=len(steps), desc="Processing PR", leave=False) as pbar:
|
||||
try:
|
||||
for message, action in steps:
|
||||
pbar.set_postfix({"doing": message, "started at": datetime.now().strftime("%d/%m, %H:%M:%S")})
|
||||
pbar.set_postfix(
|
||||
{
|
||||
"doing": message,
|
||||
"started at": datetime.now().strftime("%d/%m, %H:%M:%S"),
|
||||
}
|
||||
)
|
||||
action()
|
||||
pbar.update(1)
|
||||
except HandlerException as e:
|
||||
@ -174,10 +222,16 @@ def process_pull(repo: Repository, pr: PullRequest, dataset: Dataset, repos_dir:
|
||||
reset_repo_to_latest_commit(repo_path)
|
||||
|
||||
if entry.metadata.successful:
|
||||
entry.metadata.reason_for_failure = "" # was set to 'still processing', since it's done being processed and was successful, there are no reasons for failure
|
||||
entry.metadata.reason_for_failure = "" # was set to 'still processing', since it's done being processed and was successful, there are no reasons for failure
|
||||
dataset.to_json(args.output)
|
||||
|
||||
def process_repo(repo_name: str, dataset: Dataset, repos_dir: str, cache: dict[str, dict[int, DatasetEntry]] = {}):
|
||||
|
||||
def process_repo(
|
||||
repo_name: str,
|
||||
dataset: Dataset,
|
||||
repos_dir: str,
|
||||
cache: dict[str, dict[int, DatasetEntry]] = {},
|
||||
):
|
||||
repo = g.get_repo(repo_name)
|
||||
if repo.full_name in cache:
|
||||
dataset.entries.extend(cache[repo.full_name].values())
|
||||
@ -191,7 +245,13 @@ def process_repo(repo_name: str, dataset: Dataset, repos_dir: str, cache: dict[s
|
||||
pbar.set_postfix({"pr": pr.number})
|
||||
process_pull(repo, pr, dataset, repos_dir, cache)
|
||||
|
||||
def process_repos(df: pd.DataFrame, dataset: Dataset, repos_dir: str, cache: dict[str, dict[int, DatasetEntry]] = {}):
|
||||
|
||||
def process_repos(
|
||||
df: pd.DataFrame,
|
||||
dataset: Dataset,
|
||||
repos_dir: str,
|
||||
cache: dict[str, dict[int, DatasetEntry]] = {},
|
||||
):
|
||||
"""
|
||||
Processes the repos in the given csv file, extracting the good ones and
|
||||
creating the "triplets" for the dataset.
|
||||
@ -206,23 +266,50 @@ def process_repos(df: pd.DataFrame, dataset: Dataset, repos_dir: str, cache: dic
|
||||
for _, row in df.iterrows():
|
||||
repo_name = row["name"]
|
||||
assert isinstance(repo_name, str)
|
||||
pbar.set_postfix({
|
||||
"repo": repo_name,
|
||||
"started at": datetime.now().strftime("%d/%m, %H:%M:%S"),
|
||||
"# triplets": f"{len(dataset)}/{len(dataset.entries)} ({len(dataset)/len(dataset.entries) if len(dataset.entries) > 0 else 0:.2%})"
|
||||
})
|
||||
pbar.set_postfix(
|
||||
{
|
||||
"repo": repo_name,
|
||||
"started at": datetime.now().strftime("%d/%m, %H:%M:%S"),
|
||||
"# triplets": f"{len(dataset)}/{len(dataset.entries)} ({len(dataset)/len(dataset.entries) if len(dataset.entries) > 0 else 0:.2%})",
|
||||
}
|
||||
)
|
||||
process_repo(repo_name, dataset, repos_dir, cache)
|
||||
pbar.update(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Creates the triplets for the CRAB dataset.')
|
||||
parser.add_argument('csv_file', type=str, help='The csv file containing the projects (the results from clone_repos.py).')
|
||||
parser.add_argument('-o', '--output', type=str, default="./dataset.json", help='The file in which the dataset will be contained. Default is "./dataset.json"')
|
||||
parser.add_argument('-r', '--repos', type=str, default="./results/", help='The directory in which the repos were cloned (will be cloned if they aren\'t there already). Default: "./results/"')
|
||||
parser.add_argument('-c', '--cache', type=str, help="The name of the output file from another run of this script. This is for when the script unexpectedly got interrupted and you want to resume from where you left off.")
|
||||
parser.add_argument(
|
||||
'csv_file',
|
||||
type=str,
|
||||
help='The csv file containing the projects (the results from clone_repos.py).',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-o',
|
||||
'--output',
|
||||
type=str,
|
||||
default="./dataset.json",
|
||||
help='The file in which the dataset will be contained. Default is "./dataset.json"',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-r',
|
||||
'--repos',
|
||||
type=str,
|
||||
default="./results/",
|
||||
help='The directory in which the repos were cloned (will be cloned if they aren\'t there already). Default: "./results/"',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-c',
|
||||
'--cache',
|
||||
type=str,
|
||||
help="The name of the output file from another run of this script. This is for when the script unexpectedly got interrupted and you want to resume from where you left off.",
|
||||
)
|
||||
# parser.add_argument('-v', '--verbose', action='store_true', help='Prints the number of good projects.')
|
||||
parser.add_argument("--only-repo", type=str, help="If this argument is not provided, all the repos in the '--repos' csv will be processed. If instead you want to run the script on a single repo (for testing purposes mainly) provide a string of form 'XXX/YYY' to this argument, where XXX is the owner of the repo and YYY is the name of the repo")
|
||||
parser.add_argument(
|
||||
"--only-repo",
|
||||
type=str,
|
||||
help="If this argument is not provided, all the repos in the '--repos' csv will be processed. If instead you want to run the script on a single repo (for testing purposes mainly) provide a string of form 'XXX/YYY' to this argument, where XXX is the owner of the repo and YYY is the name of the repo",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
g = Github(os.environ["GITHUB_AUTH_TOKEN_CRAB"])
|
||||
@ -239,7 +326,7 @@ if __name__ == "__main__":
|
||||
cache_dataset = Dataset.from_json(args.cache)
|
||||
for cache_entry in cache_dataset.entries:
|
||||
cache[cache_entry.metadata.repo][cache_entry.metadata.pr_number] = cache_entry
|
||||
|
||||
|
||||
dataset = Dataset()
|
||||
try:
|
||||
# try and finally to save, regardless of an error occuring or the program finished correctly
|
||||
|
||||
Reference in New Issue
Block a user