# Arguments # --cpython: Path to cpython source code # --updated-libs: Libraries that have been updated in RustPython import argparse import dataclasses import difflib import pathlib from typing import Optional import warnings import requests from jinja2 import Environment, FileSystemLoader parser = argparse.ArgumentParser( description="Find equivalent files in cpython and rustpython" ) parser.add_argument( "--cpython", type=pathlib.Path, required=True, help="Path to cpython source code" ) parser.add_argument( "--notes", type=pathlib.Path, required=False, help="Path to notes file" ) args = parser.parse_args() def check_pr(pr_id: str) -> bool: if pr_id.startswith("#"): pr_id = pr_id[1:] int_pr_id = int(pr_id) req = f"https://api.github.com/repos/RustPython/RustPython/pulls/{int_pr_id}" response = requests.get(req).json() return response["merged_at"] is not None @dataclasses.dataclass class LibUpdate: pr: Optional[str] = None done: bool = True def parse_updated_lib_issue(issue_body: str) -> dict[str, LibUpdate]: lines = issue_body.splitlines() updated_libs = {} for line in lines: if line.strip().startswith("- "): line = line.strip()[2:] out = line.split(" ") out = [x for x in out if x] assert len(out) < 3 if len(out) == 1: updated_libs[out[0]] = LibUpdate() elif len(out) == 2: updated_libs[out[0]] = LibUpdate(out[1], check_pr(out[1])) return updated_libs def get_updated_libs() -> dict[str, LibUpdate]: issue_id = "5736" req = f"https://api.github.com/repos/RustPython/RustPython/issues/{issue_id}" response = requests.get(req).json() return parse_updated_lib_issue(response["body"]) updated_libs = get_updated_libs() if not args.cpython.exists(): raise FileNotFoundError(f"Path {args.cpython} does not exist") if not args.cpython.is_dir(): raise NotADirectoryError(f"Path {args.cpython} is not a directory") if not args.cpython.is_absolute(): args.cpython = args.cpython.resolve() notes: dict = {} if args.notes: # check if the file exists in the rustpython lib directory notes_path = args.notes if notes_path.exists(): with open(notes_path) as f: for line in f: line = line.strip() if not line.startswith("//") and line: line_split = line.split(" ") if len(line_split) > 1: rest = " ".join(line_split[1:]) if line_split[0] in notes: notes[line_split[0]].append(rest) else: notes[line_split[0]] = [rest] else: raise ValueError(f"Invalid note: {line}") else: raise FileNotFoundError(f"Path {notes_path} does not exist") cpython_lib = args.cpython / "Lib" rustpython_lib = pathlib.Path(__file__).parent.parent / "Lib" assert rustpython_lib.exists(), ( "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place" ) ignored_objs = ["__pycache__", "test"] # loop through the top-level directories in the cpython lib directory libs = [] for path in cpython_lib.iterdir(): if path.is_dir() and path.name not in ignored_objs: # add the directory name to the list of libraries libs.append(path.name) elif path.is_file() and path.name.endswith(".py") and path.name not in ignored_objs: # add the file name to the list of libraries libs.append(path.name) tests = [] cpython_lib_test = cpython_lib / "test" for path in cpython_lib_test.iterdir(): if ( path.is_dir() and path.name not in ignored_objs and path.name.startswith("test_") ): # add the directory name to the list of libraries tests.append(path.name) elif ( path.is_file() and path.name.endswith(".py") and path.name not in ignored_objs and path.name.startswith("test_") ): # add the file name to the list of libraries file_name = path.name.replace("test_", "") if file_name not in libs and file_name.replace(".py", "") not in libs: tests.append(path.name) def check_diff(file1, file2): try: with open(file1, "r") as f1, open(file2, "r") as f2: f1_lines = f1.readlines() f2_lines = f2.readlines() diff = difflib.unified_diff(f1_lines, f2_lines, lineterm="") diff_lines = list(diff) return len(diff_lines) except UnicodeDecodeError: return False def check_completion_pr(display_name): for lib in updated_libs: if lib == str(display_name): return updated_libs[lib].done, updated_libs[lib].pr return False, None def check_test_completion(rustpython_path, cpython_path): if rustpython_path.exists() and rustpython_path.is_file(): if cpython_path.exists() and cpython_path.is_file(): if not rustpython_path.exists() or not rustpython_path.is_file(): return False elif check_diff(rustpython_path, cpython_path) > 0: return False return True return False def check_lib_completion(rustpython_path, cpython_path): test_name = "test_" + rustpython_path.name rustpython_test_path = rustpython_lib / "test" / test_name cpython_test_path = cpython_lib / "test" / test_name if cpython_test_path.exists() and not check_test_completion( rustpython_test_path, cpython_test_path ): return False if rustpython_path.exists() and rustpython_path.is_file(): if check_diff(rustpython_path, cpython_path) > 0: return False return True return False def handle_notes(display_path) -> list[str]: if str(display_path) in notes: res = notes[str(display_path)] # remove the note from the notes list del notes[str(display_path)] return res return [] @dataclasses.dataclass class Output: name: str pr: Optional[str] completed: Optional[bool] notes: list[str] update_libs_output = [] add_libs_output = [] for path in libs: # check if the file exists in the rustpython lib directory rustpython_path = rustpython_lib / path # remove the file extension if it exists display_path = pathlib.Path(path).with_suffix("") (completed, pr) = check_completion_pr(display_path) if rustpython_path.exists(): if not completed: # check if the file exists in the cpython lib directory cpython_path = cpython_lib / path # check if the file exists in the rustpython lib directory if rustpython_path.exists() and rustpython_path.is_file(): completed = check_lib_completion(rustpython_path, cpython_path) update_libs_output.append( Output(str(display_path), pr, completed, handle_notes(display_path)) ) else: if pr is not None and completed: update_libs_output.append( Output(str(display_path), pr, None, handle_notes(display_path)) ) else: add_libs_output.append( Output(str(display_path), pr, None, handle_notes(display_path)) ) update_tests_output = [] add_tests_output = [] for path in tests: # check if the file exists in the rustpython lib directory rustpython_path = rustpython_lib / "test" / path # remove the file extension if it exists display_path = pathlib.Path(path).with_suffix("") (completed, pr) = check_completion_pr(display_path) if rustpython_path.exists(): if not completed: # check if the file exists in the cpython lib directory cpython_path = cpython_lib / "test" / path # check if the file exists in the rustpython lib directory if rustpython_path.exists() and rustpython_path.is_file(): completed = check_lib_completion(rustpython_path, cpython_path) update_tests_output.append( Output(str(display_path), pr, completed, handle_notes(display_path)) ) else: if pr is not None and completed: update_tests_output.append( Output(str(display_path), pr, None, handle_notes(display_path)) ) else: add_tests_output.append( Output(str(display_path), pr, None, handle_notes(display_path)) ) for note in notes: # add a warning for each note that is not attached to a file for n in notes[note]: warnings.warn(f"Unattached Note: {note} - {n}") env = Environment(loader=FileSystemLoader(".")) template = env.get_template("checklist_template.md") output = template.render( update_libs=update_libs_output, add_libs=add_libs_output, update_tests=update_tests_output, add_tests=add_tests_output, ) print(output)