Skip to content

Add pyright testcases / regression tests to the runtests script #10002

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Apr 14, 2023
Merged

Add pyright testcases / regression tests to the runtests script #10002

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dc637b8
Add pyright testcases / regression tests to the runtests script
Avasam Apr 3, 2023
2a20391
Apply suggestions from code review
Avasam Apr 14, 2023
e5686dc
Check for test_cases existance first
Avasam Apr 14, 2023
33ab3ad
Merge branch 'main' into pyright-testcases-runtest
Avasam Apr 14, 2023
83c06ae
Fix a few more minor nits
AlexWaygood Apr 14, 2023
5c8e6c2
make mypy happy
AlexWaygood Apr 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 55 additions & 17 deletions scripts/runtests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ def colored(text: str, color: str | None = None, on_color: str | None = None, at


_STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
_TESTCASES_CONFIG_FILE = "pyrightconfig.testcases.json"
_TESTCASES = "test_cases"
_NPX_ERROR_PATTERN = r"error (runn|find)ing npx"
_NPX_ERROR_MESSAGE = colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow")
_SUCCESS = colored("Success", "green")
_SKIPPED = colored("Skipped", "yellow")
_FAILED = colored("Failed", "red")
Expand Down Expand Up @@ -89,14 +93,15 @@ def main() -> None:
print("\nRunning check_new_syntax.py...")
check_new_syntax_result = subprocess.run([sys.executable, "tests/check_new_syntax.py"])

print(f"\nRunning Pyright on Python {_PYTHON_VERSION}...")
strict_params = _get_strict_params(path)
print(f"\nRunning Pyright ({'stricter' if strict_params else 'base' } configs) for Python {_PYTHON_VERSION}...")
pyright_result = subprocess.run(
[sys.executable, "tests/pyright_test.py", path, "--pythonversion", _PYTHON_VERSION] + _get_strict_params(path),
stderr=subprocess.PIPE,
[sys.executable, "tests/pyright_test.py", path, "--pythonversion", _PYTHON_VERSION] + strict_params,
capture_output=True,
text=True,
)
if re.match(r"error (runn|find)ing npx", pyright_result.stderr):
print(colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow"))
if re.match(_NPX_ERROR_PATTERN, pyright_result.stderr):
print(_NPX_ERROR_MESSAGE)
pyright_returncode = 0
pyright_skipped = True
else:
Expand Down Expand Up @@ -133,19 +138,47 @@ def main() -> None:
print("\nRunning pytype...")
pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])

print(f"\nRunning regression tests for Python {_PYTHON_VERSION}...")
regr_test_result = subprocess.run(
[sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub, "--python-version", _PYTHON_VERSION],
stderr=subprocess.PIPE,
text=True,
)
# No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
if "No test cases found" in regr_test_result.stderr:
test_cases_path = Path(path) / "@tests" / _TESTCASES if folder == "stubs" else Path(_TESTCASES)
if not test_cases_path.exists():
# No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
print(colored(f"\nRegression tests: No {_TESTCASES} folder for {stub!r}!", "green"))
pyright_testcases_returncode = 0
pyright_testcases_skipped = False
regr_test_returncode = 0
print(colored(f"\nNo test cases found for {stub!r}!", "green"))
else:
regr_test_returncode = regr_test_result.returncode
print(regr_test_result.stderr)
print(f"\nRunning Pyright regression tests for Python {_PYTHON_VERSION}...")
command = [
sys.executable,
"tests/pyright_test.py",
str(test_cases_path),
"--pythonversion",
_PYTHON_VERSION,
"-p",
_TESTCASES_CONFIG_FILE,
]
pyright_testcases_result = subprocess.run(command, capture_output=True, text=True)
if re.match(_NPX_ERROR_PATTERN, pyright_testcases_result.stderr):
print(_NPX_ERROR_MESSAGE)
pyright_testcases_returncode = 0
pyright_testcases_skipped = True
else:
print(pyright_result.stderr)
pyright_testcases_returncode = pyright_testcases_result.returncode
pyright_testcases_skipped = False

print(f"\nRunning mypy regression tests for Python {_PYTHON_VERSION}...")
regr_test_result = subprocess.run(
[sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub, "--python-version", _PYTHON_VERSION],
capture_output=True,
text=True,
)
# No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
if "No test cases found" in regr_test_result.stderr:
regr_test_returncode = 0
print(colored(f"\nNo test cases found for {stub!r}!", "green"))
else:
regr_test_returncode = regr_test_result.returncode
print(regr_test_result.stderr)

any_failure = any(
[
Expand All @@ -156,6 +189,7 @@ def main() -> None:
mypy_result.returncode,
getattr(stubtest_result, "returncode", 0),
getattr(pytype_result, "returncode", 0),
pyright_testcases_returncode,
regr_test_returncode,
]
)
Expand All @@ -180,7 +214,11 @@ def main() -> None:
print("pytype:", _SKIPPED)
else:
print("pytype:", _SUCCESS if pytype_result.returncode == 0 else _FAILED)
print("Regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)
if pyright_testcases_skipped:
print("Pyright regression tests:", _SKIPPED)
else:
print("Pyright regression tests:", _SUCCESS if pyright_testcases_returncode == 0 else _FAILED)
print("mypy regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)

sys.exit(int(any_failure))

Expand Down