Skip to content

Print consolidated log file for pytorch unit test automation scripts #1433

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 18, 2024
Merged

Print consolidated log file for pytorch unit test automation scripts #1433

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9df3943
Print consolidated log file for pytorch uts
jithunnair-amd Jun 14, 2024
6165adc
Update run_entire_tests subprocess call as well
jithunnair-amd Jun 17, 2024
a14a5bd
lint
jithunnair-amd Jun 18, 2024
b8233ca
Add ERROR string
jithunnair-amd Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 23 additions & 7 deletions .automation_scripts/run_pytorch_unit_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import os
import shutil
import subprocess
from subprocess import STDOUT, CalledProcessError

from collections import namedtuple
from datetime import datetime
Expand Down Expand Up @@ -63,6 +64,8 @@
"distributed/test_distributed_spawn"
]

CONSOLIDATED_LOG_FILE_NAME="pytorch_unit_tests.log"

def parse_xml_reports_as_dict(workflow_run_id, workflow_run_attempt, tag, workflow_name, path="."):
test_cases = {}
items_list = os.listdir(path)
Expand Down Expand Up @@ -198,6 +201,14 @@ def summarize_xml_files(path, workflow_name):

return res

def run_command_and_capture_output(cmd):
try:
print(f"Running command '{cmd}'")
with open(CONSOLIDATED_LOG_FILE_PATH, "a+") as output_file:
p = subprocess.run(cmd, shell=True, stdout=output_file, stderr=STDOUT, text=True)
except CalledProcessError as e:
print(f"ERROR: Cmd {cmd} failed with return code: {e.returncode}!")

def run_entire_tests(workflow_name, test_shell_path, overall_logs_path_current_run, test_reports_src):
if os.path.exists(test_reports_src):
shutil.rmtree(test_reports_src)
Expand All @@ -214,7 +225,7 @@ def run_entire_tests(workflow_name, test_shell_path, overall_logs_path_current_r
os.environ['TEST_CONFIG'] = 'inductor'
copied_logs_path = overall_logs_path_current_run + "inductor_xml_results_entire_tests/"
# use test.sh for tests execution
subprocess.call(test_shell_path, shell=True)
run_command_and_capture_output(test_shell_path)
copied_logs_path_destination = shutil.copytree(test_reports_src, copied_logs_path)
entire_results_dict = summarize_xml_files(copied_logs_path_destination, workflow_name)
return entire_results_dict
Expand All @@ -232,7 +243,7 @@ def run_priority_tests(workflow_name, test_run_test_path, overall_logs_path_curr
# use run_test.py for tests execution
default_priority_test_suites = " ".join(DEFAULT_CORE_TESTS)
command = "python " + test_run_test_path + " --include " + default_priority_test_suites + " --exclude-jit-executor --exclude-distributed-tests --verbose"
subprocess.call(command, shell=True)
run_command_and_capture_output(command)
del os.environ['HIP_VISIBLE_DEVICES']
elif workflow_name == "distributed":
os.environ['TEST_CONFIG'] = 'distributed'
Expand All @@ -241,7 +252,7 @@ def run_priority_tests(workflow_name, test_run_test_path, overall_logs_path_curr
# use run_test.py for tests execution
distributed_priority_test_suites = " ".join(DISTRIBUTED_CORE_TESTS)
command = "python " + test_run_test_path + " --include " + distributed_priority_test_suites + " --distributed-tests --verbose"
subprocess.call(command, shell=True)
run_command_and_capture_output(command)
del os.environ['HIP_VISIBLE_DEVICES']
copied_logs_path_destination = shutil.copytree(test_reports_src, copied_logs_path)
priority_results_dict = summarize_xml_files(copied_logs_path_destination, workflow_name)
Expand All @@ -261,7 +272,7 @@ def run_selected_tests(workflow_name, test_run_test_path, overall_logs_path_curr
# use run_test.py for tests execution
default_selected_test_suites = " ".join(selected_list)
command = "python " + test_run_test_path + " --include " + default_selected_test_suites + " --exclude-jit-executor --exclude-distributed-tests --verbose"
subprocess.call(command, shell=True)
run_command_and_capture_output(command)
del os.environ['HIP_VISIBLE_DEVICES']
elif workflow_name == "distributed":
os.environ['TEST_CONFIG'] = 'distributed'
Expand All @@ -270,7 +281,7 @@ def run_selected_tests(workflow_name, test_run_test_path, overall_logs_path_curr
# use run_test.py for tests execution
distributed_selected_test_suites = " ".join(selected_list)
command = "python " + test_run_test_path + " --include " + distributed_selected_test_suites + " --distributed-tests --verbose"
subprocess.call(command, shell=True)
run_command_and_capture_output(command)
del os.environ['HIP_VISIBLE_DEVICES']
elif workflow_name == "inductor":
os.environ['TEST_CONFIG'] = 'inductor'
Expand All @@ -287,11 +298,11 @@ def run_selected_tests(workflow_name, test_run_test_path, overall_logs_path_curr
if inductor_selected_test_suites != "":
inductor_selected_test_suites = inductor_selected_test_suites[:-1]
command = "python " + test_run_test_path + " --include " + inductor_selected_test_suites + " --verbose"
subprocess.call(command, shell=True)
run_command_and_capture_output(command)
if non_inductor_selected_test_suites != "":
non_inductor_selected_test_suites = non_inductor_selected_test_suites[:-1]
command = "python " + test_run_test_path + " --inductor --include " + non_inductor_selected_test_suites + " --verbose"
subprocess.call(command, shell=True)
run_command_and_capture_output(command)
copied_logs_path_destination = shutil.copytree(test_reports_src, copied_logs_path)
selected_results_dict = summarize_xml_files(copied_logs_path_destination, workflow_name)

Expand Down Expand Up @@ -336,6 +347,11 @@ def run_test_and_summarize_results(
# performed as Job ID
str_current_datetime = str(current_datetime)
overall_logs_path_current_run = repo_test_log_folder_path + str_current_datetime + "/"
os.mkdir(overall_logs_path_current_run)

global CONSOLIDATED_LOG_FILE_PATH
CONSOLIDATED_LOG_FILE_PATH = overall_logs_path_current_run + CONSOLIDATED_LOG_FILE_NAME

# Run entire tests for each workflow
if not priority_tests and not default_list and not distributed_list and not inductor_list:
# run entire tests for default, distributed and inductor workflows → use test.sh
Expand Down