Skip to content

Commit 8d68984

Browse files
mikekgfbmalfet
authored andcommitted
Updown (#682)
* improve updown parser, and use in README.md execution * cut/paste errors * typo: true -> false * we scan each partial line, so need to suppress at partial line level :( * make it twice as nice
1 parent 4d2f76c commit 8d68984

File tree

5 files changed

+190
-54
lines changed

5 files changed

+190
-54
lines changed

.github/workflows/run-readme-periodic.yml

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,19 @@ jobs:
3535
# echo "::endgroup::"
3636
3737
echo "::group::Create script"
38-
echo "set -eou pipefail" > ./we-run-this.sh
39-
python3 scripts/process-readme.py >> ./we-run-this.sh
40-
echo "exit 1" >> ./we-run-this.sh
38+
python3 scripts/updown.py --file README.md > ./we-run-this.sh
39+
# for good measure, if something happened to updown processor,
40+
# and it did not error out, fail with an exit 1
41+
echo "exit 1" >> ./we-run-this.sh
4142
echo "::endgroup::"
4243
4344
echo "::group::Run This"
45+
echo "*******************************************"
46+
cat ./we-run-this.sh
47+
echo "*******************************************"
4448
bash -x ./we-run-this.sh
4549
4650
echo "tests complete"
47-
echo "******************************************"
51+
echo "*******************************************"
4852
echo "::endgroup::"
53+

.github/workflows/run-readme-pr-macos.yml

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,18 +39,19 @@ jobs:
3939
# echo "::endgroup::"
4040

4141
echo "::group::Create script"
42-
echo "set -eou pipefail" > ./readme-commands.sh
43-
python3 scripts/process-readme.py >> ./readme-commands.sh
44-
echo "exit 1" >> ./readme-commands.sh
42+
python3 scripts/updown.py --file README.md --replace llama3:stories15M --suppress huggingface-cli,HF_TOKEN > ./we-run-this.sh
43+
# for good measure, if something happened to updown processor,
44+
# and it did not error out, fail with an exit 1
45+
echo "exit 1" >> ./we-run-this.sh
4546
echo "::endgroup::"
46-
47+
4748
echo "::group::Run This"
48-
grep -v login ./readme-commands.sh | sed -e '1,$s/llama3/stories15M/g' > ./we-run-this.sh
4949
echo "*******************************************"
5050
cat ./we-run-this.sh
5151
echo "*******************************************"
5252
bash -x ./we-run-this.sh
53-
53+
5454
echo "tests complete"
5555
echo "*******************************************"
5656
echo "::endgroup::"
57+

.github/workflows/run-readme-pr.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,13 @@ jobs:
2626
# echo "::endgroup::"
2727
2828
echo "::group::Create script"
29-
echo "set -eou pipefail" > ./readme-commands.sh
30-
python3 scripts/process-readme.py >> ./readme-commands.sh
31-
echo "exit 1" >> ./readme-commands.sh
29+
python3 scripts/updown.py --file README.md --replace llama3:stories15M --suppress huggingface-cli,HF_TOKEN > ./we-run-this.sh
30+
# for good measure, if something happened to updown processor,
31+
# and it did not error out, fail with an exit 1
32+
echo "exit 1" >> ./we-run-this.sh
3233
echo "::endgroup::"
3334
3435
echo "::group::Run This"
35-
grep -v login ./readme-commands.sh | sed -e '1,$s/llama3/stories15M/g' > ./we-run-this.sh
3636
echo "*******************************************"
3737
cat ./we-run-this.sh
3838
echo "*******************************************"

scripts/process-readme.py

Lines changed: 0 additions & 40 deletions
This file was deleted.

scripts/updown.py

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
import argparse
2+
import os
3+
import re
4+
5+
6+
def output(*args, **kwargs):
7+
"""
8+
Prints the given arguments after performing replacements and suppressions.
9+
Args:
10+
*args: The values to print.
11+
replace_list (list, optional): A list of tuples where the first element of each tuple is replaced with the second element in the output. Defaults to None.
12+
suppress_list (list, optional): A list of strings. If any string in this list is found in the output, the line is not printed. Defaults to None.
13+
file (file, optional): A file-like object (stream). Defaults to the current sys.stdout.
14+
end (str, optional): Specifies what to print at the end. Defaults to '\n'.
15+
Returns:
16+
None
17+
"""
18+
# expand kwargs, make it error to not specify replace list or suppress list
19+
# because we should always have these for updown processor
20+
replace_list = kwargs["replace_list"] # .get("replace_list", None)
21+
suppress_list = kwargs["suppress_list"] # get("suppress_list", None)
22+
file = kwargs.get("file", None)
23+
end = kwargs.get("end", "\n")
24+
25+
# Convert args to a list of strings
26+
str_args = [str(arg) for arg in args]
27+
# If replace_list is provided
28+
if replace_list:
29+
# For each tuple in replace_list
30+
for before, after in replace_list:
31+
# Replace all occurrences of 'before' with 'after' in each string
32+
str_args = [arg.replace(before, after) for arg in str_args]
33+
# If suppress_list is provided
34+
if suppress_list:
35+
# For each string in suppress_list
36+
for suppress in suppress_list:
37+
# If the suppress string is found in any of the str_args, return without printing
38+
if any(suppress in arg for arg in str_args):
39+
return
40+
# Print the modified strings
41+
print(*str_args, file=file, end=end)
42+
43+
44+
def command_regexp(command):
45+
"""
46+
Processes a file based on the given predicates, replacements, and suppressions.
47+
Args:
48+
filename (str): The name of the file to process.
49+
predicate_list (list): A list of predicates to match in the file.
50+
replace_list (list): A list of tuples where the first element of each tuple is replaced with the second element in the output.
51+
suppress_list (list): A list of strings. If any string in this list is found in the output, the line is not printed.
52+
Returns:
53+
None
54+
"""
55+
return rf"^\[\s*{command}\s+(\w+)\s*\]\s*:\s*(.*)"
56+
57+
58+
def updown_processor(filename, predicate_list, replace_list, suppress_list):
59+
"""
60+
Processes a file based on the given predicates, replacements, and suppressions.
61+
Args:
62+
filename (str): The name of the file to process.
63+
predicate_list (list): A list of predicates to match in the file.
64+
replace_list (list): A list of tuples where the first element of each tuple is replaced with the second element in the output.
65+
suppress_list (list): A list of strings. If any string in this list is found in the output, the line is not printed.
66+
Returns:
67+
None
68+
"""
69+
with open(filename, "r") as file:
70+
lines = file.readlines()
71+
print_flag = False
72+
output("set -eou pipefail", replace_list=None, suppress_list=None)
73+
for i, line in enumerate(lines):
74+
shell = command_regexp("shell")
75+
prefix = command_regexp("prefix")
76+
skip = command_regexp("skip")
77+
end = command_regexp("end")
78+
if match := re.search(shell, line):
79+
# Extract the matched groups
80+
predicate = match.group(1)
81+
trailing_command = match.group(2)
82+
if predicate in predicate_list:
83+
output(
84+
trailing_command,
85+
replace_list=replace_list,
86+
suppress_list=suppress_list,
87+
)
88+
elif match := re.search(prefix, line):
89+
# Extract the matched groups
90+
predicate = match.group(1)
91+
trailing_command = match.group(2)
92+
if predicate in predicate_list:
93+
output(
94+
trailing_command,
95+
end="",
96+
replace_list=replace_list,
97+
suppress_list=suppress_list,
98+
)
99+
elif match := re.search(skip, line):
100+
# Extract the matched groups
101+
predicate = match.group(1)
102+
trailing_command = match.group(2)
103+
if predicate in predicate_list:
104+
if trailing_command == "begin":
105+
output(
106+
"if false; then",
107+
replace_list=replace_list,
108+
suppress_list=suppress_list,
109+
)
110+
elif trailing_command == "end":
111+
output(
112+
"fi",
113+
replace_list=replace_list,
114+
suppress_list=suppress_list,
115+
)
116+
else:
117+
output(
118+
f"echo 'error in line {i} of {filename}'\nexit 1;",
119+
suppress_list=None,
120+
replace_list=None,
121+
)
122+
exit(1)
123+
elif match := re.search(end, line):
124+
# Extract the matched groups
125+
predicate = match.group(1)
126+
trailing_command = match.group(2)
127+
if predicate in predicate_list:
128+
output(
129+
"exit 0",
130+
replace_list=replace_list,
131+
suppress_list=suppress_list,
132+
)
133+
return
134+
elif line.startswith("```"):
135+
print_flag = not print_flag
136+
elif print_flag:
137+
output(line, end="", replace_list=replace_list, suppress_list=suppress_list)
138+
139+
output(
140+
"echo 'reached end of file without exit command'\nexit 1;",
141+
suppress_list=None,
142+
replace_list=None,
143+
)
144+
145+
146+
# Initialize the ArgumentParser object
147+
parser = argparse.ArgumentParser()
148+
# Add arguments
149+
parser.add_argument("-f", "--filename", help="Input filename", default="README.md")
150+
parser.add_argument("-p", "--predicate", help="Input predicates", default="")
151+
parser.add_argument("-r", "--replace", help="Input replace pairs", default="")
152+
parser.add_argument("-s", "--suppress", help="Input suppress strings", default="")
153+
args = parser.parse_args()
154+
# Get filename
155+
filename = args.filename
156+
# Check if file exists
157+
if not os.path.isfile(filename):
158+
output(f"echo 'File {filename} does not exist.'\n exit 1;")
159+
exit(1)
160+
# Get predicates, split by comma, and add "default"
161+
predicate_list = args.predicate.split(",") if args.predicate else []
162+
predicate_list.append("default")
163+
# Get replace pairs, split by comma, and turn into a list of tuples
164+
replace_list = (
165+
[tuple(pair.split(":")) for pair in args.replace.split(",")] if args.replace else []
166+
)
167+
# Get suppress strings, split by comma
168+
suppress_list = args.suppress.split(",") if args.suppress else []
169+
# Call updown_processor function
170+
updown_processor(filename, predicate_list, replace_list, suppress_list)

0 commit comments

Comments
 (0)