Skip to content

Commit 71190ac

Browse files
gvanrossumFidget-Spinner
authored andcommitted
pythongh-102654: Insert #line directives in generated_cases.c.h (python#102669)
This behavior is optional, because in some extreme cases it may just make debugging harder. The tool defaults it to off, but it is on in Makefile.pre.in. Also note that this makes diffs to generated_cases.c.h noisier, since whenever you insert or delete a line in bytecodes.c, all subsequent #line directives will change.
1 parent 939f5a6 commit 71190ac

File tree

3 files changed

+77
-19
lines changed

3 files changed

+77
-19
lines changed

Makefile.pre.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1485,6 +1485,7 @@ regen-cases:
14851485
PYTHONPATH=$(srcdir)/Tools/cases_generator \
14861486
$(PYTHON_FOR_REGEN) \
14871487
$(srcdir)/Tools/cases_generator/generate_cases.py \
1488+
--emit-line-directives \
14881489
-o $(srcdir)/Python/generated_cases.c.h.new \
14891490
-m $(srcdir)/Python/opcode_metadata.h.new \
14901491
$(srcdir)/Python/bytecodes.c

Python/generated_cases.c.h

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Tools/cases_generator/generate_cases.py

Lines changed: 73 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
from enum import Enum, auto
1717

18+
import lexer as lx
1819
import parser
1920
from parser import StackEffect
2021
from parser import TypeSrcLiteral, TypeSrcConst, TypeSrcLocals, TypeSrcStackInput
@@ -87,6 +88,9 @@
8788
arg_parser.add_argument(
8889
"-m", "--metadata", type=str, help="Generated metadata", default=DEFAULT_METADATA_OUTPUT
8990
)
91+
arg_parser.add_argument(
92+
"-l", "--emit-line-directives", help="Emit #line directives", action="store_true"
93+
)
9094
arg_parser.add_argument(
9195
"input", nargs=argparse.REMAINDER, help="Instruction definition file(s)"
9296
)
@@ -158,21 +162,53 @@ class Formatter:
158162
stream: typing.TextIO
159163
prefix: str
160164
postfix: str
161-
162-
def __init__(self, stream: typing.TextIO, indent: int) -> None:
165+
emit_line_directives: bool = False
166+
lineno: int # Next line number, 1-based
167+
filename: str # Slightly improved stream.filename
168+
nominal_lineno: int
169+
nominal_filename: str
170+
171+
def __init__(
172+
self, stream: typing.TextIO, indent: int, emit_line_directives: bool = False
173+
) -> None:
163174
self.stream = stream
164175
self.prefix = " " * indent
165176
self.postfix = ""
177+
self.emit_line_directives = emit_line_directives
178+
self.lineno = 1
179+
# Make filename more user-friendly and less platform-specific
180+
filename = self.stream.name.replace("\\", "/")
181+
if filename.startswith("./"):
182+
filename = filename[2:]
183+
if filename.endswith(".new"):
184+
filename = filename[:-4]
185+
self.filename = filename
186+
self.nominal_lineno = 1
187+
self.nominal_filename = filename
166188

167189
def write_raw(self, s: str) -> None:
168190
self.stream.write(s)
191+
newlines = s.count("\n")
192+
self.lineno += newlines
193+
self.nominal_lineno += newlines
169194

170195
def emit(self, arg: str) -> None:
171196
if arg:
172197
self.write_raw(f"{self.prefix}{arg}{self.postfix}\n")
173198
else:
174199
self.write_raw("\n")
175200

201+
def set_lineno(self, lineno: int, filename: str) -> None:
202+
if self.emit_line_directives:
203+
if lineno != self.nominal_lineno or filename != self.nominal_filename:
204+
self.emit(f'#line {lineno} "{filename}"')
205+
self.nominal_lineno = lineno
206+
self.nominal_filename = filename
207+
208+
def reset_lineno(self) -> None:
209+
if self.lineno != self.nominal_lineno or self.filename != self.nominal_filename:
210+
self.set_lineno(self.lineno + 1, self.filename)
211+
176212
@contextlib.contextmanager
177213
def indent(self):
178214
self.prefix += " "
@@ -253,6 +289,7 @@ class Instruction:
253289
block: parser.Block
254290
block_text: list[str] # Block.text, less curlies, less PREDICT() calls
255291
predictions: list[str] # Prediction targets (instruction names)
292+
block_line: int # First line of block in original code
256293

257294
# Computed by constructor
258295
always_exits: bool
@@ -278,7 +315,7 @@ def __init__(self, inst: parser.InstDef):
278315
self.kind = inst.kind
279316
self.name = inst.name
280317
self.block = inst.block
281-
self.block_text, self.check_eval_breaker, self.predictions = \
318+
self.block_text, self.check_eval_breaker, self.predictions, self.block_line = \
282319
extract_block_text(self.block)
283320
self.always_exits = always_exits(self.block_text)
284321
self.cache_effects = [
@@ -587,7 +624,13 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None
587624
assert dedent <= 0
588625
extra = " " * -dedent
589626
names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"})
627+
offset = 0
628+
context = self.block.context
629+
assert context != None
630+
filename = context.owner.filename
590631
for line in self.block_text:
632+
out.set_lineno(self.block_line + offset, filename)
633+
offset += 1
591634
if m := re.match(r"(\s*)U_INST\((.+)\);\s*$", line):
592635
space, label = m.groups()
593636
out.emit(f"UOP_{label}();")
@@ -618,6 +661,7 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None
618661
out.write_raw(f"{space}if ({cond}) goto {label};{out.postfix}\n")
619662
elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line):
620663
if not self.register:
664+
out.reset_lineno()
621665
space = extra + m.group(1)
622666
for ieff in self.input_effects:
623667
if ieff.name in names_to_skip:
@@ -633,6 +677,7 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None
633677
out.write_raw(f"{space}Py_{decref}({ieff.name});\n")
634678
else:
635679
out.write_raw(extra + line.rstrip("\n") + out.postfix + "\n")
680+
out.reset_lineno()
636681

637682

638683
InstructionOrCacheEffect = Instruction | parser.CacheEffect
@@ -707,6 +752,7 @@ class Analyzer:
707752
output_filename: str
708753
metadata_filename: str
709754
errors: int = 0
755+
emit_line_directives: bool = False
710756

711757
def __init__(self, input_filenames: list[str], output_filename: str, metadata_filename: str):
712758
"""Read the input file."""
@@ -772,6 +818,10 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None:
772818
with open(filename) as file:
773819
src = file.read()
774820

821+
# Make filename more user-friendly and less platform-specific
822+
filename = filename.replace("\\", "/")
823+
if filename.startswith("./"):
824+
filename = filename[2:]
775825
psr = parser.Parser(src, filename=filename)
776826

777827
# Skip until begin marker
@@ -1220,13 +1270,14 @@ def write_metadata(self) -> None:
12201270
format_enums = [INSTR_FMT_PREFIX + format for format in sorted(all_formats)]
12211271

12221272
with open(self.metadata_filename, "w") as f:
1273+
# Create formatter
1274+
self.out = Formatter(f, 0)
1275+
12231276
# Write provenance header
1224-
f.write(f"// This file is generated by {THIS}\n")
1225-
f.write(self.from_source_files())
1226-
f.write(f"// Do not edit!\n")
1277+
self.out.write_raw(f"// This file is generated by {THIS}\n")
1278+
self.out.write_raw(self.from_source_files())
1279+
self.out.write_raw(f"// Do not edit!\n")
12271280

1228-
# Create formatter; the rest of the code uses this
1229-
self.out = Formatter(f, 0)
12301281

12311282
self.write_stack_effect_functions()
12321283

@@ -1302,13 +1353,13 @@ def write_metadata_for_macro(self, mac: MacroInstruction) -> None:
13021353
def write_instructions(self) -> None:
13031354
"""Write instructions to output file."""
13041355
with open(self.output_filename, "w") as f:
1305-
# Write provenance header
1306-
f.write(f"// This file is generated by {THIS}\n")
1307-
f.write(self.from_source_files())
1308-
f.write(f"// Do not edit!\n")
1356+
# Create formatter
1357+
self.out = Formatter(f, 8, self.emit_line_directives)
13091358

1310-
# Create formatter; the rest of the code uses this
1311-
self.out = Formatter(f, 8)
1359+
# Write provenance header
1360+
self.out.write_raw(f"// This file is generated by {THIS}\n")
1361+
self.out.write_raw(self.from_source_files())
1362+
self.out.write_raw(f"// Do not edit!\n")
13121363

13131364
# Write and count instructions of all kinds
13141365
n_instrs = 0
@@ -1478,13 +1529,16 @@ def wrap_super_or_macro(self, up: SuperOrMacroInstruction):
14781529
self.out.emit(f"DISPATCH();")
14791530

14801531

1481-
def extract_block_text(block: parser.Block) -> tuple[list[str], bool, list[str]]:
1532+
def extract_block_text(block: parser.Block) -> tuple[list[str], bool, list[str], int]:
14821533
# Get lines of text with proper dedent
14831534
blocklines = block.text.splitlines(True)
1535+
first_token: lx.Token = block.tokens[0] # IndexError means the context is broken
1536+
block_line = first_token.begin[0]
14841537

14851538
# Remove blank lines from both ends
14861539
while blocklines and not blocklines[0].strip():
14871540
blocklines.pop(0)
1541+
block_line += 1
14881542
while blocklines and not blocklines[-1].strip():
14891543
blocklines.pop()
14901544

@@ -1493,6 +1547,7 @@ def extract_block_text(block: parser.Block) -> tuple[list[str], bool, list[str]]
14931547
assert blocklines and blocklines[-1].strip() == "}"
14941548
blocklines.pop()
14951549
blocklines.pop(0)
1550+
block_line += 1
14961551

14971552
# Remove trailing blank lines
14981553
while blocklines and not blocklines[-1].strip():
@@ -1512,7 +1567,7 @@ def extract_block_text(block: parser.Block) -> tuple[list[str], bool, list[str]]
15121567
predictions.insert(0, m.group(1))
15131568
blocklines.pop()
15141569

1515-
return blocklines, check_eval_breaker, predictions
1570+
return blocklines, check_eval_breaker, predictions, block_line
15161571

15171572

15181573
def always_exits(lines: list[str]) -> bool:
@@ -1548,6 +1603,8 @@ def main():
15481603
if len(args.input) == 0:
15491604
args.input.append(DEFAULT_INPUT)
15501605
a = Analyzer(args.input, args.output, args.metadata) # Raises OSError if input unreadable
1606+
if args.emit_line_directives:
1607+
a.emit_line_directives = True
15511608
a.parse() # Raises SyntaxError on failure
15521609
a.analyze() # Prints messages and sets a.errors on failure
15531610
if a.errors:

0 commit comments

Comments
 (0)