Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions parso/python/tokenize.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,10 @@ def _get_token_collection(version_info):
fstring_string_multi_line = _compile(
r'(?:\{\{|\}\}|\\N\{' + unicode_character_name + r'\}|\\[^N]|[^{}\\])+'
)
# In raw f-strings, backslash is a literal character and does NOT escape
# the following character. So \{ means literal \ followed by expression {.
fstring_string_single_line_raw = _compile(r'(?:\{\{|\}\}|[^{}\r\n])+')
fstring_string_multi_line_raw = _compile(r'(?:\{\{|\}\}|[^{}])+')
fstring_format_spec_single_line = _compile(r'(?:\\(?:\r\n?|\n)|[^{}\r\n])+')
fstring_format_spec_multi_line = _compile(r'[^{}]+')

Expand Down Expand Up @@ -219,7 +223,7 @@ def _create_token_collection(version_info):

for t in fstring_prefixes:
for quote in all_quotes:
fstring_pattern_map[t + quote] = quote
fstring_pattern_map[t + quote] = (quote, 'r' in t.lower())

ALWAYS_BREAK_TOKENS = (';', 'import', 'class', 'def', 'try', 'except',
'finally', 'while', 'with', 'return', 'continue',
Expand Down Expand Up @@ -253,8 +257,9 @@ def __repr__(self):


class FStringNode:
def __init__(self, quote):
def __init__(self, quote, is_raw=False):
self.quote = quote
self.is_raw = is_raw
self.parentheses_count = 0
self.previous_lines = ''
self.last_string_start_pos: Any = None
Expand Down Expand Up @@ -307,6 +312,11 @@ def _find_fstring_string(endpats, fstring_stack, line, lnum, pos):
regex = fstring_format_spec_multi_line
else:
regex = fstring_format_spec_single_line
elif tos.is_raw:
if allow_multiline:
regex = fstring_string_multi_line_raw
else:
regex = fstring_string_single_line_raw
else:
if allow_multiline:
regex = fstring_string_multi_line
Expand Down Expand Up @@ -593,7 +603,8 @@ def dedent_if_necessary(start):
else: # ordinary string
yield PythonToken(STRING, token, spos, prefix)
elif token in fstring_pattern_map: # The start of an fstring.
fstring_stack.append(FStringNode(fstring_pattern_map[token]))
quote, is_raw = fstring_pattern_map[token]
fstring_stack.append(FStringNode(quote, is_raw=is_raw))
yield PythonToken(FSTRING_START, token, spos, prefix)
elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n', '\\\r'): # continued stmt
additional_prefix += prefix + line[start:]
Expand Down
10 changes: 10 additions & 0 deletions test/test_fstring.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,16 @@ def grammar():
'f"\\N{SOYOMBO LETTER -A}"',
'f"\\N{DOMINO TILE HORIZONTAL-00-00}"',
'f"""\\N{NO ENTRY}"""',

# raw f-strings: backslash is literal, doesn't escape braces (#207)
'rf"\\{x}"',
'rf"\\{x:0>3}"',
'rf"abc\\{x}def"',
'rf"""\\{x}"""',
'Rf"\\{x}"',
'rF"\\{x}"',
'fR"\\{x}"',
'Fr"\\{x}"',
]
)
def test_valid(code, grammar):
Expand Down
Loading