From 19cc96b37797dd331c7b3a36f5fd412dd27904e1 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Tue, 30 Apr 2024 19:28:02 +0530 Subject: [PATCH 1/8] tokenizer: skip lines that are just whitespace --- src/blib2to3/pgen2/tokenize.py | 6 ++++++ tests/data/cases/backslash_before_indent.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/data/cases/backslash_before_indent.py diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index b86b91ed37f..901c0465309 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -608,6 +608,12 @@ def generate_tokens( except StopIteration: line = "" lnum += 1 + + # skip lines that are just a slash, to avoid storing that line's + # indent information. + if line.rstrip("\n").strip(" \t") == "\\": + continue + pos, max = 0, len(line) if contstr: # continued string diff --git a/tests/data/cases/backslash_before_indent.py b/tests/data/cases/backslash_before_indent.py new file mode 100644 index 00000000000..e58818fc861 --- /dev/null +++ b/tests/data/cases/backslash_before_indent.py @@ -0,0 +1,18 @@ +class Plotter: +\ + pass + +class AnotherCase: + \ + pass + +# output + +class Plotter: + + pass + + +class AnotherCase: + + pass From 72a84b59bfea26c6d76154ad5edb1ba888d946aa Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Tue, 30 Apr 2024 19:34:20 +0530 Subject: [PATCH 2/8] make test 3.10+ --- tests/data/cases/backslash_before_indent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/data/cases/backslash_before_indent.py b/tests/data/cases/backslash_before_indent.py index e58818fc861..90cbd0657ba 100644 --- a/tests/data/cases/backslash_before_indent.py +++ b/tests/data/cases/backslash_before_indent.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 class Plotter: \ pass From 44ae6153865fe7753b2fcbf0bc192350f8b746b7 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Tue, 30 Apr 2024 19:47:44 +0530 Subject: [PATCH 3/8] fix failing test --- tests/data/cases/comment_after_escaped_newline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/data/cases/comment_after_escaped_newline.py b/tests/data/cases/comment_after_escaped_newline.py index 133f4898a47..133c230a3a8 100644 --- a/tests/data/cases/comment_after_escaped_newline.py +++ b/tests/data/cases/comment_after_escaped_newline.py @@ -14,5 +14,7 @@ def bob(): # pylint: disable=W9016 pass -def bobtwo(): # some comment here +def bobtwo(): + + # some comment here pass From 9daa7e496fb4f56f4fdb29aa4b96b45f80dd5ef0 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Tue, 30 Apr 2024 19:54:08 +0530 Subject: [PATCH 4/8] add changelog entry --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index e3df420b5a0..43eed8a9110 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,6 +29,8 @@ - Fix regression where Black failed to parse a multiline f-string containing another multiline string (#4339) +- Fix bug with Black incorrectly parsing empty lines with a backslash (#4343) + ### Performance From 4f91e3b1013fae3ea6c8c0a4fca0c8c284d432d6 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Tue, 30 Apr 2024 20:12:26 +0530 Subject: [PATCH 5/8] don't ignore slashes inside strings --- src/blib2to3/pgen2/tokenize.py | 2 +- tests/data/cases/backslash_before_indent.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 901c0465309..f2c3c6d6768 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -611,7 +611,7 @@ def generate_tokens( # skip lines that are just a slash, to avoid storing that line's # indent information. - if line.rstrip("\n").strip(" \t") == "\\": + if not contstr and line.rstrip("\n").strip(" \t") == "\\": continue pos, max = 0, len(line) diff --git a/tests/data/cases/backslash_before_indent.py b/tests/data/cases/backslash_before_indent.py index 90cbd0657ba..2d126a945e4 100644 --- a/tests/data/cases/backslash_before_indent.py +++ b/tests/data/cases/backslash_before_indent.py @@ -5,7 +5,10 @@ class Plotter: class AnotherCase: \ - pass + """Some + \ + Docstring + """ # output @@ -15,5 +18,7 @@ class Plotter: class AnotherCase: - - pass + """Some + \ + Docstring + """ From 339fd0dad7f1724112506aafad9d68fa41b2b83d Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Thu, 2 May 2024 13:01:22 +0530 Subject: [PATCH 6/8] Add name to authors --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index e0511bb9b7c..fa1ce8f7112 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -181,6 +181,7 @@ Multiple contributions by: - [Tony Narlock](mailto:tony@git-pull.com) - [Tsuyoshi Hombashi](mailto:tsuyoshi.hombashi@gmail.com) - [Tushar Chandra](mailto:tusharchandra2018@u.northwestern.edu) +- [Tushar Sadhwani](mailto:tushar.sadhwani000@gmail.com) - [Tzu-ping Chung](mailto:uranusjr@gmail.com) - [Utsav Shah](mailto:ukshah2@illinois.edu) - utsav-dbx From 216f836f951668d106ebadff3fa393666a2faf4e Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sat, 4 May 2024 21:31:22 +0530 Subject: [PATCH 7/8] add \f to indent chars --- src/blib2to3/pgen2/tokenize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index f2c3c6d6768..28972a9bd78 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -609,9 +609,9 @@ def generate_tokens( line = "" lnum += 1 - # skip lines that are just a slash, to avoid storing that line's - # indent information. - if not contstr and line.rstrip("\n").strip(" \t") == "\\": + # skip lines that are just indent characters ending with a slash + # to avoid storing that line's indent information. + if not contstr and line.rstrip("\n").strip(" \t\f") == "\\": continue pos, max = 0, len(line) From 2313537a8786712e589de9812c09c444f8b63245 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sat, 4 May 2024 21:39:46 +0530 Subject: [PATCH 8/8] fix failing test --- tests/data/cases/form_feeds.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/data/cases/form_feeds.py b/tests/data/cases/form_feeds.py index 48ffc98106b..957b4a1db95 100644 --- a/tests/data/cases/form_feeds.py +++ b/tests/data/cases/form_feeds.py @@ -156,6 +156,7 @@ def something(self): # + # pass