Tokenize escaped quotes in raw f-strings correctly

native/libcst/src/tokenizer/core/mod.rs

@@ -953,6 +953,13 @@ impl<'t> TokState<'t> {
                         }
                     }
                 }
+                (Some('\\'), _) if is_raw_string => {
+                    self.text_pos.next();
+                    if let Some('"' | '\'') = self.text_pos.peek() {
+                        // these aren't end of string markers, skip them
+                        self.text_pos.next();
+                    }
+                }
                 (Some('{'), _) => {
                     if is_in_format_spec {
                         // don't actually consume the {, and generate an OP for it instead

native/libcst/src/tokenizer/tests.rs

@@ -519,6 +519,36 @@ fn test_string_prefix() {
             (TokType::String, "''"),
         ]),
     );
+
+    // raw string escapes
+    assert_eq!(
+        tokenize_all("r'\\''", &default_config()),
+        Ok(vec![(TokType::String, "r'\\''")]),
+    );
+    assert_eq!(
+        tokenize_all(r#"r"\"""#, &default_config()),
+        Ok(vec![(TokType::String, r#"r"\"""#)]),
+    );
+    let config = TokConfig {
+        split_fstring: true,
+        ..default_config()
+    };
+    assert_eq!(
+        tokenize_all("rf'\\''", &config),
+        Ok(vec![
+            (TokType::FStringStart, "rf'"),
+            (TokType::FStringString, "\\'"),
+            (TokType::FStringEnd, "'"),
+        ]),
+    );
+    assert_eq!(
+        tokenize_all(r#"rf"\"""#, &config),
+        Ok(vec![
+            (TokType::FStringStart, "rf\""),
+            (TokType::FStringString, r#"\""#),
+            (TokType::FStringEnd, "\""),
+        ]),
+    );
 }
 
 #[test]