Javascript: Support dotted names as keywords

babel/messages/extract.py

@@ -462,7 +462,7 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
                          in the results
     :param options: a dictionary of additional options (optional)
     """
-    from babel.messages.jslexer import tokenize, unquote_string
+    from babel.messages.jslexer import tokenize, tokenize_dotted, unquote_string
     funcname = message_lineno = None
     messages = []
     last_argument = None
@@ -472,7 +472,13 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
     last_token = None
     call_stack = -1
 
-    for token in tokenize(fileobj.read().decode(encoding)):
+    tokenizer = (  # Don't bother using the more complex dotted logic if we don't need it
+        tokenize_dotted
+        if any('.' in kw for kw in keywords)
+        else tokenize
+    )
+
+    for token in tokenizer(fileobj.read().decode(encoding)):
         if token.type == 'operator' and token.value == '(':
             if funcname:
                 message_lineno = token.lineno
@@ -558,7 +564,7 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
         elif funcname and call_stack == -1:
             funcname = None
 
-        elif call_stack == -1 and token.type == 'name' and \
+        elif call_stack == -1 and token.type in ('name', 'dotted_name') and \
              token.value in keywords and \
              (last_token is None or last_token.type != 'name' or
               last_token.value != 'function'):

babel/messages/jslexer.py

@@ -164,3 +164,40 @@ def tokenize(source):
             yield token
         lineno += len(line_re.findall(token_value))
         pos = match.end()
+
+
+def tokenize_dotted(source):
+    """
+    Tokenize JavaScript source, regrouping dotted names into dotted_name tokens.
+
+    Returns a generator of tokens.
+
+    :return: Iterable[tuple]
+    """
+    DOT_OP = ("operator", ".")
+    dotted_buf = []
+
+    def release_buf(dotted_buf):
+        if not dotted_buf:
+            return
+        if len(dotted_buf) > 1:  # Have something to join?
+            dotted_name = "".join(buftok[1] for buftok in dotted_buf)
+            yield Token("dotted_name", dotted_name, dotted_buf[0][2])
+        else:  # Otherwise just release the single token as it was
+            yield dotted_buf[0]
+        dotted_buf[:] = []
+
+    for tok in tokenize(source):
+        if tok[0] == "name" and (not dotted_buf or dotted_buf[-1][:2] == DOT_OP):
+            dotted_buf.append(tok)
+            continue
+        if tok[:2] == DOT_OP and (dotted_buf and dotted_buf[-1][0] == "name"):
+            dotted_buf.append(tok)
+            continue
+        if dotted_buf:  # Release captured tokens when reaching a noncapturable token
+            for buftok in release_buf(dotted_buf):
+                yield buftok
+        yield tok
+
+    for buftok in release_buf(dotted_buf):  # And release when reaching the end
+        yield buftok

tests/messages/test_extract.py

@@ -478,6 +478,14 @@ def test_misplaced_comments(self):
         self.assertEqual(u'no comment here', messages[2][2])
         self.assertEqual([], messages[2][3])
 
+    def test_dotted_keyword_extract(self):
+        buf = BytesIO(b"msg1 = com.corporate.i18n.formatMessage('Insert coin to continue')")
+        messages = list(
+            extract.extract('javascript', buf, {"com.corporate.i18n.formatMessage": None}, [], {})
+        )
+
+        assert messages == [(1, 'Insert coin to continue', [], None)]
+
 
 class ExtractTestCase(unittest.TestCase):
 

tests/messages/test_jslexer.py

@@ -6,3 +6,18 @@
 def test_unquote():
     assert jslexer.unquote_string('""') == ''
     assert jslexer.unquote_string(r'"h\u00ebllo"') == u"hëllo"
+
+
+def test_dotted_name():
+    assert list(jslexer.tokenize_dotted("foo.bar(quux)")) == [
+        ('dotted_name', 'foo.bar', 1),
+        ('operator', '(', 1),
+        ('name', 'quux', 1),
+        ('operator', ')', 1)
+    ]
+
+
+def test_dotted_name_end():
+    assert list(jslexer.tokenize_dotted("foo.bar")) == [
+        ('dotted_name', 'foo.bar', 1),
+    ]