gh-104400: Add more tests to pygettext

Lib/test/test_tools/test_i18n/__init__.py

@@ -0,0 +1,6 @@
+import os.path
+from test import support
+
+
+def load_tests(*args):
+    return support.load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_tools/test_i18n/data/docstrings.pot

@@ -0,0 +1,40 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2023-08-20 16:42+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <[email protected]>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: docstrings.py:7
+#, docstring
+msgid ""
+msgstr ""
+
+#: docstrings.py:18
+#, docstring
+msgid ""
+"multiline\n"
+"    docstring\n"
+"    "
+msgstr ""
+
+#: docstrings.py:25
+#, docstring
+msgid "docstring1"
+msgstr ""
+
+#: docstrings.py:30
+#, docstring
+msgid "Hello, {}!"
+msgstr ""
+

Lib/test/test_tools/test_i18n/data/docstrings.py

@@ -0,0 +1,41 @@
+# Test docstring extraction
+from gettext import gettext as _
+
+
+# Empty docstring
+def test(x):
+    """"""
+
+
+# Leading empty line
+def test2(x):
+
+    """docstring"""  # XXX This should be extracted but isn't.
+
+
+# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`.
+def test3(x):
+    """multiline
+    docstring
+    """
+
+
+# Multiple docstrings - only the first should be extracted
+def test4(x):
+    """docstring1"""
+    """docstring2"""
+
+
+def test5(x):
+    """Hello, {}!""".format("world!")  # XXX This should not be extracted.
+
+
+# Nested docstrings
+def test6(x):
+    def inner(y):
+        """nested docstring"""  # XXX This should be extracted but isn't.
+
+
+class Outer:
+    class Inner:
+        "nested class docstring"  # XXX This should be extracted but isn't.

Lib/test/test_tools/test_i18n/data/fileloc.pot

@@ -0,0 +1,35 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2023-08-20 16:15+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <[email protected]>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: fileloc.py:5 fileloc.py:6
+msgid "foo"
+msgstr ""
+
+#: fileloc.py:9
+msgid "bar"
+msgstr ""
+
+#: fileloc.py:14 fileloc.py:18
+#, docstring
+msgid "docstring"
+msgstr ""
+
+#: fileloc.py:22 fileloc.py:26
+#, docstring
+msgid "baz"
+msgstr ""
+

Lib/test/test_tools/test_i18n/data/fileloc.py

@@ -0,0 +1,26 @@
+# Test file locations
+from gettext import gettext as _
+
+# Duplicate strings
+_('foo')
+_('foo')
+
+# Duplicate strings on the same line should only add one location to the output
+_('bar'), _('bar')
+
+
+# Duplicate docstrings
+class A:
+    """docstring"""
+
+
+def f():
+    """docstring"""
+
+
+# Duplicate message and docstring
+_('baz')
+
+
+def g():
+    """baz"""

Lib/test/test_tools/test_i18n/data/messages.pot

@@ -0,0 +1,67 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2023-08-20 16:42+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <[email protected]>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: messages.py:5
+msgid ""
+msgstr ""
+
+#: messages.py:8 messages.py:9
+msgid "parentheses"
+msgstr ""
+
+#: messages.py:12
+msgid "Hello, world!"
+msgstr ""
+
+#: messages.py:15
+msgid ""
+"Hello,\n"
+"    multiline!\n"
+msgstr ""
+
+#: messages.py:29
+msgid "Hello, {}!"
+msgstr ""
+
+#: messages.py:33
+msgid "1"
+msgstr ""
+
+#: messages.py:33
+msgid "2"
+msgstr ""
+
+#: messages.py:34 messages.py:35
+msgid "A"
+msgstr ""
+
+#: messages.py:34 messages.py:35
+msgid "B"
+msgstr ""
+
+#: messages.py:36
+msgid "set"
+msgstr ""
+
+#: messages.py:42
+msgid "nested string"
+msgstr ""
+
+#: messages.py:47
+msgid "baz"
+msgstr ""
+

Lib/test/test_tools/test_i18n/data/messages.py

@@ -0,0 +1,64 @@
+# Test message extraction
+from gettext import gettext as _
+
+# Empty string
+_("")
+
+# Extra parentheses
+(_("parentheses"))
+((_("parentheses")))
+
+# Multiline strings
+_("Hello, "
+  "world!")
+
+_("""Hello,
+    multiline!
+""")
+
+# Invalid arguments
+_()
+_(None)
+_(1)
+_(False)
+_(x="kwargs are not allowed")
+_("foo", "bar")
+_("something", x="something else")
+
+# .format()
+_("Hello, {}!").format("world")  # valid
+_("Hello, {}!".format("world"))  # invalid
+
+# Nested structures
+_("1"), _("2")
+arr = [_("A"), _("B")]
+obj = {'a': _("A"), 'b': _("B")}
+{{{_('set')}}}
+
+
+# Nested functions and classes
+def test():
+    _("nested string")  # XXX This should be extracted but isn't.
+    [_("nested string")]
+
+
+class Foo:
+    def bar(self):
+        return _("baz")
+
+
+def bar(x=_('default value')):  # XXX This should be extracted but isn't.
+    pass
+
+
+def baz(x=[_('default value')]):  # XXX This should be extracted but isn't.
+    pass
+
+
+# Shadowing _()
+def _(x):
+    pass
+
+
+def _(x="don't extract me"):
+    pass

Lib/test/test_tools/test_i18n.py → Lib/test/test_tools/test_i18n/test_i18n.py

@@ -1,6 +1,8 @@
 """Tests to cover the Tools/i18n package"""
 
 import os
+from pathlib import Path
+import re
 import sys
 import unittest
 from textwrap import dedent
@@ -12,20 +14,22 @@
 
 skip_if_missing()
 
+DATA_DIR = Path(__file__).resolve().parent / 'data'
+
 
 class Test_pygettext(unittest.TestCase):
     """Tests for the pygettext.py tool"""
 
-    script = os.path.join(toolsdir,'i18n', 'pygettext.py')
+    script = os.path.join(toolsdir, 'i18n', 'pygettext.py')
 
     def get_header(self, data):
         """ utility: return the header of a .po file as a dictionary """
         headers = {}
         for line in data.split('\n'):
-            if not line or line.startswith(('#', 'msgid','msgstr')):
+            if not line or line.startswith(('#', 'msgid', 'msgstr')):
                 continue
             line = line.strip('"')
-            key, val = line.split(':',1)
+            key, val = line.split(':', 1)
             headers[key] = val.strip()
         return headers
 
@@ -53,6 +57,31 @@ def get_msgids(self, data):
 
         return msgids
 
+    def assert_POT_equal(self, expected, actual):
+        """Check if two POT files are equal"""
+        # Normalize the creation date
+        date_pattern = re.compile(r'"POT-Creation-Date: .+?\n"')
+        header = '"POT-Creation-Date: 2000-01-01 00:00+0000\\n"'
+        expected = re.sub(date_pattern, header, expected)
+        actual = re.sub(date_pattern, header, actual)
+
+        # Normalize charset to UTF-8 (currently there's no way to specify the output charset)
+        charset_pattern = re.compile(r'"Content-Type: text/plain; charset=.+?\n"')
+        charset = "Content-Type: text/plain; charset=UTF-8\\n"
+        expected = re.sub(charset_pattern, charset, expected)
+        actual = re.sub(charset_pattern, charset, actual)
+
+        # Normalize the file location path separators in case this test is
+        # running on Windows (which uses '\')
+        fileloc_pattern = re.compile(r'#:.+')
+
+        def replace(match):
+            return match[0].replace(os.sep, "/")
+        expected = re.sub(fileloc_pattern, replace, expected)
+        actual = re.sub(fileloc_pattern, replace, actual)
+
+        self.assertEqual(expected, actual)
+
     def extract_docstrings_from_str(self, module_content):
         """ utility: return all msgids extracted from module_content """
         filename = 'test_docstrings.py'
@@ -310,6 +339,23 @@ def test_calls_in_fstring_with_partially_wrong_expression(self):
         self.assertNotIn('foo', msgids)
         self.assertIn('bar', msgids)
 
+    def test_pygettext_output(self):
+        """Test that the pygettext output exactly matches a file."""
+        filenames = (('messages.py', 'messages.pot'),
+                     ('docstrings.py', 'docstrings.pot'),
+                     ('fileloc.py', 'fileloc.pot'))
+
+        for input_file, output_file in filenames:
+            with self.subTest(input_file=f'data/{input_file}'):
+                contents = (DATA_DIR / input_file).read_text(encoding='utf-8')
+                with temp_cwd(None):
+                    Path(input_file).write_text(contents)
+                    assert_python_ok(self.script, '--docstrings', input_file)
+                    output = Path('messages.pot').read_text()
+
+                expected = (DATA_DIR / output_file).read_text(encoding='utf-8')
+                self.assert_POT_equal(expected, output)
+
     def test_files_list(self):
         """Make sure the directories are inspected for source files
            bpo-31920