Da

✏️ Editing: test_utf8_mode.py

"""
Test the implementation of the PEP 540: the UTF-8 Mode.
"""

import locale
import subprocess
import sys
import textwrap
import unittest
from test import support
from test.support.script_helper import assert_python_ok, assert_python_failure
from test.support import os_helper

MS_WINDOWS = (sys.platform == 'win32')
POSIX_LOCALES = ('C', 'POSIX')
VXWORKS = (sys.platform == "vxworks")

class UTF8ModeTests(unittest.TestCase):
    DEFAULT_ENV = {
        'PYTHONUTF8': '',
        'PYTHONLEGACYWINDOWSFSENCODING': '',
        'PYTHONCOERCECLOCALE': '0',
    }

def posix_locale(self):
        loc = locale.setlocale(locale.LC_CTYPE, None)
        return (loc in POSIX_LOCALES)

def get_output(self, *args, failure=False, **kw):
        kw = dict(self.DEFAULT_ENV, **kw)
        if failure:
            out = assert_python_failure(*args, **kw)
            out = out[2]
        else:
            out = assert_python_ok(*args, **kw)
            out = out[1]
        return out.decode().rstrip("\n\r")

@unittest.skipIf(MS_WINDOWS, 'Windows has no POSIX locale')
    def test_posix_locale(self):
        code = 'import sys; print(sys.flags.utf8_mode)'

for loc in POSIX_LOCALES:
            with self.subTest(LC_ALL=loc):
                out = self.get_output('-c', code, LC_ALL=loc)
                self.assertEqual(out, '1')

def test_xoption(self):
        code = 'import sys; print(sys.flags.utf8_mode)'

out = self.get_output('-X', 'utf8', '-c', code)
        self.assertEqual(out, '1')

# undocumented but accepted syntax: -X utf8=1
        out = self.get_output('-X', 'utf8=1', '-c', code)
        self.assertEqual(out, '1')

out = self.get_output('-X', 'utf8=0', '-c', code)
        self.assertEqual(out, '0')

if MS_WINDOWS:
            # PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 Mode
            # and has the priority over -X utf8
            out = self.get_output('-X', 'utf8', '-c', code,
                                  PYTHONLEGACYWINDOWSFSENCODING='1')
            self.assertEqual(out, '0')

def test_env_var(self):
        code = 'import sys; print(sys.flags.utf8_mode)'

out = self.get_output('-c', code, PYTHONUTF8='1')
        self.assertEqual(out, '1')

out = self.get_output('-c', code, PYTHONUTF8='0')
        self.assertEqual(out, '0')

# -X utf8 has the priority over PYTHONUTF8
        out = self.get_output('-X', 'utf8=0', '-c', code, PYTHONUTF8='1')
        self.assertEqual(out, '0')

if MS_WINDOWS:
            # PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode
            # and has the priority over PYTHONUTF8
            out = self.get_output('-X', 'utf8', '-c', code, PYTHONUTF8='1',
                                  PYTHONLEGACYWINDOWSFSENCODING='1')
            self.assertEqual(out, '0')

# Cannot test with the POSIX locale, since the POSIX locale enables
        # the UTF-8 mode
        if not self.posix_locale():
            # PYTHONUTF8 should be ignored if -E is used
            out = self.get_output('-E', '-c', code, PYTHONUTF8='1')
            self.assertEqual(out, '0')

# invalid mode
        out = self.get_output('-c', code, PYTHONUTF8='xxx', failure=True)
        self.assertIn('invalid PYTHONUTF8 environment variable value',
                      out.rstrip())

def test_filesystemencoding(self):
        code = textwrap.dedent('''
            import sys
            print("{}/{}".format(sys.getfilesystemencoding(),
                                 sys.getfilesystemencodeerrors()))
        ''')

if MS_WINDOWS:
            expected = 'utf-8/surrogatepass'
        else:
            expected = 'utf-8/surrogateescape'

out = self.get_output('-X', 'utf8', '-c', code)
        self.assertEqual(out, expected)

if MS_WINDOWS:
            # PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode
            # and has the priority over -X utf8 and PYTHONUTF8
            out = self.get_output('-X', 'utf8', '-c', code,
                                  PYTHONUTF8='strict',
                                  PYTHONLEGACYWINDOWSFSENCODING='1')
            self.assertEqual(out, 'mbcs/replace')

def test_stdio(self):
        code = textwrap.dedent('''
            import sys
            print(f"stdin: {sys.stdin.encoding}/{sys.stdin.errors}")
            print(f"stdout: {sys.stdout.encoding}/{sys.stdout.errors}")
            print(f"stderr: {sys.stderr.encoding}/{sys.stderr.errors}")
        ''')

out = self.get_output('-X', 'utf8', '-c', code,
                              PYTHONIOENCODING='')
        self.assertEqual(out.splitlines(),
                         ['stdin: utf-8/surrogateescape',
                          'stdout: utf-8/surrogateescape',
                          'stderr: utf-8/backslashreplace'])

# PYTHONIOENCODING has the priority over PYTHONUTF8
        out = self.get_output('-X', 'utf8', '-c', code,
                              PYTHONIOENCODING="latin1")
        self.assertEqual(out.splitlines(),
                         ['stdin: iso8859-1/strict',
                          'stdout: iso8859-1/strict',
                          'stderr: iso8859-1/backslashreplace'])

out = self.get_output('-X', 'utf8', '-c', code,
                              PYTHONIOENCODING=":namereplace")
        self.assertEqual(out.splitlines(),
                         ['stdin: utf-8/namereplace',
                          'stdout: utf-8/namereplace',
                          'stderr: utf-8/backslashreplace'])

def test_io(self):
        code = textwrap.dedent('''
            import sys
            filename = sys.argv[1]
            with open(filename) as fp:
                print(f"{fp.encoding}/{fp.errors}")
        ''')
        filename = __file__

out = self.get_output('-c', code, filename, PYTHONUTF8='1')
        self.assertEqual(out, 'UTF-8/strict')

def _check_io_encoding(self, module, encoding=None, errors=None):
        filename = __file__

# Encoding explicitly set
        args = []
        if encoding:
            args.append(f'encoding={encoding!r}')
        if errors:
            args.append(f'errors={errors!r}')
        code = textwrap.dedent('''
            import sys
            from %s import open
            filename = sys.argv[1]
            with open(filename, %s) as fp:
                print(f"{fp.encoding}/{fp.errors}")
        ''') % (module, ', '.join(args))
        out = self.get_output('-c', code, filename,
                              PYTHONUTF8='1')

if not encoding:
            encoding = 'UTF-8'
        if not errors:
            errors = 'strict'
        self.assertEqual(out, f'{encoding}/{errors}')

def check_io_encoding(self, module):
        self._check_io_encoding(module, encoding="latin1")
        self._check_io_encoding(module, errors="namereplace")
        self._check_io_encoding(module,
                                encoding="latin1", errors="namereplace")

def test_io_encoding(self):
        self.check_io_encoding('io')

def test_pyio_encoding(self):
        self.check_io_encoding('_pyio')

def test_locale_getpreferredencoding(self):
        code = 'import locale; print(locale.getpreferredencoding(False), locale.getpreferredencoding(True))'
        out = self.get_output('-X', 'utf8', '-c', code)
        self.assertEqual(out, 'UTF-8 UTF-8')

for loc in POSIX_LOCALES:
            with self.subTest(LC_ALL=loc):
                out = self.get_output('-X', 'utf8', '-c', code, LC_ALL=loc)
                self.assertEqual(out, 'UTF-8 UTF-8')

@unittest.skipIf(MS_WINDOWS, 'test specific to Unix')
    def test_cmd_line(self):
        arg = 'h\xe9\u20ac'.encode('utf-8')
        arg_utf8 = arg.decode('utf-8')
        arg_ascii = arg.decode('ascii', 'surrogateescape')
        code = 'import locale, sys; print("%s:%s" % (locale.getpreferredencoding(), ascii(sys.argv[1:])))'

def check(utf8_opt, expected, **kw):
            out = self.get_output('-X', utf8_opt, '-c', code, arg, **kw)
            args = out.partition(':')[2].rstrip()
            self.assertEqual(args, ascii(expected), out)

check('utf8', [arg_utf8])
        for loc in POSIX_LOCALES:
            with self.subTest(LC_ALL=loc):
                check('utf8', [arg_utf8], LC_ALL=loc)

if sys.platform == 'darwin' or support.is_android or VXWORKS:
            c_arg = arg_utf8
        elif sys.platform.startswith("aix"):
            c_arg = arg.decode('iso-8859-1')
        else:
            c_arg = arg_ascii
        for loc in POSIX_LOCALES:
            with self.subTest(LC_ALL=loc):
                check('utf8=0', [c_arg], LC_ALL=loc)

def test_optim_level(self):
        # CPython: check that Py_Main() doesn't increment Py_OptimizeFlag
        # twice when -X utf8 requires to parse the configuration twice (when
        # the encoding changes after reading the configuration, the
        # configuration is read again with the new encoding).
        code = 'import sys; print(sys.flags.optimize)'
        out = self.get_output('-X', 'utf8', '-O', '-c', code)
        self.assertEqual(out, '1')
        out = self.get_output('-X', 'utf8', '-OO', '-c', code)
        self.assertEqual(out, '2')

code = 'import sys; print(sys.flags.ignore_environment)'
        out = self.get_output('-X', 'utf8', '-E', '-c', code)
        self.assertEqual(out, '1')

@unittest.skipIf(MS_WINDOWS,
                     "os.device_encoding() doesn't implement "
                     "the UTF-8 Mode on Windows")
    def test_device_encoding(self):
        # Use stdout as TTY
        if not sys.stdout.isatty():
            self.skipTest("sys.stdout is not a TTY")

filename = 'out.txt'
        self.addCleanup(os_helper.unlink, filename)

code = (f'import os, sys; fd = sys.stdout.fileno(); '
                f'out = open({filename!r}, "w", encoding="utf-8"); '
                f'print(os.isatty(fd), os.device_encoding(fd), file=out); '
                f'out.close()')
        cmd = [sys.executable, '-X', 'utf8', '-c', code]
        # The stdout TTY is inherited to the child process
        proc = subprocess.run(cmd, text=True)
        self.assertEqual(proc.returncode, 0, proc)

# In UTF-8 Mode, device_encoding(fd) returns "UTF-8" if fd is a TTY
        with open(filename, encoding="utf8") as fp:
            out = fp.read().rstrip()
        self.assertEqual(out, 'True UTF-8')

if __name__ == "__main__":
    unittest.main()

📁 Sid Gifari File Manager