[3.8] bpo-41138: Fix trace CLI for non-UTF-8 files. (GH-21177) (GH-21200)

Fix also a resource warning when store counts and module info.
(cherry picked from commit 04cdeb7)

Author: serhiy-storchaka

Lib/test/test_trace.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from test.support import TESTFN, rmtree, unlink, captured_stdout
+from test.support import TESTFN, TESTFN_UNICODE, FS_NONASCII, rmtree, unlink, captured_stdout
 from test.support.script_helper import assert_python_ok, assert_python_failure
 import textwrap
 import unittest
@@ -429,9 +429,10 @@ class TestCoverageCommandLineOutput(unittest.TestCase):
     coverfile = 'tmp.cover'
 
     def setUp(self):
-        with open(self.codefile, 'w') as f:
+        with open(self.codefile, 'w', encoding='iso-8859-15') as f:
             f.write(textwrap.dedent('''\
-                x = 42
+                # coding: iso-8859-15
+                x = 'spœm'
                 if []:
                     print('unreachable')
             '''))
@@ -452,9 +453,10 @@ def test_cover_files_written_no_highlight(self):
         self.assertEqual(stderr, b'')
         self.assertFalse(os.path.exists(tracecoverpath))
         self.assertTrue(os.path.exists(self.coverfile))
-        with open(self.coverfile) as f:
+        with open(self.coverfile, encoding='iso-8859-15') as f:
             self.assertEqual(f.read(),
-                "    1: x = 42\n"
+                "       # coding: iso-8859-15\n"
+                "    1: x = 'spœm'\n"
                 "    1: if []:\n"
                 "           print('unreachable')\n"
             )
@@ -463,9 +465,10 @@ def test_cover_files_written_with_highlight(self):
         argv = '-m trace --count --missing'.split() + [self.codefile]
         status, stdout, stderr = assert_python_ok(*argv)
         self.assertTrue(os.path.exists(self.coverfile))
-        with open(self.coverfile) as f:
+        with open(self.coverfile, encoding='iso-8859-15') as f:
             self.assertEqual(f.read(), textwrap.dedent('''\
-                    1: x = 42
+                       # coding: iso-8859-15
+                    1: x = 'spœm'
                     1: if []:
                 >>>>>>     print('unreachable')
             '''))
@@ -486,14 +489,19 @@ def test_failures(self):
             self.assertIn(message, stderr)
 
     def test_listfuncs_flag_success(self):
-        with open(TESTFN, 'w') as fd:
-            self.addCleanup(unlink, TESTFN)
+        filename = TESTFN + '.py'
+        modulename = os.path.basename(TESTFN)
+        with open(filename, 'w', encoding='utf-8') as fd:
+            self.addCleanup(unlink, filename)
             fd.write("a = 1\n")
-            status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN)
+            status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', filename,
+                                                      PYTHONIOENCODING='utf-8')
             self.assertIn(b'functions called:', stdout)
+            expected = f'filename: {filename}, modulename: {modulename}, funcname: <module>'
+            self.assertIn(expected.encode(), stdout)
 
     def test_sys_argv_list(self):
-        with open(TESTFN, 'w') as fd:
+        with open(TESTFN, 'w', encoding='utf-8') as fd:
             self.addCleanup(unlink, TESTFN)
             fd.write("import sys\n")
             fd.write("print(type(sys.argv))\n")
@@ -505,7 +513,8 @@ def test_sys_argv_list(self):
     def test_count_and_summary(self):
         filename = f'{TESTFN}.py'
         coverfilename = f'{TESTFN}.cover'
-        with open(filename, 'w') as fd:
+        modulename = os.path.basename(TESTFN)
+        with open(filename, 'w', encoding='utf-8') as fd:
             self.addCleanup(unlink, filename)
             self.addCleanup(unlink, coverfilename)
             fd.write(textwrap.dedent("""\
@@ -522,7 +531,7 @@ def f():
         stdout = stdout.decode()
         self.assertEqual(status, 0)
         self.assertIn('lines   cov%   module   (path)', stdout)
-        self.assertIn(f'6   100%   {TESTFN}   ({filename})', stdout)
+        self.assertIn(f'6   100%   {modulename}   ({filename})', stdout)
 
     def test_run_as_module(self):
         assert_python_ok('-m', 'trace', '-l', '--module', 'timeit', '-n', '1')

Lib/trace.py

@@ -287,8 +287,9 @@ def write_results(self, show_missing=True, summary=False, coverdir=None):
         if self.outfile:
             # try and store counts and module info into self.outfile
             try:
-                pickle.dump((self.counts, self.calledfuncs, self.callers),
-                            open(self.outfile, 'wb'), 1)
+                with open(self.outfile, 'wb') as f:
+                    pickle.dump((self.counts, self.calledfuncs, self.callers),
+                                f, 1)
             except OSError as err:
                 print("Can't save counts files because %s" % err, file=sys.stderr)
 
@@ -731,7 +732,7 @@ def parse_ignore_dir(s):
             sys.argv = [opts.progname, *opts.arguments]
             sys.path[0] = os.path.dirname(opts.progname)
 
-            with open(opts.progname) as fp:
+            with open(opts.progname, 'rb') as fp:
                 code = compile(fp.read(), opts.progname, 'exec')
             # try to emulate __main__ namespace as much as possible
             globs = {

Misc/NEWS.d/next/Library/2020-06-27-13-51-36.bpo-41138.bIpf7g.rst

@@ -0,0 +1,2 @@
+Fixed the :mod:`trace` module CLI for Python source files with non-UTF-8
+encoding.