BUG: unicode characters when reading JSON lines

Fixes UnicodeDecodeError bug when reading JSON lines input with Ascii decoder, which is often the default setting in Python 2.7. Avoids issues with mixing unicode and ascii strings.

xref #15132

Author: rouzazari

doc/source/whatsnew/v0.20.0.txt

@@ -384,3 +384,4 @@ Bug Fixes
 - Bug in ``pd.read_csv()`` for the C engine where ``usecols`` were being indexed incorrectly with ``parse_dates`` (:issue:`14792`)
 
 - Bug in ``Series.dt.round`` inconsistent behaviour on NAT's with different arguments (:issue:`14940`)
+- Bug in ``.read_json()`` for Python 2 where ``lines=True`` and contents contain non-ascii unicode characters (:issue:`15132`)

pandas/io/json.py

@@ -274,7 +274,7 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
         # If given a json lines file, we break the string into lines, add
         # commas and put it in a json list to make a valid json object.
         lines = list(StringIO(json.strip()))
-        json = u'[' + u','.join(lines) + u']'
+        json = '[' + ','.join(lines) + ']'
 
     obj = None
     if typ == 'frame':

pandas/io/tests/json/test_pandas.py

@@ -959,6 +959,25 @@ def test_read_jsonl(self):
         expected = DataFrame([[1, 2], [1, 2]], columns=['a', 'b'])
         assert_frame_equal(result, expected)
 
+        # GH15132: ascii input of non-ascii unicode characters for Python 2
+        # \xe2\x80\x9d == \u201d == RIGHT DOUBLE QUOTATION MARK
+        if compat.PY2:
+            # simulate file handle
+            json = StringIO('{"a": "foo\xe2\x80\x9d", "b": "bar"}\n'
+                            '{"a": "foo", "b": "bar"}\n')
+            result = read_json(json, lines=True)
+            expected = DataFrame([[u"foo\u201d", "bar"], ["foo", "bar"]],
+                                 columns=['a', 'b'])
+            assert_frame_equal(result, expected)
+
+            # simulate string
+            json = ('{"a": "foo\xe2\x80\x9d", "b": "bar"}\n'
+                    '{"a": "foo", "b": "bar"}\n')
+            result = read_json(json, lines=True)
+            expected = DataFrame([[u"foo\u201d", "bar"], ["foo", "bar"]],
+                                 columns=['a', 'b'])
+            assert_frame_equal(result, expected)
+
     def test_to_jsonl(self):
         # GH9180
         df = DataFrame([[1, 2], [1, 2]], columns=['a', 'b'])