Sync Fork from Upstream Repo

doc/source/whatsnew/v1.0.2.rst

@@ -25,8 +25,9 @@ Fixed regressions
 Bug fixes
 ~~~~~~~~~
 
--
--
+**I/O**
+
+- Using ``pd.NA`` with :meth:`DataFrame.to_json` now correctly outputs a null value instead of an empty object (:issue:`31615`)
 
 .. ---------------------------------------------------------------------------
 

pandas/_libs/src/ujson/python/objToJSON.c

@@ -53,6 +53,7 @@ static PyTypeObject *cls_dataframe;
 static PyTypeObject *cls_series;
 static PyTypeObject *cls_index;
 static PyTypeObject *cls_nat;
+static PyTypeObject *cls_na;
 PyObject *cls_timedelta;
 
 npy_int64 get_nat(void) { return NPY_MIN_INT64; }
@@ -149,6 +150,7 @@ int PdBlock_iterNext(JSOBJ, JSONTypeContext *);
 void *initObjToJSON(void) {
     PyObject *mod_pandas;
     PyObject *mod_nattype;
+    PyObject *mod_natype;
     PyObject *mod_decimal = PyImport_ImportModule("decimal");
     type_decimal =
         (PyTypeObject *)PyObject_GetAttrString(mod_decimal, "Decimal");
@@ -174,6 +176,12 @@ void *initObjToJSON(void) {
         Py_DECREF(mod_nattype);
     }
 
+    mod_natype = PyImport_ImportModule("pandas._libs.missing");
+    if (mod_natype) {
+        cls_na = (PyTypeObject *)PyObject_GetAttrString(mod_natype, "NAType");
+        Py_DECREF(mod_natype);
+    }
+
     /* Initialise numpy API */
     import_array();
     // GH 31463
@@ -1789,6 +1797,10 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
                      "%R (0d array) is not JSON serializable at the moment",
                      obj);
         goto INVALID;
+    } else if (PyObject_TypeCheck(obj, cls_na)) {
+        PRINTMARK();
+        tc->type = JT_NULL;
+        return;
     }
 
 ISITERABLE:

pandas/core/indexes/base.py

@@ -3158,9 +3158,9 @@ def _convert_slice_indexer(self, key: slice, kind=None):
 
         # validate iloc
         if kind == "iloc":
-            self._validate_indexer("slice", key.start, "iloc")
-            self._validate_indexer("slice", key.stop, "iloc")
-            self._validate_indexer("slice", key.step, "iloc")
+            self._validate_indexer("positional", key.start, "iloc")
+            self._validate_indexer("positional", key.stop, "iloc")
+            self._validate_indexer("positional", key.step, "iloc")
             return key
 
         # potentially cast the bounds to integers
@@ -3285,8 +3285,8 @@ def _invalid_indexer(self, form: str_t, key):
         Consistent invalid indexer message.
         """
         raise TypeError(
-            f"cannot do {form} indexing on {type(self)} with these "
-            f"indexers [{key}] of {type(key)}"
+            f"cannot do {form} indexing on {type(self).__name__} with these "
+            f"indexers [{key}] of type {type(key).__name__}"
         )
 
     # --------------------------------------------------------------------

pandas/core/indexes/datetimelike.py

@@ -406,9 +406,9 @@ def _convert_scalar_indexer(self, key, kind: str):
         is_int = is_integer(key)
         is_flt = is_float(key)
         if kind == "loc" and (is_int or is_flt):
-            self._invalid_indexer("index", key)
+            self._invalid_indexer("label", key)
         elif kind == "getitem" and is_flt:
-            self._invalid_indexer("index", key)
+            self._invalid_indexer("label", key)
 
         return super()._convert_scalar_indexer(key, kind=kind)
 

pandas/tests/frame/indexing/test_indexing.py

@@ -1050,9 +1050,8 @@ def test_getitem_setitem_float_labels(self):
 
         # positional slicing only via iloc!
         msg = (
-            "cannot do slice indexing on "
-            r"<class 'pandas\.core\.indexes\.numeric\.Float64Index'> with "
-            r"these indexers \[1.0\] of <class 'float'>"
+            "cannot do positional indexing on Float64Index with "
+            r"these indexers \[1.0\] of type float"
         )
         with pytest.raises(TypeError, match=msg):
             df.iloc[1.0:5]

pandas/tests/frame/test_constructors.py

@@ -1860,9 +1860,8 @@ def check(df):
             # No NaN found -> error
             if len(indexer) == 0:
                 msg = (
-                    "cannot do label indexing on "
-                    r"<class 'pandas\.core\.indexes\.range\.RangeIndex'> "
-                    r"with these indexers \[nan\] of <class 'float'>"
+                    "cannot do label indexing on RangeIndex "
+                    r"with these indexers \[nan\] of type float"
                 )
                 with pytest.raises(TypeError, match=msg):
                     df.loc[:, np.nan]

pandas/tests/indexing/test_categorical.py

@@ -83,8 +83,8 @@ def test_loc_scalar(self):
             df.loc["d", "C"] = 10
 
         msg = (
-            r"cannot do label indexing on <class 'pandas\.core\.indexes\.category"
-            r"\.CategoricalIndex'> with these indexers \[1\] of <class 'int'>"
+            "cannot do label indexing on CategoricalIndex with these "
+            r"indexers \[1\] of type int"
         )
         with pytest.raises(TypeError, match=msg):
             df.loc[1]

pandas/tests/indexing/test_floats.py

@@ -54,7 +54,7 @@ def test_scalar_error(self, index_func):
 
         msg = (
             "cannot do positional indexing on {klass} with these "
-            r"indexers \[3\.0\] of {kind}".format(klass=type(i), kind=str(float))
+            r"indexers \[3\.0\] of type float".format(klass=type(i).__name__)
         )
         with pytest.raises(TypeError, match=msg):
             s.iloc[3.0] = 0
@@ -92,11 +92,11 @@ def test_scalar_non_numeric(self):
                     else:
                         error = TypeError
                         msg = (
-                            r"cannot do (label|index|positional) indexing "
+                            r"cannot do (label|positional) indexing "
                             r"on {klass} with these indexers \[3\.0\] of "
-                            r"{kind}|"
+                            r"type float|"
                             "Cannot index by location index with a "
-                            "non-integer key".format(klass=type(i), kind=str(float))
+                            "non-integer key".format(klass=type(i).__name__)
                         )
                     with pytest.raises(error, match=msg):
                         idxr(s)[3.0]
@@ -113,9 +113,9 @@ def test_scalar_non_numeric(self):
                 else:
                     error = TypeError
                     msg = (
-                        r"cannot do (label|index) indexing "
+                        r"cannot do label indexing "
                         r"on {klass} with these indexers \[3\.0\] of "
-                        r"{kind}".format(klass=type(i), kind=str(float))
+                        r"type float".format(klass=type(i).__name__)
                     )
                 with pytest.raises(error, match=msg):
                     s.loc[3.0]
@@ -125,9 +125,9 @@ def test_scalar_non_numeric(self):
 
                 # setting with a float fails with iloc
                 msg = (
-                    r"cannot do (label|index|positional) indexing "
+                    r"cannot do (label|positional) indexing "
                     r"on {klass} with these indexers \[3\.0\] of "
-                    r"{kind}".format(klass=type(i), kind=str(float))
+                    r"type float".format(klass=type(i).__name__)
                 )
                 with pytest.raises(TypeError, match=msg):
                     s.iloc[3.0] = 0
@@ -162,9 +162,9 @@ def test_scalar_non_numeric(self):
             s = Series(np.arange(len(i)), index=i)
             s[3]
             msg = (
-                r"cannot do (label|index) indexing "
+                r"cannot do label indexing "
                 r"on {klass} with these indexers \[3\.0\] of "
-                r"{kind}".format(klass=type(i), kind=str(float))
+                r"type float".format(klass=type(i).__name__)
             )
             with pytest.raises(TypeError, match=msg):
                 s[3.0]
@@ -181,9 +181,9 @@ def test_scalar_with_mixed(self):
             msg = (
                 r"cannot do label indexing "
                 r"on {klass} with these indexers \[1\.0\] of "
-                r"{kind}|"
+                r"type float|"
                 "Cannot index by location index with a non-integer key".format(
-                    klass=str(Index), kind=str(float)
+                    klass=Index.__name__
                 )
             )
             with pytest.raises(TypeError, match=msg):
@@ -203,7 +203,7 @@ def test_scalar_with_mixed(self):
             msg = (
                 r"cannot do label indexing "
                 r"on {klass} with these indexers \[1\.0\] of "
-                r"{kind}".format(klass=str(Index), kind=str(float))
+                r"type float".format(klass=Index.__name__)
             )
             with pytest.raises(TypeError, match=msg):
                 idxr(s3)[1.0]
@@ -317,7 +317,7 @@ def test_scalar_float(self):
             msg = (
                 r"cannot do positional indexing "
                 r"on {klass} with these indexers \[3\.0\] of "
-                r"{kind}".format(klass=str(Float64Index), kind=str(float))
+                r"type float".format(klass=Float64Index.__name__)
             )
             with pytest.raises(TypeError, match=msg):
                 s2.iloc[3.0] = 0
@@ -346,24 +346,20 @@ def test_slice_non_numeric(self):
                 for l in [slice(3.0, 4), slice(3, 4.0), slice(3.0, 4.0)]:
 
                     msg = (
-                        "cannot do slice indexing "
+                        "cannot do positional indexing "
                         r"on {klass} with these indexers \[(3|4)\.0\] of "
-                        "{kind}".format(klass=type(index), kind=str(float))
+                        "type float".format(klass=type(index).__name__)
                     )
                     with pytest.raises(TypeError, match=msg):
                         s.iloc[l]
 
                     for idxr in [lambda x: x.loc, lambda x: x.iloc, lambda x: x]:
 
                         msg = (
-                            "cannot do slice indexing "
+                            "cannot do (slice|positional) indexing "
                             r"on {klass} with these indexers "
                             r"\[(3|4)(\.0)?\] "
-                            r"of ({kind_float}|{kind_int})".format(
-                                klass=type(index),
-                                kind_float=str(float),
-                                kind_int=str(int),
-                            )
+                            r"of type (float|int)".format(klass=type(index).__name__)
                         )
                         with pytest.raises(TypeError, match=msg):
                             idxr(s)[l]
@@ -372,23 +368,19 @@ def test_slice_non_numeric(self):
                 for l in [slice(3.0, 4), slice(3, 4.0), slice(3.0, 4.0)]:
 
                     msg = (
-                        "cannot do slice indexing "
+                        "cannot do positional indexing "
                         r"on {klass} with these indexers \[(3|4)\.0\] of "
-                        "{kind}".format(klass=type(index), kind=str(float))
+                        "type float".format(klass=type(index).__name__)
                     )
                     with pytest.raises(TypeError, match=msg):
                         s.iloc[l] = 0
 
                     for idxr in [lambda x: x.loc, lambda x: x.iloc, lambda x: x]:
                         msg = (
-                            "cannot do slice indexing "
+                            "cannot do (slice|positional) indexing "
                             r"on {klass} with these indexers "
                             r"\[(3|4)(\.0)?\] "
-                            r"of ({kind_float}|{kind_int})".format(
-                                klass=type(index),
-                                kind_float=str(float),
-                                kind_int=str(int),
-                            )
+                            r"of type (float|int)".format(klass=type(index).__name__)
                         )
                         with pytest.raises(TypeError, match=msg):
                             idxr(s)[l] = 0
@@ -428,7 +420,7 @@ def test_slice_integer(self):
                 msg = (
                     "cannot do slice indexing "
                     r"on {klass} with these indexers \[(3|4)\.0\] of "
-                    "{kind}".format(klass=type(index), kind=str(float))
+                    "type float".format(klass=type(index).__name__)
                 )
                 with pytest.raises(TypeError, match=msg):
                     s[l]
@@ -452,7 +444,7 @@ def test_slice_integer(self):
             msg = (
                 "cannot do slice indexing "
                 r"on {klass} with these indexers \[-6\.0\] of "
-                "{kind}".format(klass=type(index), kind=str(float))
+                "type float".format(klass=type(index).__name__)
             )
             with pytest.raises(TypeError, match=msg):
                 s[slice(-6.0, 6.0)]
@@ -478,7 +470,7 @@ def test_slice_integer(self):
                 msg = (
                     "cannot do slice indexing "
                     r"on {klass} with these indexers \[(2|3)\.5\] of "
-                    "{kind}".format(klass=type(index), kind=str(float))
+                    "type float".format(klass=type(index).__name__)
                 )
                 with pytest.raises(TypeError, match=msg):
                     s[l]
@@ -496,7 +488,7 @@ def test_slice_integer(self):
                 msg = (
                     "cannot do slice indexing "
                     r"on {klass} with these indexers \[(3|4)\.0\] of "
-                    "{kind}".format(klass=type(index), kind=str(float))
+                    "type float".format(klass=type(index).__name__)
                 )
                 with pytest.raises(TypeError, match=msg):
                     s[l] = 0
@@ -517,9 +509,9 @@ def test_integer_positional_indexing(self):
 
                 klass = RangeIndex
                 msg = (
-                    "cannot do slice indexing "
+                    "cannot do (slice|positional) indexing "
                     r"on {klass} with these indexers \[(2|4)\.0\] of "
-                    "{kind}".format(klass=str(klass), kind=str(float))
+                    "type float".format(klass=klass.__name__)
                 )
                 with pytest.raises(TypeError, match=msg):
                     idxr(s)[l]
@@ -544,7 +536,7 @@ def f(idxr):
                     msg = (
                         "cannot do slice indexing "
                         r"on {klass} with these indexers \[(0|1)\.0\] of "
-                        "{kind}".format(klass=type(index), kind=str(float))
+                        "type float".format(klass=type(index).__name__)
                     )
                     with pytest.raises(TypeError, match=msg):
                         s[l]
@@ -559,7 +551,7 @@ def f(idxr):
                 msg = (
                     "cannot do slice indexing "
                     r"on {klass} with these indexers \[-10\.0\] of "
-                    "{kind}".format(klass=type(index), kind=str(float))
+                    "type float".format(klass=type(index).__name__)
                 )
                 with pytest.raises(TypeError, match=msg):
                     s[slice(-10.0, 10.0)]
@@ -578,7 +570,7 @@ def f(idxr):
                     msg = (
                         "cannot do slice indexing "
                         r"on {klass} with these indexers \[0\.5\] of "
-                        "{kind}".format(klass=type(index), kind=str(float))
+                        "type float".format(klass=type(index).__name__)
                     )
                     with pytest.raises(TypeError, match=msg):
                         s[l]
@@ -595,7 +587,7 @@ def f(idxr):
                     msg = (
                         "cannot do slice indexing "
                         r"on {klass} with these indexers \[(3|4)\.0\] of "
-                        "{kind}".format(klass=type(index), kind=str(float))
+                        "type float".format(klass=type(index).__name__)
                     )
                     with pytest.raises(TypeError, match=msg):
                         s[l] = 0

pandas/tests/indexing/test_scalar.py

@@ -140,8 +140,8 @@ def test_series_at_raises_type_error(self):
         assert result == 1
 
         msg = (
-            "cannot do label indexing on <class 'pandas.core.indexes.base.Index'> "
-            r"with these indexers \[0\] of <class 'int'>"
+            "cannot do label indexing on Index "
+            r"with these indexers \[0\] of type int"
         )
         with pytest.raises(TypeError, match=msg):
             ser.at[0]
@@ -157,8 +157,8 @@ def test_frame_raises_type_error(self):
         assert result == 1
 
         msg = (
-            "cannot do label indexing on <class 'pandas.core.indexes.base.Index'> "
-            r"with these indexers \[0\] of <class 'int'>"
+            "cannot do label indexing on Index "
+            r"with these indexers \[0\] of type int"
         )
         with pytest.raises(TypeError, match=msg):
             df.at["a", 0]

pandas/tests/io/json/test_pandas.py

@@ -1671,3 +1671,13 @@ def test_to_s3(self, s3_resource):
         assert target_file in (
             obj.key for obj in s3_resource.Bucket("pandas-test").objects.all()
         )
+
+    def test_json_pandas_na(self):
+        # GH 31615
+        result = pd.DataFrame([[pd.NA]]).to_json()
+        assert result == '{"0":{"0":null}}'
+
+    def test_json_pandas_nulls(self, nulls_fixture):
+        # GH 31615
+        result = pd.DataFrame([[nulls_fixture]]).to_json()
+        assert result == '{"0":{"0":null}}'