Merge branch 'master' of https://github.com/pandas-dev/pandas into cln-idx

Author: jbrockmendel

.devcontainer.json

@@ -0,0 +1,28 @@
+// For format details, see https://aka.ms/vscode-remote/devcontainer.json or the definition README at
+// https://github.com/microsoft/vscode-dev-containers/tree/master/containers/python-3-miniconda
+{
+	"name": "pandas",
+	"context": ".",
+	"dockerFile": "Dockerfile",
+
+	// Use 'settings' to set *default* container specific settings.json values on container create.
+	// You can edit these settings after create using File > Preferences > Settings > Remote.
+	"settings": {
+		"terminal.integrated.shell.linux": "/bin/bash",
+		"python.condaPath": "/opt/conda/bin/conda",
+		"python.pythonPath": "/opt/conda/bin/python",
+		"python.formatting.provider": "black",
+		"python.linting.enabled": true,
+		"python.linting.flake8Enabled": true,
+		"python.linting.pylintEnabled": false,
+		"python.linting.mypyEnabled": true,
+		"python.testing.pytestEnabled": true,
+		"python.testing.cwd": "pandas/tests"
+	},
+
+	// Add the IDs of extensions you want installed when the container is created in the array below.
+	"extensions": [
+		"ms-python.python",
+		"ms-vscode.cpptools"
+	]
+}

.travis.yml

@@ -7,10 +7,10 @@ python: 3.7
 # travis cache --delete inside the project directory from the travis command line client
 # The cache directories will be deleted if anything in ci/ changes in a commit
 cache:
- ccache: true
- directories:
-  - $HOME/.cache # cython cache
-  - $HOME/.ccache # compiler cache
+  ccache: true
+  directories:
+    - $HOME/.cache # cython cache
+    - $HOME/.ccache # compiler cache
 
 env:
   global:
@@ -20,13 +20,13 @@ env:
     - secure: "EkWLZhbrp/mXJOx38CHjs7BnjXafsqHtwxPQrqWy457VDFWhIY1DMnIR/lOWG+a20Qv52sCsFtiZEmMfUjf0pLGXOqurdxbYBGJ7/ikFLk9yV2rDwiArUlVM9bWFnFxHvdz9zewBH55WurrY4ShZWyV+x2dWjjceWG5VpWeI6sA="
 
 git:
-    # for cloning
-    depth: false
+  # for cloning
+  depth: false
 
 matrix:
-    fast_finish: true
+  fast_finish: true
 
-    include:
+  include:
     - env:
         - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
 
@@ -40,6 +40,9 @@ matrix:
         - postgresql
 
     - env:
+        # Enabling Deprecations when running tests
+        # PANDAS_TESTING_MODE="deprecate" causes DeprecationWarning messages to be displayed in the logs
+        # See pandas/_testing.py for more details.
         - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
       services:
         - mysql
@@ -70,7 +73,6 @@ before_install:
   # This overrides travis and tells it to look nowhere.
   - export BOTO_CONFIG=/dev/null
 
-
 install:
   - echo "install start"
   - ci/prep_cython_cache.sh
@@ -87,5 +89,5 @@ script:
 after_script:
   - echo "after_script start"
   - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
-  - ci/print_skipped.py 
+  - ci/print_skipped.py
   - echo "after_script done"

Dockerfile

@@ -0,0 +1,47 @@
+FROM continuumio/miniconda3
+
+# if you forked pandas, you can pass in your own GitHub username to use your fork
+# i.e. gh_username=myname
+ARG gh_username=pandas-dev
+ARG pandas_home="/home/pandas"
+
+# Avoid warnings by switching to noninteractive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Configure apt and install packages
+RUN apt-get update \
+    && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 \
+    #
+    # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed
+    && apt-get -y install git iproute2 procps iproute2 lsb-release \
+    #
+    # Install C compilers (gcc not enough, so just went with build-essential which admittedly might be overkill),
+    # needed to build pandas C extensions
+    && apt-get -y install build-essential \
+    #
+    # cleanup
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Switch back to dialog for any ad-hoc use of apt-get
+ENV DEBIAN_FRONTEND=dialog
+
+# Clone pandas repo
+RUN mkdir "$pandas_home" \
+    && git clone "https://github.com/$gh_username/pandas.git" "$pandas_home" \
+    && cd "$pandas_home" \
+    && git remote add upstream "https://github.com/pandas-dev/pandas.git" \
+    && git pull upstream master
+
+# Because it is surprisingly difficult to activate a conda environment inside a DockerFile
+# (from personal experience and per https://github.com/ContinuumIO/docker-images/issues/89),
+# we just update the base/root one from the 'environment.yml' file instead of creating a new one.
+#
+# Set up environment
+RUN conda env update -n base -f "$pandas_home/environment.yml"
+
+# Build C extensions and pandas
+RUN cd "$pandas_home" \
+    && python setup.py build_ext --inplace -j 4 \
+    && python -m pip install -e .

LICENSE

@@ -1,8 +1,10 @@
 BSD 3-Clause License
 
-Copyright (c) 2008-2012, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team
+Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team
 All rights reserved.
 
+Copyright (c) 2011-2020, Open source contributors.
+
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
 

doc/source/development/contributing.rst

@@ -146,6 +146,17 @@ requires a C compiler and Python environment. If you're making documentation
 changes, you can skip to :ref:`contributing.documentation` but you won't be able
 to build the documentation locally before pushing your changes.
 
+Using a Docker Container
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Instead of manually setting up a development environment, you can use Docker to
+automatically create the environment with just several commands. Pandas provides a `DockerFile`
+in the root directory to build a Docker image with a full pandas development environment.
+
+Even easier, you can use the DockerFile to launch a remote session with Visual Studio Code,
+a popular free IDE, using the `.devcontainer.json` file.
+See https://code.visualstudio.com/docs/remote/containers for details.
+
 .. _contributing.dev_c:
 
 Installing a C compiler
@@ -1525,3 +1536,19 @@ The branch will still exist on GitHub, so to delete it there do::
     git push origin --delete shiny-new-feature
 
 .. _Gitter: https://gitter.im/pydata/pandas
+
+
+Tips for a successful Pull Request
+==================================
+
+If you have made it to the `Review your code`_ phase, one of the core contributors may
+take a look. Please note however that a handful of people are responsible for reviewing
+all of the contributions, which can often lead to bottlenecks.
+
+To improve the chances of your pull request being reviewed, you should:
+
+- **Reference an open issue** for non-trivial changes to clarify the PR's purpose
+- **Ensure you have appropriate tests**. These should be the first part of any PR
+- **Keep your pull requests as simple as possible**. Larger PRs take longer to review
+- **Ensure that CI is in a green state**. Reviewers may not even look otherwise
+- **Keep** `Updating your pull request`_, either by request or every few days

doc/source/ecosystem.rst

@@ -122,16 +122,14 @@ also goes beyond matplotlib and pandas with the option to perform statistical
 estimation while plotting, aggregating across observations and visualizing the
 fit of statistical models to emphasize patterns in a dataset.
 
-`yhat/ggpy <https://github.com/yhat/ggpy>`__
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+`plotnine <https://github.com/has2k1/plotnine/>`__
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Hadley Wickham's `ggplot2 <https://ggplot2.tidyverse.org/>`__ is a foundational exploratory visualization package for the R language.
 Based on `"The Grammar of Graphics" <https://www.cs.uic.edu/~wilkinson/TheGrammarOfGraphics/GOG.html>`__ it
 provides a powerful, declarative and extremely general way to generate bespoke plots of any kind of data.
-It's really quite incredible. Various implementations to other languages are available,
-but a faithful implementation for Python users has long been missing. Although still young
-(as of Jan-2014), the `yhat/ggpy <https://github.com/yhat/ggpy>`__ project has been
-progressing quickly in that direction.
+Various implementations to other languages are available.
+A good implementation for Python users is `has2k1/plotnine <https://github.com/has2k1/plotnine/>`__.
 
 `IPython Vega <https://github.com/vega/ipyvega>`__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/whatsnew/v1.0.0.rst

@@ -483,6 +483,25 @@ Use :meth:`arrays.IntegerArray.to_numpy` with an explicit ``na_value`` instead.
 
    a.to_numpy(dtype="float", na_value=np.nan)
 
+**Reductions can return ``pd.NA``**
+
+When performing a reduction such as a sum with ``skipna=False``, the result
+will now be ``pd.NA`` instead of ``np.nan`` in presence of missing values
+(:issue:`30958`).
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+    >>> pd.Series(a).sum(skipna=False)
+    nan
+
+*pandas 1.0.0*
+
+.. ipython:: python
+
+   pd.Series(a).sum(skipna=False)
+
 **value_counts returns a nullable integer dtype**
 
 :meth:`Series.value_counts` with a nullable integer dtype now returns a nullable

doc/source/whatsnew/v1.1.0.rst

@@ -60,7 +60,7 @@ Categorical
 Datetimelike
 ^^^^^^^^^^^^
 - Bug in :class:`Timestamp` where constructing :class:`Timestamp` from ambiguous epoch time and calling constructor again changed :meth:`Timestamp.value` property (:issue:`24329`)
--
+- :meth:`DatetimeArray.searchsorted`, :meth:`TimedeltaArray.searchsorted`, :meth:`PeriodArray.searchsorted` not recognizing non-pandas scalars and incorrectly raising ``ValueError`` instead of ``TypeError`` (:issue:`30950`)
 -
 
 Timedelta
@@ -102,7 +102,7 @@ Interval
 
 Indexing
 ^^^^^^^^
-
+- Bug in slicing on a :class:`DatetimeIndex` with a partial-timestamp dropping high-resolution indices near the end of a year, quarter, or month (:issue:`31064`)
 -
 -
 
@@ -142,6 +142,9 @@ Reshaping
 
 -
 - Bug in :meth:`DataFrame.pivot_table` when only MultiIndexed columns is set (:issue:`17038`)
+- Fix incorrect error message in :meth:`DataFrame.pivot` when ``columns`` is set to ``None``. (:issue:`30924`)
+- Bug in :func:`crosstab` when inputs are two Series and have tuple names, the output will keep dummy MultiIndex as columns. (:issue:`18321`)
+
 
 Sparse
 ^^^^^^

pandas/_libs/index.pyx

@@ -213,7 +213,8 @@ cdef class IndexEngine:
         return self.monotonic_dec == 1
 
     cdef inline _do_monotonic_check(self):
-        cdef object is_unique
+        cdef:
+            bint is_unique
         try:
             values = self._get_index_values()
             self.monotonic_inc, self.monotonic_dec, is_unique = \
@@ -236,10 +237,10 @@ cdef class IndexEngine:
     cdef _call_monotonic(self, values):
         return algos.is_monotonic(values, timelike=False)
 
-    def get_backfill_indexer(self, other, limit=None):
+    def get_backfill_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
         return algos.backfill(self._get_index_values(), other, limit=limit)
 
-    def get_pad_indexer(self, other, limit=None):
+    def get_pad_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
         return algos.pad(self._get_index_values(), other, limit=limit)
 
     cdef _make_hash_table(self, Py_ssize_t n):
@@ -477,13 +478,13 @@ cdef class DatetimeEngine(Int64Engine):
         values = np.asarray(values).view('i8')
         return self.mapping.lookup(values)
 
-    def get_pad_indexer(self, other, limit=None):
+    def get_pad_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
         if other.dtype != self._get_box_dtype():
             return np.repeat(-1, len(other)).astype('i4')
         other = np.asarray(other).view('i8')
         return algos.pad(self._get_index_values(), other, limit=limit)
 
-    def get_backfill_indexer(self, other, limit=None):
+    def get_backfill_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
         if other.dtype != self._get_box_dtype():
             return np.repeat(-1, len(other)).astype('i4')
         other = np.asarray(other).view('i8')
@@ -506,16 +507,13 @@ cdef class PeriodEngine(Int64Engine):
     cdef _get_index_values(self):
         return super(PeriodEngine, self).vgetter().view("i8")
 
-    cdef void _call_map_locations(self, values):
-        # super(...) pattern doesn't seem to work with `cdef`
-        Int64Engine._call_map_locations(self, values.view('i8'))
-
     cdef _call_monotonic(self, values):
         # super(...) pattern doesn't seem to work with `cdef`
         return Int64Engine._call_monotonic(self, values.view('i8'))
 
     def get_indexer(self, values):
-        cdef ndarray[int64_t, ndim=1] ordinals
+        cdef:
+            ndarray[int64_t, ndim=1] ordinals
 
         super(PeriodEngine, self)._ensure_mapping_populated()
 
@@ -524,14 +522,14 @@ cdef class PeriodEngine(Int64Engine):
 
         return self.mapping.lookup(ordinals)
 
-    def get_pad_indexer(self, other, limit=None):
+    def get_pad_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
         freq = super(PeriodEngine, self).vgetter().freq
         ordinal = periodlib.extract_ordinals(other, freq)
 
         return algos.pad(self._get_index_values(),
                          np.asarray(ordinal), limit=limit)
 
-    def get_backfill_indexer(self, other, limit=None):
+    def get_backfill_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
         freq = super(PeriodEngine, self).vgetter().freq
         ordinal = periodlib.extract_ordinals(other, freq)
 

pandas/_libs/index_class_helper.pxi.in

@@ -53,10 +53,7 @@ cdef class {{name}}Engine(IndexEngine):
             ndarray[{{ctype}}] values
             int count = 0
 
-        {{if name not in {'Float64', 'Float32'} }}
-        if not util.is_integer_object(val):
-            raise KeyError(val)
-        {{endif}}
+        self._check_type(val)
 
         # A view is needed for some subclasses, such as PeriodEngine:
         values = self._get_index_values().view('{{dtype}}')