Skip to content

Error on bad lines pyarrow #45029

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 29, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 47 additions & 46 deletions pandas/io/parsers/base_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,52 +77,6 @@
)
from pandas.io.date_converters import generic_parser

parser_defaults = {
"delimiter": None,
"escapechar": None,
"quotechar": '"',
"quoting": csv.QUOTE_MINIMAL,
"doublequote": True,
"skipinitialspace": False,
"lineterminator": None,
"header": "infer",
"index_col": None,
"names": None,
"prefix": None,
"skiprows": None,
"skipfooter": 0,
"nrows": None,
"na_values": None,
"keep_default_na": True,
"true_values": None,
"false_values": None,
"converters": None,
"dtype": None,
"cache_dates": True,
"thousands": None,
"comment": None,
"decimal": ".",
# 'engine': 'c',
"parse_dates": False,
"keep_date_col": False,
"dayfirst": False,
"date_parser": None,
"usecols": None,
# 'iterator': False,
"chunksize": None,
"verbose": False,
"encoding": None,
"squeeze": None,
"compression": None,
"mangle_dupe_cols": True,
"infer_datetime_format": False,
"skip_blank_lines": True,
"encoding_errors": "strict",
"on_bad_lines": "error",
"error_bad_lines": None,
"warn_bad_lines": None,
}


class ParserBase:
class BadLineHandleMethod(Enum):
Expand Down Expand Up @@ -1178,6 +1132,53 @@ def converter(*date_cols):
return converter


parser_defaults = {
"delimiter": None,
"escapechar": None,
"quotechar": '"',
"quoting": csv.QUOTE_MINIMAL,
"doublequote": True,
"skipinitialspace": False,
"lineterminator": None,
"header": "infer",
"index_col": None,
"names": None,
"prefix": None,
"skiprows": None,
"skipfooter": 0,
"nrows": None,
"na_values": None,
"keep_default_na": True,
"true_values": None,
"false_values": None,
"converters": None,
"dtype": None,
"cache_dates": True,
"thousands": None,
"comment": None,
"decimal": ".",
# 'engine': 'c',
"parse_dates": False,
"keep_date_col": False,
"dayfirst": False,
"date_parser": None,
"usecols": None,
# 'iterator': False,
"chunksize": None,
"verbose": False,
"encoding": None,
"squeeze": None,
"compression": None,
"mangle_dupe_cols": True,
"infer_datetime_format": False,
"skip_blank_lines": True,
"encoding_errors": "strict",
"on_bad_lines": ParserBase.BadLineHandleMethod.ERROR,
"error_bad_lines": None,
"warn_bad_lines": None,
}


def _process_date_conversion(
data_dict,
converter: Callable,
Expand Down
16 changes: 12 additions & 4 deletions pandas/io/parsers/readers.py
Original file line number Diff line number Diff line change
Expand Up @@ -434,10 +434,7 @@
"dialect",
"warn_bad_lines",
"error_bad_lines",
# TODO(1.4)
# This doesn't error properly ATM, fix for release
# but not blocker for initial PR
# "on_bad_lines",
"on_bad_lines",
"delim_whitespace",
"quoting",
"lineterminator",
Expand Down Expand Up @@ -932,7 +929,18 @@ def _get_options_with_defaults(self, engine):
engine == "pyarrow"
and argname in _pyarrow_unsupported
and value != default
and value != getattr(value, "value", default)
):
if (
argname == "on_bad_lines"
and kwds.get("error_bad_lines") is not None
):
argname = "error_bad_lines"
elif (
argname == "on_bad_lines" and kwds.get("warn_bad_lines") is not None
):
argname = "warn_bad_lines"

raise ValueError(
f"The {repr(argname)} option is not supported with the "
f"'pyarrow' engine"
Expand Down
4 changes: 3 additions & 1 deletion pandas/tests/io/parser/test_unsupported.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,12 @@ def test_pyarrow_engine(self):
f"supported with the 'pyarrow' engine"
)
kwargs = {default: object()}
default_needs_bool = {"on_bad_lines", "error_bad_lines"}
default_needs_bool = {"warn_bad_lines", "error_bad_lines"}
if default == "dialect":
kwargs[default] = "excel" # test a random dialect
elif default in default_needs_bool:
kwargs[default] = True
elif default == "on_bad_lines":
kwargs[default] = "warn"
with pytest.raises(ValueError, match=msg):
read_csv(StringIO(data), engine="pyarrow", **kwargs)