Skip to content

Error on bad lines pyarrow #45029

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 29, 2021
Merged

Error on bad lines pyarrow #45029

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8e87ad1
Error for on_bad_lines in pyarrow engine
phofl Dec 23, 2021
6c58fe2
Handle enum
phofl Dec 23, 2021
549be36
Fix tests
phofl Dec 28, 2021
21605b4
Remove test
phofl Dec 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 47 additions & 46 deletions pandas/io/parsers/base_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,52 +77,6 @@
)
from pandas.io.date_converters import generic_parser

parser_defaults = {
"delimiter": None,
"escapechar": None,
"quotechar": '"',
"quoting": csv.QUOTE_MINIMAL,
"doublequote": True,
"skipinitialspace": False,
"lineterminator": None,
"header": "infer",
"index_col": None,
"names": None,
"prefix": None,
"skiprows": None,
"skipfooter": 0,
"nrows": None,
"na_values": None,
"keep_default_na": True,
"true_values": None,
"false_values": None,
"converters": None,
"dtype": None,
"cache_dates": True,
"thousands": None,
"comment": None,
"decimal": ".",
# 'engine': 'c',
"parse_dates": False,
"keep_date_col": False,
"dayfirst": False,
"date_parser": None,
"usecols": None,
# 'iterator': False,
"chunksize": None,
"verbose": False,
"encoding": None,
"squeeze": None,
"compression": None,
"mangle_dupe_cols": True,
"infer_datetime_format": False,
"skip_blank_lines": True,
"encoding_errors": "strict",
"on_bad_lines": "error",
"error_bad_lines": None,
"warn_bad_lines": None,
}


class ParserBase:
class BadLineHandleMethod(Enum):
Expand Down Expand Up @@ -1178,6 +1132,53 @@ def converter(*date_cols):
return converter


parser_defaults = {
"delimiter": None,
"escapechar": None,
"quotechar": '"',
"quoting": csv.QUOTE_MINIMAL,
"doublequote": True,
"skipinitialspace": False,
"lineterminator": None,
"header": "infer",
"index_col": None,
"names": None,
"prefix": None,
"skiprows": None,
"skipfooter": 0,
"nrows": None,
"na_values": None,
"keep_default_na": True,
"true_values": None,
"false_values": None,
"converters": None,
"dtype": None,
"cache_dates": True,
"thousands": None,
"comment": None,
"decimal": ".",
# 'engine': 'c',
"parse_dates": False,
"keep_date_col": False,
"dayfirst": False,
"date_parser": None,
"usecols": None,
# 'iterator': False,
"chunksize": None,
"verbose": False,
"encoding": None,
"squeeze": None,
"compression": None,
"mangle_dupe_cols": True,
"infer_datetime_format": False,
"skip_blank_lines": True,
"encoding_errors": "strict",
"on_bad_lines": ParserBase.BadLineHandleMethod.ERROR,
"error_bad_lines": None,
"warn_bad_lines": None,
}


def _process_date_conversion(
data_dict,
converter: Callable,
Expand Down
11 changes: 7 additions & 4 deletions pandas/io/parsers/readers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,10 +434,7 @@
"dialect",
"warn_bad_lines",
"error_bad_lines",
# TODO(1.4)
# This doesn't error properly ATM, fix for release
# but not blocker for initial PR
# "on_bad_lines",
"on_bad_lines",
"delim_whitespace",
"quoting",
"lineterminator",
Expand Down Expand Up @@ -932,7 +929,13 @@ def _get_options_with_defaults(self, engine):
engine == "pyarrow"
and argname in _pyarrow_unsupported
and value != default
and value != getattr(value, "value", default)
):
if argname == "on_bad_lines" and kwds.get("error_bad_lines"):
argname = "error_bad_lines"
elif argname == "on_bad_lines" and kwds.get("warn_bad_lines"):
argname = "warn_bad_lines"

raise ValueError(
f"The {repr(argname)} option is not supported with the "
f"'pyarrow' engine"
Expand Down
20 changes: 20 additions & 0 deletions pandas/tests/io/parser/test_unsupported.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,9 @@ def test_pyarrow_engine(self):
1,2,3,4,"""

for default in pa_unsupported:
if default == "on_bad_lines":
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason that on_bad_lines is skipped here and tested separately?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, missed that. The test was failing before but forget to fix. Adjusted it now accordingly

continue

msg = (
f"The {repr(default)} option is not "
f"supported with the 'pyarrow' engine"
Expand All @@ -147,3 +150,20 @@ def test_pyarrow_engine(self):
kwargs[default] = True
with pytest.raises(ValueError, match=msg):
read_csv(StringIO(data), engine="pyarrow", **kwargs)

@pytest.mark.parametrize(
"kwds",
[{"on_bad_lines": "warn"}, {"error_bad_lines": True}, {"warn_bad_lines": True}],
)
def test_pyarrow_bad_lines_fails(self, pyarrow_parser_only, kwds):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this test be removed now? It seems to duplicate the test_pyarrow_engine test above now.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good point, thx

# GH#
data = """a,b,c
1,2,3
"""
parser = pyarrow_parser_only
msg = (
f"The '{list(kwds.keys())[0]}' option is not supported "
f"with the 'pyarrow' engine"
)
with pytest.raises(ValueError, match=msg):
parser.read_csv(StringIO(data), **kwds)