Skip to content

ISSUE 227: add parser_settings.debug flag and enable csv2 reader to optionally inject debug (line) info into record IDR. #228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 21, 2025
Merged

ISSUE 227: add parser_settings.debug flag and enable csv2 reader to optionally inject debug (line) info into record IDR. #228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions doc/csv2_in_depth.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,27 @@ Then the output for the above sample input will be:
]
```

## Debug Info
If debug information is desired to be added into each record's IDR therefore becoming available for
`transform_declarations` use, one can specify `"debug": 1` in `parser_settings` section of the schema:
```
{
"parser_settings": {
"version": "omni.2.1",
"file_format_type": "csv2",
"debug": 1
},
...
}
```
If the `csv2` parser detects `"debug": 1` setting, it will add the following debug info into the
current record's IDR structure:

- xpath: `__debug/line_num`: contains the starting line number in the CSV file of the current
record.

Check this [sample](../extensions/omniv21/samples/csv2/1_single_row.schema.json) for usage pattern.

## Migration from `'csv'` Schemas

If one looks at the documentation for the old `csv` schema [here](./csv_in_depth.md), you notice
Expand Down
6 changes: 5 additions & 1 deletion extensions/omniv21/fileformat/csv/format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat"
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
v21validation "github.com/jf-tech/omniparser/extensions/omniv21/validation"
"github.com/jf-tech/omniparser/header"
"github.com/jf-tech/omniparser/validation"
)

Expand Down Expand Up @@ -97,7 +98,10 @@ func (f *csvFileFormat) validateColumns(columns []Column) error {
}

func (f *csvFileFormat) CreateFormatReader(
name string, r io.Reader, runtime interface{}) (fileformat.FormatReader, error) {
_ header.Header,
name string,
r io.Reader,
runtime interface{}) (fileformat.FormatReader, error) {
csv := runtime.(*csvFormatRuntime)
return NewReader(name, r, csv.Decl, csv.XPath)
}
Expand Down
2 changes: 2 additions & 0 deletions extensions/omniv21/fileformat/csv/format_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/jf-tech/omniparser/errs"
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
"github.com/jf-tech/omniparser/header"
"github.com/jf-tech/omniparser/idr"
)

Expand Down Expand Up @@ -157,6 +158,7 @@ func TestValidateSchema(t *testing.T) {

func TestCreateFormatReader(t *testing.T) {
r, err := NewCSVFileFormat("test").CreateFormatReader(
header.Header{},
"test-input",
strings.NewReader(
lf("A|B|C")+
Expand Down
6 changes: 5 additions & 1 deletion extensions/omniv21/fileformat/edi/format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat"
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
v21validation "github.com/jf-tech/omniparser/extensions/omniv21/validation"
"github.com/jf-tech/omniparser/header"
"github.com/jf-tech/omniparser/validation"
)

Expand Down Expand Up @@ -74,7 +75,10 @@ func (f *ediFileFormat) validateFileDecl(decl *FileDecl) error {
}

func (f *ediFileFormat) CreateFormatReader(
name string, r io.Reader, runtime interface{}) (fileformat.FormatReader, error) {
_ header.Header,
name string,
r io.Reader,
runtime interface{}) (fileformat.FormatReader, error) {
edi := runtime.(*ediFormatRuntime)
return NewReader(name, r, edi.Decl, edi.XPath)
}
Expand Down
4 changes: 3 additions & 1 deletion extensions/omniv21/fileformat/edi/format_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/assert"

"github.com/jf-tech/omniparser/extensions/omniv21/transform"
"github.com/jf-tech/omniparser/header"
"github.com/jf-tech/omniparser/idr"
)

Expand Down Expand Up @@ -172,7 +173,8 @@ func TestCreateFormatReader(t *testing.T) {
}`
rt, err := format.ValidateSchema(fileFormatEDI, []byte(fileDecl), &transform.Decl{XPath: strs.StrPtr(".")})
assert.NoError(t, err)
reader, err := format.CreateFormatReader("test", strings.NewReader("ISA*e1*e2*e3\nISA*e4*e5*e6\n"), rt)
reader, err := format.CreateFormatReader(
header.Header{}, "test", strings.NewReader("ISA*e1*e2*e3\nISA*e4*e5*e6\n"), rt)
assert.NoError(t, err)
n, err := reader.Read()
assert.NoError(t, err)
Expand Down
6 changes: 5 additions & 1 deletion extensions/omniv21/fileformat/fileformat.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/jf-tech/omniparser/errs"
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
"github.com/jf-tech/omniparser/header"
"github.com/jf-tech/omniparser/idr"
)

Expand All @@ -18,7 +19,10 @@ type FileFormat interface {

// CreateFormatReader creates an FormatReader which reads records of input data for this file format.
CreateFormatReader(
inputName string, input io.Reader, formatRuntime interface{}) (FormatReader, error)
schemaHeader header.Header,
inputName string,
input io.Reader,
formatRuntime interface{}) (FormatReader, error)
}

// FormatReader is an interface for reading a specific input format in omni schema handler. We'll have
Expand Down
6 changes: 5 additions & 1 deletion extensions/omniv21/fileformat/fixedlength/format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat"
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
v21validation "github.com/jf-tech/omniparser/extensions/omniv21/validation"
"github.com/jf-tech/omniparser/header"
"github.com/jf-tech/omniparser/validation"
)

Expand Down Expand Up @@ -128,7 +129,10 @@ func (f *fixedLengthFileFormat) validateColumns(cols []*ColumnDecl) error {
}

func (f *fixedLengthFileFormat) CreateFormatReader(
name string, r io.Reader, runtime interface{}) (fileformat.FormatReader, error) {
_ header.Header,
name string,
r io.Reader,
runtime interface{}) (fileformat.FormatReader, error) {
rt := runtime.(*fixedLengthFormatRuntime)
return NewReader(name, r, rt.Decl, rt.XPath)
}
Expand Down
2 changes: 2 additions & 0 deletions extensions/omniv21/fileformat/fixedlength/format_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/jf-tech/omniparser/errs"
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
"github.com/jf-tech/omniparser/header"
"github.com/jf-tech/omniparser/idr"
)

Expand Down Expand Up @@ -308,6 +309,7 @@ func TestValidateSchema(t *testing.T) {

func TestCreateFormatReader(t *testing.T) {
r, err := NewFixedLengthFileFormat("test").CreateFormatReader(
header.Header{},
"test",
strings.NewReader("abcd\n1234\n"),
&fixedLengthFormatRuntime{
Expand Down
78 changes: 74 additions & 4 deletions extensions/omniv21/fileformat/flatfile/csv/.snapshots/TestRead-multiple_records
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,40 @@
{
"Children": [
{
"Children": [
{
"Children": [
{
"Children": null,
"Data": "1",
"FirstChild": null,
"FormatSpecific": null,
"LastChild": null,
"NextSibling": null,
"Parent": "(ElementNode line_num)",
"PrevSibling": null,
"Type": "TextNode"
}
],
"Data": "line_num",
"FirstChild": "(TextNode '1')",
"FormatSpecific": null,
"LastChild": "(TextNode '1')",
"NextSibling": null,
"Parent": "(ElementNode __debug)",
"PrevSibling": null,
"Type": "ElementNode"
}
],
"Data": "__debug",
"FirstChild": "(ElementNode line_num)",
"FormatSpecific": null,
"LastChild": "(ElementNode line_num)",
"NextSibling": "(ElementNode r1c1)",
"Parent": "(ElementNode r1)",
"PrevSibling": null,
"Type": "ElementNode"
},
{
"Children": [
{
Expand All @@ -20,7 +55,7 @@
"LastChild": "(TextNode 'v1')",
"NextSibling": "(ElementNode r1c2)",
"Parent": "(ElementNode r1)",
"PrevSibling": null,
"PrevSibling": "(ElementNode __debug)",
"Type": "ElementNode"
},
{
Expand Down Expand Up @@ -48,7 +83,7 @@
}
],
"Data": "r1",
"FirstChild": "(ElementNode r1c1)",
"FirstChild": "(ElementNode __debug)",
"FormatSpecific": null,
"LastChild": "(ElementNode r1c2)",
"NextSibling": null,
Expand All @@ -58,6 +93,41 @@
},
{
"Children": [
{
"Children": [
{
"Children": [
{
"Children": null,
"Data": "3",
"FirstChild": null,
"FormatSpecific": null,
"LastChild": null,
"NextSibling": null,
"Parent": "(ElementNode line_num)",
"PrevSibling": null,
"Type": "TextNode"
}
],
"Data": "line_num",
"FirstChild": "(TextNode '3')",
"FormatSpecific": null,
"LastChild": "(TextNode '3')",
"NextSibling": null,
"Parent": "(ElementNode __debug)",
"PrevSibling": null,
"Type": "ElementNode"
}
],
"Data": "__debug",
"FirstChild": "(ElementNode line_num)",
"FormatSpecific": null,
"LastChild": "(ElementNode line_num)",
"NextSibling": "(ElementNode r1c1)",
"Parent": "(ElementNode r1)",
"PrevSibling": null,
"Type": "ElementNode"
},
{
"Children": [
{
Expand All @@ -78,12 +148,12 @@
"LastChild": "(TextNode '')",
"NextSibling": null,
"Parent": "(ElementNode r1)",
"PrevSibling": null,
"PrevSibling": "(ElementNode __debug)",
"Type": "ElementNode"
}
],
"Data": "r1",
"FirstChild": "(ElementNode r1c1)",
"FirstChild": "(ElementNode __debug)",
"FormatSpecific": null,
"LastChild": "(ElementNode r1c1)",
"NextSibling": null,
Expand Down
8 changes: 6 additions & 2 deletions extensions/omniv21/fileformat/flatfile/csv/format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat"
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
v21validation "github.com/jf-tech/omniparser/extensions/omniv21/validation"
"github.com/jf-tech/omniparser/header"
"github.com/jf-tech/omniparser/validation"
)

Expand Down Expand Up @@ -76,7 +77,10 @@ func (f *csvFormat) validateFileDecl(decl *FileDecl) error {
}

func (f *csvFormat) CreateFormatReader(
name string, r io.Reader, runtime interface{}) (fileformat.FormatReader, error) {
schemaHeader header.Header,
name string,
r io.Reader,
runtime interface{}) (fileformat.FormatReader, error) {
rt := runtime.(*csvFormatRuntime)
targetXPathExpr, err := func() (*xpath.Expr, error) {
if rt.XPath == "" || rt.XPath == "." {
Expand All @@ -87,7 +91,7 @@ func (f *csvFormat) CreateFormatReader(
if err != nil {
return nil, f.FmtErr("xpath '%s' on 'FINAL_OUTPUT' is invalid: %s", rt.XPath, err.Error())
}
return NewReader(name, r, rt.Decl, targetXPathExpr), nil
return NewReader(schemaHeader, name, r, rt.Decl, targetXPathExpr), nil
}

func (f *csvFormat) FmtErr(format string, args ...interface{}) error {
Expand Down
3 changes: 3 additions & 0 deletions extensions/omniv21/fileformat/flatfile/csv/format_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/jf-tech/omniparser/errs"
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
"github.com/jf-tech/omniparser/header"
"github.com/jf-tech/omniparser/idr"
)

Expand Down Expand Up @@ -284,6 +285,7 @@ func TestCreateFormatReader(t *testing.T) {
&transform.Decl{XPath: finalOutputXPath})
assert.NoError(t, err)
reader, err := format.CreateFormatReader(
header.Header{},
"test-input",
strings.NewReader("abcd|efgh|jklm\n123|456|789\n"),
runtime)
Expand All @@ -301,6 +303,7 @@ func TestCreateFormatReader(t *testing.T) {

// test CreateFormatReader called with invalid target xpath.
reader, err := NewCSVFileFormat("test-schema").CreateFormatReader(
header.Header{},
"test-input",
strings.NewReader("abcd\n1234\n"),
&csvFormatRuntime{XPath: "["})
Expand Down
Loading