Skip to content

Commit a9eae23

Browse files
alallemaalallema
committed
Add csv delimiter
1 parent 8e8a557 commit a9eae23

File tree

5 files changed

+117
-8
lines changed

5 files changed

+117
-8
lines changed

‎datasets/songs_delimiter.csv

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
id;title;album;artist;genre;country;released;duration;released-timestamp;duration-float
2+
702481615;Armatage Shanks;Dookie: The Ultimate Critical Review;Green Day;Rock;Europe;2005;;1104537600;
3+
888221515;Old Folks;Six Classic Albums Plus Bonus Tracks;Harold Land;Jazz;Europe;2013;6:36;1356998400;6.36
4+
1382413601;คำขอร้อง;สำเนียงคนจันทร์ / เอาเถอะถ้าเห็นเขาดีกว่า;อิทธิพล บำรุงกุล;"Folk; World; & Country";Thailand;;;;
5+
190889300;Track 1;Summer Breeze;Dreas;Funk / Soul;US;2008;18:56;1199145600;18.56
6+
813645611;Slave (Alternative Version);Honky Château;Elton John;Rock;Europe;;2:53;;2.5300000000000002
7+
394018506;Sex & Geld;Trackz Für Den Index;Mafia Clikk;Hip Hop;Germany;2006;5:02;1136073600;5.02
8+
1522481803;Pisciaunella;Don Pepp U Pacce;Giovanni Russo (2);"Folk; World; & Country";Italy;1980;;315532800;
9+
862296713;不知;Kiss 2001 Hong Kong Live Concert;Various;Electronic;Hong Kong;2002-04-13;;1018656000;
10+
467946423;Rot;Be Quick Or Be Dead Vol. 3;Various;Electronic;Serbia;2013-06-20;1:00;1371686400;1
11+
1323854803;"Simulation Project 1; ツキハナ「Moonflower」";Unlimited Dream Company;Amun Dragoon;Electronic;US;2018-04-10;2:44;1523318400;2.44
12+
235115704;Doctor Vine;The Big F;The Big F;Rock;US;1989;5:29;599616000;5.29
13+
249025232;"Ringel; Ringel; Reihe";Kinderlieder ABC - Der Bielefelder Kinderchor Singt 42 Lieder Von A-Z;Der Bielefelder Kinderchor;Children's;Germany;1971;;31536000;
14+
710094000;Happy Safari = Safari Feliz;Safari Swings Again = El Safari Sigue En Su Swing;Bert Kaempfert & His Orchestra;Jazz;Argentina;1977;2:45;220924800;2.45
15+
538632700;Take Me Up;Spring;Various;Electronic;US;2000;3:06;946684800;3.06
16+
1556505508;Doin To Me ( Radio Version );Say My Name;Netta Dogg;Hip Hop;US;2005;;1104537600;
17+
1067031900;Concerto For Balloon & Orchestra / Concerto For Synthesizer & Orchestra;Concerto For Balloon & Orchestra And Three Overtures;Stanyan String & Wind Ensemble;Classical;US;1977;;220924800;
18+
137251914;"I Love The Nightlife (Disco 'Round) (Real Rapino 7"" Mix)";The Adventures Of Priscilla: Queen Of The Desert - Original Motion Picture Soundtrack;Various;Stage & Screen;US;1994;3:31;757382400;3.31
19+
554983904;Walking On The Moon;Certifiable (Live In Buenos Aires);The Police;Rock;Malaysia;2008-11-00;;1225497600;
20+
557616002;Two Soldiers;Jerry Garcia / David Grisman;David Grisman;"Folk; World; & Country";US;2014-04-00;4:24;1396310400;4.24
21+
878936809;When You Gonna Learn;Live At Firenze 93;Jamiroquai;Funk / Soul;France;2004;13:01;1072915200;13.01
22+
368960707;Sardo D.O.C.;Sardinia Pride Compilation Vol.2;Various;Hip Hop;Italy;2012-06-22;4:41;1340323200;4.41
23+
1416041312;Sympathy For The Devil;Under Cover;Ozzy Osbourne;Rock;Russia;2005;7:11;1104537600;7.11
24+
1260509200;Grosse Overturen;noxious effects garanty;Nocif (3);Rock;France;1990;;631152000;
25+
1466381324;Πρινιώτης;Μουσικά Πατήματα Της Κρήτης;Αντώνης Μαρτσάκης;"Folk; World; & Country";Greece;2019;;1546300800;
26+
256009724;Here I Stand And Face The Rain (Demo);Hunting High And Low;a-ha;Electronic;UK & Europe;2010-07-23;;1279843200;
27+
565253008;Born Free;At His Best Goldfinger;The Royal Philharmonic Orchestra;Blues;Japan;1976;;189302400;
28+
492519701;Others;Where Did She Go;Stephan Sulke;Rock;US;1965;2:43;-157766400;2.43
29+
1268208902;Electric Fires;All That Divides;Black Peaks;Rock;UK;2018-10-05;3:22;1538697600;3.22
30+
1077074411;Your Summer Dream;Surfer Girl;The Beach Boys;Pop;Australia;2012-09-24;;1348444800;
31+
1275072979;Untitled;Die Legenden Der Albae: Tobender Sturm;Johannes Steck;Non-Music;Germany;2014;6:49;1388534400;6.49
32+
826446602;Salvaje;Introglicerina;Seguridad Social;Rock;France;1990;;631152000;
33+
180155008;Man;Bajzel;Bajzel;Rock;Poland;2007;2:37;1167609600;2.37
34+
949980604;Ebb Tide;21Channel Sound;David Rose & His Orchestra;Pop;US;1962;;-252460800;
35+
1241480200;Things;Things / Is There A Missing Piece;Donny Mann;Funk / Soul;Canada;1973;3:15;94694400;3.15
36+
38038907;Waving My Arms In The Air;Barrett;Syd Barrett;Rock;UK & Europe;2004;2:07;1072915200;2.07
37+
359975318;The Hand;The Lost Patrol. Virginia City;Max Steiner;Stage & Screen;Europe;1995;2:29;788918400;2.29
38+
810688602;We Don't Need Another Hero;Split Tape;Rottingrex;Rock;France;2014;;1388534400;
39+
1225622907;Our Place In The Sun;Unity Drum;Dissident (2);Rock;US;2018-07-13;;1531440000;
40+
406467204;Riders On The Storm (Album Edit);U.S. Dance Party - Vol. 4;Various;Electronic;US;1995;6:35;788918400;6.35
41+
158441000;Take Time To Know Her;Take Time To Know Her;Ninjaman;Reggae;US;1988;;567993600;
42+
992096712;Suddenly;Midnight Soul;Various;Funk / Soul;US;1997;3:51;852076800;3.51
43+
244054503;Straight From My Heart;Huang Chung;Wang Chung;Electronic;Italy;1982;2:48;378691200;2.48
44+
894880317;Черное Знамя (Live In Paris);Ва-Банкъ MP3;Ва-Банкъ;Rock;Russia;2005;3:48;1104537600;3.48
45+
767521501;L'Inganno;Eclissi;A Piedi Nudi;Rock;Russia;2006;4:56;1136073600;4.5600000000000005
46+
785246800;Got It Covered;Songs Of The Next Great Depression;Barry Melton;"Folk; World; & Country";Germany;1991;2:41;662688000;2.41
47+
499471800;Collateral;Collateral;Mickmag & Justbob;Electronic;Netherlands;2013-09-16;5:39;1379289600;5.39
48+
1251220502;Can't Wait;Rosebud e​.​p.;Poncho+Casino Tart;Funk / Soul;Japan;2016-10-09;3:24;1475971200;3.24
49+
360552304;Сюита В Фа-Миноре;Своя Игра;Арсенал;Rock;Russia;2007;;1167609600;
50+
1232342909;Had It All;Sunday Morning Record;The Band Of Heathens;"Folk; World; & Country";;2013-09-17;4:23;1379376000;4.23
51+
1254489900;"Hollywood Bowl; Los Angeles; CA; U.S.A. August 23; 1964";The Lost Hollywood Bowl Films;The Beatles;Non-Music;Japan;2018;;1514764800;
52+
1584961721;Man In My Life;Fear Of Flying;Mya;Pop;;;;;
53+
1190631208;Little Bit Of Nothing;All By Myself;Kevin Klimek;Rock;US;;2:57;;2.57
54+
1254307002;Big Jim Salter;Niagara;Stone The Crows;Rock;Luxembourg;1990;15:41;631152000;15.41

‎meilisearch/index.py

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,7 @@ def add_documents_csv(
434434
self,
435435
str_documents: str,
436436
primary_key: Optional[str] = None,
437+
csv_delimiter: Optional[str] = None,
437438
) -> TaskInfo:
438439
"""Add string documents from a CSV file to the index.
439440
@@ -443,6 +444,8 @@ def add_documents_csv(
443444
String of document from a CSV file.
444445
primary_key (optional):
445446
The primary-key used in index. Ignored if already set up.
447+
csv_delimiter:
448+
One ASCII character used to customize the delimiter for CSV.
446449
447450
Returns
448451
-------
@@ -455,7 +458,7 @@ def add_documents_csv(
455458
MeiliSearchApiError
456459
An error containing details about why Meilisearch can't process your request. Meilisearch error codes are described here: https://docs.meilisearch.com/errors/#meilisearch-errors
457460
"""
458-
return self.add_documents_raw(str_documents, primary_key, "text/csv")
461+
return self.add_documents_raw(str_documents, primary_key, "text/csv", csv_delimiter)
459462

460463
def add_documents_ndjson(
461464
self,
@@ -489,6 +492,7 @@ def add_documents_raw(
489492
str_documents: str,
490493
primary_key: Optional[str] = None,
491494
content_type: Optional[str] = None,
495+
csv_delimiter: Optional[str] = None,
492496
) -> TaskInfo:
493497
"""Add string documents to the index.
494498
@@ -499,7 +503,9 @@ def add_documents_raw(
499503
primary_key (optional):
500504
The primary-key used in index. Ignored if already set up.
501505
type:
502-
The type of document. Type available: 'csv', 'json', 'jsonl'
506+
The type of document. Type available: 'csv', 'json', 'jsonl'.
507+
csv_delimiter:
508+
One ASCII character used to customize the delimiter for CSV.
503509
504510
Returns
505511
-------
@@ -512,7 +518,14 @@ def add_documents_raw(
512518
MeiliSearchApiError
513519
An error containing details about why Meilisearch can't process your request. Meilisearch error codes are described here: https://docs.meilisearch.com/errors/#meilisearch-errors
514520
"""
515-
url = self._build_url(primary_key)
521+
parameters = {}
522+
if primary_key:
523+
parameters["primaryKey"] = primary_key
524+
if csv_delimiter:
525+
parameters["csvDelimiter"] = csv_delimiter
526+
url = self._build_url()
527+
if primary_key or csv_delimiter:
528+
url = url + f"?{parse.urlencode(parameters)}"
516529
response = self.http.post(url, str_documents, content_type)
517530
return TaskInfo(**response)
518531

@@ -601,6 +614,7 @@ def update_documents_csv(
601614
self,
602615
str_documents: str,
603616
primary_key: Optional[str] = None,
617+
csv_delimiter: Optional[str] = None,
604618
) -> TaskInfo:
605619
"""Update documents as a csv string in the index.
606620
@@ -609,7 +623,9 @@ def update_documents_csv(
609623
str_documents:
610624
String of document from a CSV file.
611625
primary_key (optional):
612-
The primary-key used in index. Ignored if already set up
626+
The primary-key used in index. Ignored if already set up.
627+
csv_delimiter:
628+
One ASCII character used to customize the delimiter for CSV.
613629
614630
Returns
615631
-------
@@ -622,13 +638,14 @@ def update_documents_csv(
622638
MeiliSearchApiError
623639
An error containing details about why Meilisearch can't process your request. Meilisearch error codes are described here: https://docs.meilisearch.com/errors/#meilisearch-errors
624640
"""
625-
return self.update_documents_raw(str_documents, primary_key, "text/csv")
641+
return self.update_documents_raw(str_documents, primary_key, "text/csv", csv_delimiter)
626642

627643
def update_documents_raw(
628644
self,
629645
str_documents: str,
630646
primary_key: Optional[str] = None,
631647
content_type: Optional[str] = None,
648+
csv_delimiter: Optional[str] = None,
632649
) -> TaskInfo:
633650
"""Update documents as a string in the index.
634651
@@ -640,6 +657,8 @@ def update_documents_raw(
640657
The primary-key used in index. Ignored if already set up.
641658
type:
642659
The type of document. Type available: 'csv', 'json', 'jsonl'
660+
csv_delimiter:
661+
One ASCII character used to customize the delimiter for CSV.
643662
644663
Returns
645664
-------
@@ -652,7 +671,14 @@ def update_documents_raw(
652671
MeiliSearchApiError
653672
An error containing details about why Meilisearch can't process your request. Meilisearch error codes are described here: https://docs.meilisearch.com/errors/#meilisearch-errors
654673
"""
655-
url = self._build_url(primary_key)
674+
parameters = {}
675+
if primary_key:
676+
parameters["primaryKey"] = primary_key
677+
if csv_delimiter:
678+
parameters["csvDelimiter"] = csv_delimiter
679+
url = self._build_url()
680+
if primary_key or csv_delimiter:
681+
url = url + f"?{parse.urlencode(parameters)}"
656682
response = self.http.put(url, str_documents, content_type)
657683
return TaskInfo(**response)
658684

‎tests/conftest.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,15 @@ def songs_csv():
7676
return song_csv_file.read().encode("utf-8")
7777

7878

79+
@fixture(scope="session")
80+
def songs_csv_delimiter():
81+
"""
82+
Runs once per session. Provides the content of songs_delimiter.csv from read..
83+
"""
84+
with open("./datasets/songs_delimiter.csv", encoding="utf-8") as song_csv_file:
85+
return song_csv_file.read().encode("utf-8")
86+
87+
7988
@fixture(scope="session")
8089
def songs_ndjson():
8190
"""

‎tests/index/test_index_document_meilisearch.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,16 @@ def test_add_documents_csv(empty_index, songs_csv):
197197
assert index.get_primary_key() == "id"
198198

199199

200+
def test_add_documents_csv_with_delimiter(empty_index, songs_csv_delimiter):
201+
"""Tests adding new documents to a clean index."""
202+
index = empty_index("csv-delimiter")
203+
response = index.add_documents_csv(songs_csv_delimiter, csv_delimiter=";")
204+
assert isinstance(response, TaskInfo)
205+
assert response.task_uid is not None
206+
task = index.wait_for_task(response.task_uid)
207+
assert task.status == "succeeded"
208+
209+
200210
def test_update_documents_csv(index_with_documents, songs_csv):
201211
"""Tests updating a single document with csv string."""
202212
index = index_with_documents()
@@ -208,6 +218,16 @@ def test_update_documents_csv(index_with_documents, songs_csv):
208218
assert index.get_primary_key() == "id"
209219

210220

221+
def test_update_documents_csv_with_delimiter(index_with_documents, songs_csv_delimiter):
222+
"""Tests adding new documents to a clean index."""
223+
index = index_with_documents("csv-delimiter")
224+
response = index.update_documents_csv(songs_csv_delimiter, csv_delimiter=";")
225+
assert isinstance(response, TaskInfo)
226+
assert response.task_uid is not None
227+
task = index.wait_for_task(response.task_uid)
228+
assert task.status == "succeeded"
229+
230+
211231
def test_add_documents_json(empty_index, small_movies_json_file):
212232
"""Tests adding new documents to a clean index."""
213233
index = empty_index()

‎tests/index/test_index_search_meilisearch.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33

44
def test_basic_search(index_with_documents):
5-
"""Tests search with an simple query."""
5+
"""Tests search with a simple query."""
66
response = index_with_documents().search("How to Train Your Dragon")
77
assert isinstance(response, dict)
88
assert response["hits"][0]["id"] == "166428"
@@ -356,7 +356,7 @@ def test_phrase_search(index_with_documents):
356356

357357

358358
def test_basic_search_on_nested_documents(index_with_documents, nested_movies):
359-
"""Tests search with an simple query on nested fields."""
359+
"""Tests search with a simple query on nested fields."""
360360
response = index_with_documents("nested_fields_index", nested_movies).search("An awesome")
361361
assert isinstance(response, dict)
362362
assert response["hits"][0]["id"] == 5

0 commit comments

Comments
 (0)