Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
d43057b
feat: json and dict for docvec
JohannesMessner May 22, 2023
c45bfca
test: add tests
JohannesMessner May 22, 2023
564d144
test: add docvec to dict test
JohannesMessner May 22, 2023
76f9c8e
feat: to from dataframe for docvec
JohannesMessner May 22, 2023
73a1ac7
test: dataframe docvec tests
JohannesMessner May 22, 2023
f83fb4f
feat: to from csv for docvec
JohannesMessner May 22, 2023
ca8dc12
test: test csv with docvec
JohannesMessner May 22, 2023
2b52b1e
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 14, 2023
b115637
feat: pickle serialization for docvec
JohannesMessner Jun 14, 2023
bd86985
feat: protbuf array serialization for docvec
JohannesMessner Jun 14, 2023
c280ff2
test: test base64 deser for docvec
JohannesMessner Jun 14, 2023
ad881cf
test: test save and load for docvec
JohannesMessner Jun 14, 2023
4b1b533
feat: docvec json column wise
JohannesMessner Jun 19, 2023
60e651e
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 19, 2023
f9c97ec
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 20, 2023
0603fc5
test: add test for docvec json
JohannesMessner Jun 20, 2023
c6ace8e
test: add tensor type arg
JohannesMessner Jun 20, 2023
51719b2
fix: mypy stuff
JohannesMessner Jun 26, 2023
ad5f5bd
fix: raising of error when needed
JohannesMessner Jun 26, 2023
200dbac
fix: more exception raising
JohannesMessner Jun 26, 2023
8d1f446
fix: mypy
JohannesMessner Jun 26, 2023
6815720
refactor: don't expose to/from csv for docvec
JohannesMessner Jun 26, 2023
6b5ddc7
test: adjust tests
JohannesMessner Jun 26, 2023
587c20a
docs: add documentation for docvec io
JohannesMessner Jun 27, 2023
663f17d
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 27, 2023
7d035fb
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 28, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: adjust tests
Signed-off-by: Johannes Messner <[email protected]>
  • Loading branch information
JohannesMessner committed Jun 26, 2023
commit 6b5ddc76d4f45a686469c6ed02be63b344ce024b
2 changes: 1 addition & 1 deletion docarray/array/doc_vec/doc_vec.py
Original file line number Diff line number Diff line change
Expand Up @@ -900,7 +900,7 @@ def from_csv(
cls: Type['T'],
file_path: str,
encoding: str = 'utf-8',
dialect: Union[str, csv.Dialect] = 'excel',
dialect: Union[str, 'csv.Dialect'] = 'excel',
) -> 'T':
"""
DocVec does not support `.from_csv()`. This is because CSV is a row-based format
Expand Down
52 changes: 31 additions & 21 deletions tests/units/array/test_array_from_to_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,8 @@ class MyDocNested(MyDoc):
return MyDocNested


@pytest.mark.parametrize('array_cls', [DocList, DocVec])
def test_to_from_csv(tmpdir, nested_doc_cls, array_cls):
da = array_cls[nested_doc_cls](
def test_to_from_csv(tmpdir, nested_doc_cls):
da = DocList[nested_doc_cls](
[
nested_doc_cls(
count=0,
Expand All @@ -38,18 +37,17 @@ def test_to_from_csv(tmpdir, nested_doc_cls, array_cls):
da.to_csv(tmp_file)
assert os.path.isfile(tmp_file)

da_from = array_cls[nested_doc_cls].from_csv(tmp_file)
assert isinstance(da_from, array_cls)
da_from = DocList[nested_doc_cls].from_csv(tmp_file)
assert isinstance(da_from, DocList)
for doc1, doc2 in zip(da, da_from):
assert doc1 == doc2


@pytest.mark.parametrize('array_cls', [DocList, DocVec])
def test_from_csv_nested(nested_doc_cls, array_cls):
da = array_cls[nested_doc_cls].from_csv(
def test_from_csv_nested(nested_doc_cls):
da = DocList[nested_doc_cls].from_csv(
file_path=str(TOYDATA_DIR / 'docs_nested.csv')
)
assert isinstance(da, array_cls)
assert isinstance(da, DocList)
assert len(da) == 3

for i, doc in enumerate(da):
Expand Down Expand Up @@ -93,31 +91,26 @@ class Outer(BaseDoc):
return doc


@pytest.mark.parametrize('array_cls', [DocList, DocVec])
def test_from_csv_without_schema_raise_exception(array_cls):
def test_from_csv_without_schema_raise_exception():
with pytest.raises(TypeError, match='no document schema defined'):
array_cls.from_csv(file_path=str(TOYDATA_DIR / 'docs_nested.csv'))
DocList.from_csv(file_path=str(TOYDATA_DIR / 'docs_nested.csv'))


@pytest.mark.parametrize('array_cls', [DocList, DocVec])
def test_from_csv_with_wrong_schema_raise_exception(nested_doc, array_cls):
def test_from_csv_with_wrong_schema_raise_exception(nested_doc):
with pytest.raises(ValueError, match='Column names do not match the schema'):
array_cls[nested_doc.__class__].from_csv(
file_path=str(TOYDATA_DIR / 'docs.csv')
)
DocList[nested_doc.__class__].from_csv(file_path=str(TOYDATA_DIR / 'docs.csv'))


@pytest.mark.parametrize('array_cls', [DocList, DocVec])
def test_from_remote_csv_file(array_cls):
def test_from_remote_csv_file():
remote_url = 'https://github.com/docarray/docarray/blob/main/tests/toydata/books.csv?raw=true'

class Book(BaseDoc):
title: str
author: str
year: int

books = array_cls[Book].from_csv(file_path=remote_url)
assert isinstance(books, array_cls)
books = DocList[Book].from_csv(file_path=remote_url)
assert isinstance(books, DocList)

assert len(books) == 3

Expand Down Expand Up @@ -154,3 +147,20 @@ class BasisUnion(BaseDoc):
docs_basic.to_csv(str(tmp_path) + ".csv")
docs_copy = DocList[BasisUnion].from_csv(str(tmp_path) + ".csv")
assert docs_copy == docs_basic


def test_to_from_csv_docvec_raises():
class Book(BaseDoc):
title: str
author: str
year: int

books = DocVec[Book](
[Book(title='It\'s me, hi', author='I\'m the problem it\'s me', year=2022)]
)

with pytest.raises(NotImplementedError):
books.to_csv('dummy/file/path')

with pytest.raises(NotImplementedError):
DocVec[Book].from_csv('dummy/file/path')