Skip to content

feat: i/o for DocVec #1562

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JohannesMessner merged 26 commits into from
Jun 28, 2023
Merged

feat: i/o for DocVec #1562

Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
d43057b
feat: json and dict for docvec
JohannesMessner May 22, 2023
c45bfca
test: add tests
JohannesMessner May 22, 2023
564d144
test: add docvec to dict test
JohannesMessner May 22, 2023
76f9c8e
feat: to from dataframe for docvec
JohannesMessner May 22, 2023
73a1ac7
test: dataframe docvec tests
JohannesMessner May 22, 2023
f83fb4f
feat: to from csv for docvec
JohannesMessner May 22, 2023
ca8dc12
test: test csv with docvec
JohannesMessner May 22, 2023
2b52b1e
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 14, 2023
b115637
feat: pickle serialization for docvec
JohannesMessner Jun 14, 2023
bd86985
feat: protbuf array serialization for docvec
JohannesMessner Jun 14, 2023
c280ff2
test: test base64 deser for docvec
JohannesMessner Jun 14, 2023
ad881cf
test: test save and load for docvec
JohannesMessner Jun 14, 2023
4b1b533
feat: docvec json column wise
JohannesMessner Jun 19, 2023
60e651e
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 19, 2023
f9c97ec
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 20, 2023
0603fc5
test: add test for docvec json
JohannesMessner Jun 20, 2023
c6ace8e
test: add tensor type arg
JohannesMessner Jun 20, 2023
51719b2
fix: mypy stuff
JohannesMessner Jun 26, 2023
ad5f5bd
fix: raising of error when needed
JohannesMessner Jun 26, 2023
200dbac
fix: more exception raising
JohannesMessner Jun 26, 2023
8d1f446
fix: mypy
JohannesMessner Jun 26, 2023
6815720
refactor: don't expose to/from csv for docvec
JohannesMessner Jun 26, 2023
6b5ddc7
test: adjust tests
JohannesMessner Jun 26, 2023
587c20a
docs: add documentation for docvec io
JohannesMessner Jun 27, 2023
663f17d
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 27, 2023
7d035fb
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: test save and load for docvec 
Signed-off-by: Johannes Messner <[email protected]>
  • Loading branch information
@JohannesMessner
JohannesMessner committed Jun 14, 2023
commit ad881cf60f52992c1cb0af97b6a9c65bd773565d
25 changes: 15 additions & 10 deletions tests/units/array/test_array_save_load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import numpy as np
import pytest

from docarray import BaseDoc, DocList
from docarray import BaseDoc, DocList, DocVec
from docarray.documents import ImageDoc
from docarray.typing import NdArray

Expand All @@ -20,10 +20,11 @@ class MyDoc(BaseDoc):
)
@pytest.mark.parametrize('compress', ['lz4', 'bz2', 'lzma', 'zlib', 'gzip', None])
@pytest.mark.parametrize('show_progress', [False, True])
def test_array_save_load_binary(protocol, compress, tmp_path, show_progress):
@pytest.mark.parametrize('array_cls', [DocList, DocVec])
def test_array_save_load_binary(protocol, compress, tmp_path, show_progress, array_cls):
tmp_file = os.path.join(tmp_path, 'test')

da = DocList[MyDoc](
da = array_cls[MyDoc](
[
MyDoc(
embedding=[1, 2, 3, 4, 5], text='hello', image=ImageDoc(url='aux.png')
Expand All @@ -36,7 +37,7 @@ def test_array_save_load_binary(protocol, compress, tmp_path, show_progress):
tmp_file, protocol=protocol, compress=compress, show_progress=show_progress
)

da2 = DocList[MyDoc].load_binary(
da2 = array_cls[MyDoc].load_binary(
tmp_file, protocol=protocol, compress=compress, show_progress=show_progress
)

Expand All @@ -56,8 +57,12 @@ def test_array_save_load_binary(protocol, compress, tmp_path, show_progress):
)
@pytest.mark.parametrize('compress', ['lz4', 'bz2', 'lzma', 'zlib', 'gzip', None])
@pytest.mark.parametrize('show_progress', [False, True])
def test_array_save_load_binary_streaming(protocol, compress, tmp_path, show_progress):
@pytest.mark.parametrize('to_doc_vec', [True, False])
def test_array_save_load_binary_streaming(
protocol, compress, tmp_path, show_progress, to_doc_vec
):
tmp_file = os.path.join(tmp_path, 'test')
array_cls = DocVec if to_doc_vec else DocList

da = DocList[MyDoc]()

Expand All @@ -74,20 +79,20 @@ def _extend_da(num_docs=100):
)

_extend_da()
if to_doc_vec:
da = da.to_doc_vec()

da.save_binary(
tmp_file, protocol=protocol, compress=compress, show_progress=show_progress
)

da2 = DocList[MyDoc]()
da_generator = DocList[MyDoc].load_binary(
da_after = array_cls[MyDoc].load_binary(
tmp_file, protocol=protocol, compress=compress, show_progress=show_progress
)

for i, doc in enumerate(da_generator):
for i, doc in enumerate(da_after):
assert doc.id == da[i].id
assert doc.text == da[i].text
assert doc.image.url == da[i].image.url
da2.append(doc)

assert len(da2) == 100
assert i == 99