Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
3d45f57
feat: add apply function
Feb 28, 2023
9a07808
test: add benchmark tests
Feb 28, 2023
5709f97
fix: apply
Feb 28, 2023
366b6df
fix: benchmark test
Mar 1, 2023
6e8ff7c
test: benchmark
Mar 2, 2023
89eaf62
fix: apply
Mar 2, 2023
545d9ec
fix: clean up
Mar 2, 2023
d871363
chore: remove benchmark tests from general tests
Mar 2, 2023
8a436ae
chore: fix ci
Mar 2, 2023
c581b58
feat: add threading option and benchmark test
Mar 2, 2023
8bf57fb
test: use both backend options in tests
Mar 2, 2023
0c3524c
feat: add batching to abstract array
Mar 2, 2023
c335895
feat: add apply_batch and _map_batch and tests
Mar 2, 2023
cc23e4e
test: fix load from da
Mar 2, 2023
73c0d84
docs: update docstrings
Mar 2, 2023
b7c2cae
docs: add example for apply
Mar 2, 2023
3eb0c30
fix: mypy
Mar 2, 2023
7c6cb2f
refactor: clean up
Mar 2, 2023
afa5837
refactor: make batch method private
Mar 2, 2023
c69585a
fix: apply
Mar 2, 2023
8a3437a
Test: add for apply batch
Mar 2, 2023
66b78b3
fix: benchmark test increase ndocs
Mar 3, 2023
35e090a
test: clean up
Mar 3, 2023
3019522
test: try to fix
Mar 3, 2023
313d318
test: try to fix test
Mar 3, 2023
0afd5bd
fix: test
Mar 3, 2023
fdcfa23
fix: test
Mar 3, 2023
fc91dbf
fix: apply suggestions from code review
Mar 3, 2023
0d7cd1b
fix: remove print statemetns
Mar 3, 2023
b4c672b
fix: apply samis suggestion
Mar 3, 2023
18a377b
fix: add tests for func da to doc and da to other len da
Mar 3, 2023
245283f
fix: revert last commit
Mar 3, 2023
76fe8b7
test: add len assert
Mar 3, 2023
34b7f9c
test: add assertions
Mar 3, 2023
c7a968d
test: add test to for da extend in batch apply
Mar 3, 2023
6cf8ed2
test: extend with only one doc
Mar 3, 2023
5dc9e6d
test: fix
Mar 3, 2023
d3fc203
fix: test
Mar 3, 2023
45cdc4a
fix: test
Mar 3, 2023
9839602
fix: set docs in apply
Mar 3, 2023
87a93ff
fix: indices
Mar 3, 2023
eeb7fae
fix: indices
Mar 3, 2023
72aaf21
fix: indices
Mar 3, 2023
c0f8029
fix: indices
Mar 3, 2023
9b83c1f
fix:test
Mar 3, 2023
7638d86
fix: mypy
Mar 3, 2023
4a3a290
fix: type hint
Mar 3, 2023
38aae7a
fix: remove apply, only keep map
Mar 3, 2023
01900c9
refactor: map to map_docs
Mar 3, 2023
f6921e0
fix: apply suggestion
Mar 3, 2023
c3fb041
docs: add example usage
Mar 3, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: apply suggestion
Signed-off-by: anna-charlotte <[email protected]>
  • Loading branch information
anna-charlotte committed Mar 3, 2023
commit f6921e07b620ba0f69c8b500e51ce4fafacea45a
2 changes: 1 addition & 1 deletion docarray/utils/map.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def map_docs(
yield x


def map_batch(
def map_docs_batch(
da: T,
func: Callable[[T], Union[T, T_doc]],
batch_size: int,
Expand Down
6 changes: 3 additions & 3 deletions tests/benchmark_tests/test_map.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from docarray import BaseDocument, DocumentArray
from docarray.documents import Image
from docarray.typing import NdArray
from docarray.utils.map import map_batch, map_docs
from docarray.utils.map import map_docs, map_docs_batch
from tests.units.typing.test_bytes import IMAGE_PATHS

pytestmark = [pytest.mark.benchmark, pytest.mark.slow]
Expand Down Expand Up @@ -66,7 +66,7 @@ def time_multiprocessing(num_workers: int) -> float:
da = DocumentArray[MyMatrix]([MyMatrix(matrix=m) for m in matrices])
start_time = time()
list(
map_batch(
map_docs_batch(
da=da,
func=cpu_intensive_batch,
batch_size=8,
Expand Down Expand Up @@ -121,7 +121,7 @@ def time_multithreading_batch(num_workers: int) -> float:
)
start_time = time()
list(
map_batch(
map_docs_batch(
da=da,
func=io_intensive_batch,
backend='thread',
Expand Down
8 changes: 5 additions & 3 deletions tests/units/util/test_map.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from docarray import BaseDocument, DocumentArray
from docarray.documents import Image
from docarray.typing import ImageUrl, NdArray
from docarray.utils.map import map_batch, map_docs
from docarray.utils.map import map_docs, map_docs_batch
from tests.units.typing.test_bytes import IMAGE_PATHS

N_DOCS = 2
Expand Down Expand Up @@ -73,12 +73,14 @@ class MyImage(BaseDocument):
@pytest.mark.slow
@pytest.mark.parametrize('n_docs,batch_size', [(10, 5), (10, 8)])
@pytest.mark.parametrize('backend', ['thread', 'process'])
def test_map_batch(n_docs, batch_size, backend):
def test_map_docs_batch(n_docs, batch_size, backend):

da = DocumentArray[MyImage](
[MyImage(url=IMAGE_PATHS['png']) for _ in range(n_docs)]
)
it = map_batch(da=da, func=load_from_da, batch_size=batch_size, backend=backend)
it = map_docs_batch(
da=da, func=load_from_da, batch_size=batch_size, backend=backend
)
assert isinstance(it, Generator)

for batch in it:
Expand Down