new: convert nan and inf in local mode to none (#603)

joein · joein · commit 93a2c2da2f84 · 2024-04-22T12:30:14.000+02:00
* tests: update nan payload tests with pydantic 2.7

* fix: update local mode to mimic pydantic&gt;=2.7.0

* fix: fix mypy

* fix: do not convert bytes

* fix: allow nan values in payload in local mode
diff --git a/qdrant_client/local/local_collection.py b/qdrant_client/local/local_collection.py
@@ -1,7 +1,18 @@
 import json
 import uuid
 from collections import OrderedDict, defaultdict
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union, get_args
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    get_args,
+)
 
 import numpy as np
 from pydantic.version import VERSION as PYDANTIC_VERSION
@@ -67,10 +78,10 @@ def _to_jsonable_python(x: Any) -> Any:
 
 def to_jsonable_python(x: Any) -> Any:
     try:
-        json.dumps(x, allow_nan=False)
+        json.dumps(x, allow_nan=True)
         return x
     except Exception:
-        return json.loads(json.dumps(x, allow_nan=False, default=_to_jsonable_python))
+        return json.loads(json.dumps(x, allow_nan=True, default=_to_jsonable_python))
 
 
 class LocalCollection:
diff --git a/tests/congruence_tests/test_updates.py b/tests/congruence_tests/test_updates.py
@@ -258,61 +258,6 @@ def test_upload_collection_dict_np_arrays(local_client, remote_client):
     compare_collections(local_client, remote_client, UPLOAD_NUM_VECTORS)
 
 
-def test_upload_payload_contain_nan_values():
-    # usual case when payload is extracted from pandas dataframe
-
-    local_client = init_local()
-    remote_client = init_remote()
-
-    vector_size = 2
-    nans_collection = "nans_collection"
-    local_client.recreate_collection(
-        collection_name=nans_collection,
-        vectors_config=models.VectorParams(size=vector_size, distance=models.Distance.DOT),
-    )
-    remote_client.recreate_collection(
-        collection_name=nans_collection,
-        vectors_config=models.VectorParams(size=vector_size, distance=models.Distance.DOT),
-    )
-    points = generate_points(
-        num_points=UPLOAD_NUM_VECTORS,
-        vector_sizes=2,
-        with_payload=False,
-    )
-    ids, vectors, payload = [], [], []
-    for i in range(len(points)):
-        points[i].payload = {"surprise": math.nan}
-
-    for point in points:
-        ids.append(point.id)
-        vectors.append(point.vector)
-        payload.append(point.payload)
-
-    with pytest.raises(ValueError):
-        local_client.upload_collection(nans_collection, vectors, payload)
-    with pytest.raises(qdrant_client.http.exceptions.UnexpectedResponse):
-        remote_client.upload_collection(nans_collection, vectors, payload)
-
-    with pytest.raises(ValueError):
-        local_client.upload_points(nans_collection, points)
-    with pytest.raises(qdrant_client.http.exceptions.UnexpectedResponse):
-        remote_client.upload_points(nans_collection, points)
-
-    points_batch = models.Batch(
-        ids=ids,
-        vectors=vectors,
-        payloads=payload,
-    )
-
-    with pytest.raises(ValueError):
-        local_client.upsert(nans_collection, points=points_batch)
-    with pytest.raises(qdrant_client.http.exceptions.UnexpectedResponse):
-        remote_client.upsert(nans_collection, points=points_batch)
-
-    local_client.delete_collection(nans_collection)
-    remote_client.delete_collection(nans_collection)
-
-
 def test_upload_wrong_vectors():
     local_client = init_local()
     remote_client = init_remote()
