Skip to content

Commit 671cb51

Browse files
ikonstikonst
authored
Replace to_dict with to_simple_dict, to_dynamodb_dict (pynamodb#1126)
Since pynamodb#1112, `Model.serialize` return value is not necessarily JSON-serializable. We're adding `to_dynamodb_dict` as a replacement (basically, `bytes` are base-64 encoded), and `to_simple_dict` to provide for a common ask of a "simple" JSON representation similar to what the AWS Console defaults to (previously called `to_dict` in earlier versions of the 6.x branch). We're making those methods of `AttributeContainer` rather than `Model` so they'd be equally applicable to `MapAttribute`.
1 parent 34ebe2e commit 671cb51

File tree

9 files changed

+519
-266
lines changed

9 files changed

+519
-266
lines changed

‎docs/release_notes.rst‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ This is a major release and contains breaking changes. Please read the notes bel
1212
If your codebase uses :py:class:`~pynamodb.attributes.BinaryAttribute` or :py:class:`~pynamodb.attributes.BinarySetAttribute`,
1313
go over the attribute declarations and mark them accordingly.
1414
* When using binary attributes, the return value of :py:func:`~pynamodb.models.Model.serialize` will no longer be JSON-serializable
15-
since it will contain :code:`bytes` objects. Note that both `:py:func:`~pynamodb.models.Model.to_dict` and `:py:func:`~pynamodb.model.Model.to_json` are also affected.
15+
since it will contain :code:`bytes` objects. Use `:py:func:`~pynamodb.models.Model.to_dynamodb_dict`
16+
for a safe JSON-serializable representation.
1617

1718
* Python 3.6 is no longer supported.
1819
* Index count, query, and scan methods are now instance methods.

‎docs/tutorial.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,7 @@ Update actions use the update expression syntax (see :ref:`updates`).
295295

296296
.. deprecated:: 2.0
297297

298-
:func:`update_item` is replaced with :func:`update`
298+
:code:`update_item` is replaced with :func:`~pynamodb.models.Model.update`
299299

300300

301301
.. code-block:: python

‎pynamodb/_util.py‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
import json
2+
from base64 import b64decode
3+
from base64 import b64encode
4+
from typing import Any
5+
from typing import Dict
6+
7+
from pynamodb.constants import BINARY
8+
from pynamodb.constants import BINARY_SET
9+
from pynamodb.constants import BOOLEAN
10+
from pynamodb.constants import LIST
11+
from pynamodb.constants import MAP
12+
from pynamodb.constants import NULL
13+
from pynamodb.constants import NUMBER
14+
from pynamodb.constants import NUMBER_SET
15+
from pynamodb.constants import STRING
16+
from pynamodb.constants import STRING_SET
17+
18+
19+
def attr_value_to_simple_dict(attribute_value: Dict[str, Any], force: bool) -> Any:
20+
attr_type, attr_value = next(iter(attribute_value.items()))
21+
if attr_type == LIST:
22+
return [attr_value_to_simple_dict(v, force) for v in attr_value]
23+
if attr_type == MAP:
24+
return {k: attr_value_to_simple_dict(v, force) for k, v in attr_value.items()}
25+
if attr_type == NULL:
26+
return None
27+
if attr_type == BOOLEAN:
28+
return attr_value
29+
if attr_type == STRING:
30+
return attr_value
31+
if attr_type == NUMBER:
32+
return json.loads(attr_value)
33+
if attr_type == BINARY:
34+
if force:
35+
return b64encode(attr_value).decode()
36+
raise ValueError("Binary attributes are not supported")
37+
if attr_type == BINARY_SET:
38+
if force:
39+
return [b64encode(v).decode() for v in attr_value]
40+
raise ValueError("Binary set attributes are not supported")
41+
if attr_type == STRING_SET:
42+
if force:
43+
return attr_value
44+
raise ValueError("String set attributes are not supported")
45+
if attr_type == NUMBER_SET:
46+
if force:
47+
return [json.loads(v) for v in attr_value]
48+
raise ValueError("Number set attributes are not supported")
49+
raise ValueError("Unknown attribute type: {}".format(attr_type))
50+
51+
52+
def simple_dict_to_attr_value(value: Any) -> Dict[str, Any]:
53+
if value is None:
54+
return {NULL: True}
55+
if value is True or value is False:
56+
return {BOOLEAN: value}
57+
if isinstance(value, (int, float)):
58+
return {NUMBER: json.dumps(value)}
59+
if isinstance(value, str):
60+
return {STRING: value}
61+
if isinstance(value, list):
62+
return {LIST: [simple_dict_to_attr_value(v) for v in value]}
63+
if isinstance(value, dict):
64+
return {MAP: {k: simple_dict_to_attr_value(v) for k, v in value.items()}}
65+
raise ValueError("Unknown value type: {}".format(type(value).__name__))
66+
67+
68+
def _b64encode(b: bytes) -> str:
69+
return b64encode(b).decode()
70+
71+
72+
def bin_encode_attr(attr: Dict[str, Any]) -> None:
73+
if BINARY in attr:
74+
attr[BINARY] = _b64encode(attr[BINARY])
75+
elif BINARY_SET in attr:
76+
attr[BINARY_SET] = [_b64encode(v) for v in attr[BINARY_SET]]
77+
elif MAP in attr:
78+
for sub_attr in attr[MAP].values():
79+
bin_encode_attr(sub_attr)
80+
elif LIST in attr:
81+
for sub_attr in attr[LIST]:
82+
bin_encode_attr(sub_attr)
83+
84+
85+
def bin_decode_attr(attr: Dict[str, Any]) -> None:
86+
if BINARY in attr:
87+
attr[BINARY] = b64decode(attr[BINARY])
88+
elif BINARY_SET in attr:
89+
attr[BINARY_SET] = [b64decode(v) for v in attr[BINARY_SET]]
90+
elif MAP in attr:
91+
for sub_attr in attr[MAP].values():
92+
bin_decode_attr(sub_attr)
93+
elif LIST in attr:
94+
for sub_attr in attr[LIST]:
95+
bin_decode_attr(sub_attr)

‎pynamodb/attributes.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@
1717
from typing import Any, Callable, Dict, Generic, List, Mapping, Optional, TypeVar, Type, Union, Set, overload, Iterable
1818
from typing import TYPE_CHECKING
1919

20+
from pynamodb._util import attr_value_to_simple_dict
21+
from pynamodb._util import bin_decode_attr
22+
from pynamodb._util import bin_encode_attr
23+
from pynamodb._util import simple_dict_to_attr_value
2024
from pynamodb.constants import BINARY
2125
from pynamodb.constants import BINARY_SET
2226
from pynamodb.constants import BOOLEAN
@@ -314,6 +318,9 @@ def _initialize_attributes(cls, discriminator_value):
314318

315319

316320
class AttributeContainer(metaclass=AttributeContainerMeta):
321+
"""
322+
Base class for models and maps.
323+
"""
317324

318325
def __init__(self, _user_instantiated: bool = True, **attributes: Attribute) -> None:
319326
# The `attribute_values` dictionary is used by the Attribute data descriptors in cls._attributes
@@ -478,6 +485,71 @@ def _instantiate(cls: Type[_ACT], attribute_values: Dict[str, Dict[str, Any]]) -
478485
AttributeContainer._container_deserialize(instance, attribute_values)
479486
return instance
480487

488+
def to_dynamodb_dict(self) -> Dict[str, Dict[str, Any]]:
489+
"""
490+
Returns the contents of this instance as a JSON-serializable mapping,
491+
where each attribute is represented as a mapping with the attribute
492+
type as the key and the attribute value as the value, e.g.
493+
494+
.. code-block:: python
495+
496+
{
497+
"id": {
498+
"N": "12345"
499+
},
500+
"name": {
501+
"S": "Alice"
502+
},
503+
}
504+
505+
This matches the structure of the "DynamoDB" JSON mapping in the AWS Console.
506+
"""
507+
attr_values = self._container_serialize(null_check=False)
508+
for v in attr_values.values():
509+
bin_encode_attr(v)
510+
return attr_values
511+
512+
def from_dynamodb_dict(self, d: Dict[str, Dict[str, Any]]) -> None:
513+
"""
514+
Sets attributes from a mapping previously produced by :func:`to_dynamodb_dict`.
515+
"""
516+
for v in d.values():
517+
bin_decode_attr(v)
518+
self._update_attribute_types(d)
519+
self._container_deserialize(d)
520+
521+
def to_simple_dict(self, *, force: bool = False) -> Dict[str, Any]:
522+
"""
523+
Returns the contents of this instance as a simple JSON-serializable mapping.
524+
525+
.. code-block:: python
526+
527+
{
528+
"id": 12345,
529+
"name": "Alice",
530+
}
531+
532+
This matches the structure of the "normal" JSON mapping in the AWS Console.
533+
534+
.. note::
535+
536+
This representation is limited: by default, it cannot represent binary or set attributes,
537+
as their encoded form is indistinguishable from a string or list attribute respectively
538+
(and therefore ambiguous).
539+
540+
:param force: If :code:`True`, force the conversion even if the model contains Binary or Set attributes
541+
If :code:`False`, a :code:`ValueError` will be raised if such attributes are set.
542+
"""
543+
return {k: attr_value_to_simple_dict(v, force) for k, v in self._container_serialize(null_check=False).items()}
544+
545+
def from_simple_dict(self, d: Dict[str, Any]) -> None:
546+
"""
547+
Sets attributes from a mapping previously produced by :func:`to_simple_dict`.
548+
"""
549+
attribute_values = {k: simple_dict_to_attr_value(v) for k, v in d.items()}
550+
self._update_attribute_types(attribute_values)
551+
self._container_deserialize(attribute_values)
552+
481553
def __repr__(self) -> str:
482554
fields = ', '.join(f'{k}={v!r}' for k, v in self.attribute_values.items())
483555
return f'{type(self).__name__}({fields})'

‎pynamodb/connection/base.py‎

Lines changed: 14 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import sys
99
import time
1010
import uuid
11-
from base64 import b64decode
1211
from threading import local
1312
from typing import Any, Dict, List, Mapping, Optional, Sequence, cast
1413

@@ -21,8 +20,7 @@
2120
from botocore.session import get_session
2221

2322
from pynamodb.connection._botocore_private import BotocoreBaseClientPrivate
24-
from pynamodb.constants import LIST
25-
from pynamodb.constants import MAP
23+
from pynamodb._util import bin_decode_attr
2624
from pynamodb.constants import (
2725
RETURN_CONSUMED_CAPACITY_VALUES, RETURN_ITEM_COLL_METRICS_VALUES,
2826
RETURN_ITEM_COLL_METRICS, RETURN_CONSUMED_CAPACITY, RETURN_VALUES_VALUES,
@@ -36,7 +34,7 @@
3634
WRITE_CAPACITY_UNITS, GLOBAL_SECONDARY_INDEXES, PROJECTION, EXCLUSIVE_START_TABLE_NAME, TOTAL,
3735
DELETE_TABLE, UPDATE_TABLE, LIST_TABLES, GLOBAL_SECONDARY_INDEX_UPDATES, ATTRIBUTES,
3836
CONSUMED_CAPACITY, CAPACITY_UNITS, ATTRIBUTE_TYPES,
39-
ITEMS, BINARY, BINARY_SET, LAST_EVALUATED_KEY, RESPONSES, UNPROCESSED_KEYS,
37+
ITEMS, LAST_EVALUATED_KEY, RESPONSES, UNPROCESSED_KEYS,
4038
UNPROCESSED_ITEMS, STREAM_SPECIFICATION, STREAM_VIEW_TYPE, STREAM_ENABLED,
4139
EXPRESSION_ATTRIBUTE_NAMES, EXPRESSION_ATTRIBUTE_VALUES,
4240
CONDITION_EXPRESSION, FILTER_EXPRESSION,
@@ -503,39 +501,39 @@ def _handle_binary_attributes(data):
503501
""" Simulate botocore's binary attribute handling """
504502
if ITEM in data:
505503
for attr in data[ITEM].values():
506-
_convert_binary(attr)
504+
bin_decode_attr(attr)
507505
if ITEMS in data:
508506
for item in data[ITEMS]:
509507
for attr in item.values():
510-
_convert_binary(attr)
508+
bin_decode_attr(attr)
511509
if RESPONSES in data:
512-
if isinstance(data[RESPONSES], list):
510+
if isinstance(data[RESPONSES], list): # ExecuteTransaction response
513511
for item in data[RESPONSES]:
514512
for attr in item.values():
515-
_convert_binary(attr)
516-
else:
517-
for item_list in data[RESPONSES].values():
518-
for item in item_list:
513+
bin_decode_attr(attr)
514+
else: # BatchGetItem response
515+
for table_items in data[RESPONSES].values():
516+
for item in table_items:
519517
for attr in item.values():
520-
_convert_binary(attr)
518+
bin_decode_attr(attr)
521519
if LAST_EVALUATED_KEY in data:
522520
for attr in data[LAST_EVALUATED_KEY].values():
523-
_convert_binary(attr)
521+
bin_decode_attr(attr)
524522
if UNPROCESSED_KEYS in data:
525523
for table_data in data[UNPROCESSED_KEYS].values():
526524
for item in table_data[KEYS]:
527525
for attr in item.values():
528-
_convert_binary(attr)
526+
bin_decode_attr(attr)
529527
if UNPROCESSED_ITEMS in data:
530528
for table_unprocessed_requests in data[UNPROCESSED_ITEMS].values():
531529
for request in table_unprocessed_requests:
532530
for item_mapping in request.values():
533531
for item in item_mapping.values():
534532
for attr in item.values():
535-
_convert_binary(attr)
533+
bin_decode_attr(attr)
536534
if ATTRIBUTES in data:
537535
for attr in data[ATTRIBUTES].values():
538-
_convert_binary(attr)
536+
bin_decode_attr(attr)
539537
return data
540538

541539
@property
@@ -1384,16 +1382,3 @@ def _check_condition(self, name, condition):
13841382
@staticmethod
13851383
def _reverse_dict(d):
13861384
return {v: k for k, v in d.items()}
1387-
1388-
1389-
def _convert_binary(attr: Dict[str, Any]) -> None:
1390-
if BINARY in attr:
1391-
attr[BINARY] = b64decode(attr[BINARY].encode())
1392-
elif BINARY_SET in attr:
1393-
attr[BINARY_SET] = [b64decode(v.encode()) for v in attr[BINARY_SET]]
1394-
elif MAP in attr:
1395-
for sub_attr in attr[MAP].values():
1396-
_convert_binary(sub_attr)
1397-
elif LIST in attr:
1398-
for sub_attr in attr[LIST]:
1399-
_convert_binary(sub_attr)

‎pynamodb/models.py‎

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
"""
22
DynamoDB Models for PynamoDB
33
"""
4-
import json
54
import random
65
import time
76
import logging
@@ -58,8 +57,6 @@
5857
META_CLASS_NAME, REGION, HOST, NULL,
5958
COUNT, ITEM_COUNT, KEY, UNPROCESSED_ITEMS,
6059
)
61-
from pynamodb.util import attribute_value_to_json
62-
from pynamodb.util import json_to_attribute_value
6360

6461
_T = TypeVar('_T', bound='Model')
6562
_KeyType = Any
@@ -289,7 +286,7 @@ class Model(AttributeContainer, metaclass=MetaModel):
289286
Defines a `PynamoDB` Model
290287
291288
This model is backed by a table in DynamoDB.
292-
You can create the table by with the ``create_table`` method.
289+
You can create the table with the ``create_table`` method.
293290
"""
294291

295292
# These attributes are named to avoid colliding with user defined
@@ -1121,52 +1118,18 @@ def serialize(self, null_check: bool = True) -> Dict[str, Dict[str, Any]]:
11211118
.. warning::
11221119
BINARY and BINARY_SET attributes (whether top-level or nested) serialization would contain
11231120
:code:`bytes` objects which are not JSON-serializable by the :code:`json` module.
1124-
You may use a custom JSON encoder to serialize such models.
11251121
1126-
See :func:`~pynamodb.models.Model.to_dict` for a simple JSON-serializable dict.
1122+
Use :meth:`~pynamodb.attributes.AttributeContainer.to_dynamodb_dict`
1123+
and :meth:`~pynamodb.attributes.AttributeContainer.to_simple_dict` for JSON-serializable mappings.
11271124
"""
11281125
return self._container_serialize(null_check=null_check)
11291126

11301127
def deserialize(self, attribute_values: Dict[str, Dict[str, Any]]) -> None:
11311128
"""
11321129
Deserializes a model from botocore's DynamoDB client.
1133-
1134-
Use :func:`~pynamodb.models.Model.from_dict` to set attributes from a dict
1135-
previously produced by :func:`~pynamodb.models.Model.to_dict`.
11361130
"""
11371131
return self._container_deserialize(attribute_values=attribute_values)
11381132

1139-
def to_dict(self) -> Dict[str, Any]:
1140-
"""
1141-
Returns the contents of this instance as a JSON-serializable dict.
1142-
See :func:`~pynamodb.models.Model.serialize` if you need to serialize
1143-
into a DynamoDB record.
1144-
"""
1145-
return {k: attribute_value_to_json(v) for k, v in self.serialize().items()}
1146-
1147-
def to_json(self) -> str:
1148-
"""
1149-
Returns the contents of this instance as serialized JSON (not in DynamoDB Record format).
1150-
"""
1151-
return json.dumps(self.to_dict())
1152-
1153-
def from_dict(self, d: Dict[str, Any]) -> None:
1154-
"""
1155-
Sets attributes from a dict previously produced by :func:`~pynamodb.models.Model.to_dict`.
1156-
Use :func:`~pynamodb.models.Model.deserialize` if the dict is a DynamoDB Record
1157-
(e.g. from a DynamoDB API response or DynamoDB Streams).
1158-
"""
1159-
attribute_values = {
1160-
k: json_to_attribute_value(v) for k, v in d.items()}
1161-
self._update_attribute_types(attribute_values)
1162-
self.deserialize(attribute_values)
1163-
1164-
def from_json(self, s: str) -> None:
1165-
"""
1166-
Sets attributes from a dict previously produced by :func:`~pynamodb.models.Model.to_json`.
1167-
"""
1168-
self.from_dict(json.loads(s))
1169-
11701133

11711134
class _ModelFuture(Generic[_T]):
11721135
"""

0 commit comments

Comments
 (0)