From 64811fdc26811022385f5c53053cb3d816e19673 Mon Sep 17 00:00:00 2001 From: peter Date: Mon, 21 Aug 2023 17:49:57 -0400 Subject: [PATCH 1/3] don't convert (unindexed) ascii str to bytes --- src/google/appengine/ext/ndb/model.py | 5 ++-- .../appengine/ext/ndb/polymodel_test.py | 25 +++++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/google/appengine/ext/ndb/model.py b/src/google/appengine/ext/ndb/model.py index 4671662..541643b 100755 --- a/src/google/appengine/ext/ndb/model.py +++ b/src/google/appengine/ext/ndb/model.py @@ -2741,8 +2741,9 @@ def _db_get_value(self, v, p): sval = modelclass._from_pb(pb) elif meaning != entity_pb2.Property.BYTESTRING: try: - sval.decode('ascii') - + decoded = sval.decode('ascii') + if meaning == entity_pb2.Property.TEXT: + sval = six.ensure_str(decoded, encoding='ascii') except UnicodeDecodeError: try: sval = six.text_type(sval.decode('utf-8')) diff --git a/tests/google/appengine/ext/ndb/polymodel_test.py b/tests/google/appengine/ext/ndb/polymodel_test.py index 5608e57..515c154 100755 --- a/tests/google/appengine/ext/ndb/polymodel_test.py +++ b/tests/google/appengine/ext/ndb/polymodel_test.py @@ -143,9 +143,18 @@ class Mammal(Animal): self.assertEqual(cat1.naps, 18) self.assertEqual(cat1.sound, b'purr') - def testExpandoPoly(self): - + # un-indexed properties retain str type regardless of encoding: + Mammal._default_indexed = False + cat = Mammal(name='Clémentine', naps=18, sound=b'meow', action='stretch') + cat1 = cat.put().get(use_cache=False) + self.assertFalse(cat1 is cat) + self.assertEqual(cat1, cat) + self.assertEqual(cat1.name, 'Clémentine') + self.assertEqual(cat1.naps, 18) + self.assertEqual(cat1.sound, b'meow') + self.assertEqual(cat1.action, 'stretch') + def testExpandoPoly(self): class Animal(model.Expando, PolyModel): pass @@ -162,6 +171,18 @@ class Mammal(Animal): self.assertEqual(18, actual.naps) self.assertEqual(b'purr', actual.sound) + # un-indexed properties retain str type regardless of encoding: + Mammal._default_indexed = False + expected = Mammal(name='Clémentine', naps=18, sound=b'meow', action='stretch') + actual = expected.put().get() + + self.assertIsNot(expected, actual) + self.assertEqual(expected, actual) + self.assertEqual('Clémentine', actual.name) + self.assertEqual(18, actual.naps) + self.assertEqual(b'meow', actual.sound) + self.assertEqual('stretch', actual.action) + def testInheritance(self): class NamedThing(model.Model): From 517b58241d902892b0fdeefd3a58314b6314d7ed Mon Sep 17 00:00:00 2001 From: peter Date: Mon, 21 Aug 2023 18:42:34 -0400 Subject: [PATCH 2/3] avoid extra function call --- src/google/appengine/ext/ndb/model.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/google/appengine/ext/ndb/model.py b/src/google/appengine/ext/ndb/model.py index 541643b..9ccf0aa 100755 --- a/src/google/appengine/ext/ndb/model.py +++ b/src/google/appengine/ext/ndb/model.py @@ -2741,9 +2741,10 @@ def _db_get_value(self, v, p): sval = modelclass._from_pb(pb) elif meaning != entity_pb2.Property.BYTESTRING: try: - decoded = sval.decode('ascii') - if meaning == entity_pb2.Property.TEXT: - sval = six.ensure_str(decoded, encoding='ascii') + if six.PY2: + sval.decode('ascii') + elif meaning == entity_pb2.Property.TEXT: + sval = sval.decode('ascii') except UnicodeDecodeError: try: sval = six.text_type(sval.decode('utf-8')) From 546bb38707bc55f59d4e25006afe77b784791e43 Mon Sep 17 00:00:00 2001 From: peter Date: Mon, 21 Aug 2023 19:59:51 -0400 Subject: [PATCH 3/3] simplify + preserve existing behavior better (ie utf-8 works :P) --- src/google/appengine/ext/ndb/model.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/google/appengine/ext/ndb/model.py b/src/google/appengine/ext/ndb/model.py index 9ccf0aa..834665e 100755 --- a/src/google/appengine/ext/ndb/model.py +++ b/src/google/appengine/ext/ndb/model.py @@ -2741,10 +2741,9 @@ def _db_get_value(self, v, p): sval = modelclass._from_pb(pb) elif meaning != entity_pb2.Property.BYTESTRING: try: - if six.PY2: - sval.decode('ascii') - elif meaning == entity_pb2.Property.TEXT: - sval = sval.decode('ascii') + decoded = sval.decode('ascii') + if six.PY3 and meaning == entity_pb2.Property.TEXT: + sval = decoded except UnicodeDecodeError: try: sval = six.text_type(sval.decode('utf-8'))