Bug fixes and fix Unit test pipeline (#79)

* Allow UTF-8 surrogate characters when decoding (and encoding) TEXT properties in db and ndb

* Update README to add instruction for consuming pre-release versions of the SDK.

* Add constraint for protobuf version in tox.ini, since 'protobuf >= 4' has breaking changes that breaks our CI tests.

README.md

@@ -13,6 +13,8 @@ In your `requirements.txt` file, add the following:
 
 `appengine-python-standard>=1.0.0`
 
+Currently, `1.0.0` is the latest released stable version. To use a pre-release version (Eg. `1.0.1-rc1`), modify the above line to `appengine-python-standard>=[insert_version]` (Eg. `appengine-python-standard>=1.0.1-rc1`).
+
 In your app's `app.yaml`, add the following:
 
 `app_engine_apis: true`
@@ -38,7 +40,7 @@ Example for a Flask app:
     app = Flask(__name__)
     app.wsgi_app = google.appengine.api.wrap_wsgi_app(app.wsgi_app)
 
-Then deploy your app as usual, with `gcloud beta app deploy` (currently only the Beta version has the capability to enable these APIs). The following modules are available:
+Then deploy your app as usual, with `gcloud app deploy`. The following modules are available:
 
 - `google.appengine.api.app_identity`
 - `google.appengine.api.background_thread`

src/google/appengine/api/datastore_types.py

@@ -1629,7 +1629,7 @@ def PackString(name, value, pbvalue):
     value.decode('ascii')
     pbvalue.stringValue = value
   else:
-    pbvalue.stringValue = six.text_type(value).encode('utf-8')
+    pbvalue.stringValue = six.text_type(value).encode('utf-8', 'surrogatepass')
 
 
 def PackDatetime(name, value, pbvalue):
@@ -1917,7 +1917,7 @@ def FromPropertyPb(pb):
     value = pbval.stringValue
     if not pb.HasField('meaning') or meaning not in _NON_UTF8_MEANINGS:
 
-      value = value.decode('utf-8')
+      value = value.decode('utf-8', 'surrogatepass')
   elif pbval.HasField('int64Value'):
 
 

src/google/appengine/ext/ndb/model.py

@@ -1819,7 +1819,7 @@ def _validate(self, value):
         raise datastore_errors.BadValueError('Expected valid UTF-8, got %r' %
                                              (value,))
     elif isinstance(value, six.text_type):
-      length = len(value.encode('utf-8'))
+      length = len(value.encode('utf-8', 'surrogatepass'))
     else:
       raise datastore_errors.BadValueError('Expected string, got %r' %
                                            (value,))
@@ -1830,12 +1830,12 @@ def _validate(self, value):
 
   def _to_base_type(self, value):
     if isinstance(value, six.text_type):
-      return value.encode('utf-8')
+      return value.encode('utf-8', 'surrogatepass')
 
   def _from_base_type(self, value):
     if isinstance(value, six.binary_type):
       try:
-        return six.text_type(value, 'utf-8')
+        return six.text_type(value, 'utf-8', 'surrogatepass')
       except UnicodeDecodeError:
 
 

tox.ini

@@ -23,7 +23,7 @@ deps =
     google-auth
     mock
     Pillow
-    protobuf
+    protobuf < 4
     pytz
     requests
     ruamel.yaml