Datastore update schema

appengine/standard/ndb/schema_update/app.yaml

@@ -0,0 +1,17 @@
+runtime: python27
+api_version: 1
+threadsafe: true
+
+builtins:
+# Deferred is required to use google.appengine.ext.deferred.
+- deferred: on
+
+handlers:
+- url: /.*
+  script: main.app
+
+libraries:
+- name: webapp2
+  version: "2.5.2"
+- name: jinja2
+  version: "2.6"

appengine/standard/ndb/schema_update/main.py

@@ -0,0 +1,131 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Sample application that shows how to perform a "schema migration" using
+Google Cloud Datastore.
+
+This application uses one model named "Pictures" but two different versions
+of it. v2 contains two extra fields. The application shows how to
+populate these new fields onto entities that existed prior to adding the
+new fields to the model class.
+"""
+
+import logging
+import os
+
+from google.appengine.ext import deferred
+from google.appengine.ext import ndb
+import jinja2
+import webapp2
+
+import models_v1
+import models_v2
+
+
+JINJA_ENVIRONMENT = jinja2.Environment(
+    loader=jinja2.FileSystemLoader(
+        os.path.join(os.path.dirname(__file__), 'templates')),
+    extensions=['jinja2.ext.autoescape'],
+    autoescape=True)
+
+
+class DisplayEntitiesHandler(webapp2.RequestHandler):
+    """Displays the current set of entities and options to add entities
+    or update the schema."""
+    def get(self):
+        # Force ndb to use v2 of the model by re-loading it.
+        reload(models_v2)
+
+        entities = models_v2.Picture.query().fetch()
+        template_values = {
+            'entities': entities,
+        }
+
+        template = JINJA_ENVIRONMENT.get_template('index.html')
+        self.response.write(template.render(template_values))
+
+
+class AddEntitiesHandler(webapp2.RequestHandler):
+    """Adds new entities using the v1 schema."""
+    def post(self):
+        # Force ndb to use v1 of the model by re-loading it.
+        reload(models_v1)
+
+        # Save some example data.
+        ndb.put_multi([
+            models_v1.Picture(author='Alice', name='Sunset'),
+            models_v1.Picture(author='Bob', name='Sunrise')
+        ])
+
+        self.response.write("""
+        Entities created. <a href="/">View entities</a>.
+        """)
+
+
+class UpdateSchemaHandler(webapp2.RequestHandler):
+    """Queues a task to start updating the model schema."""
+    def post(self):
+        deferred.defer(update_schema_task)
+        self.response.write("""
+        Schema update started. Check the console for task progress.
+        <a href="/">View entities</a>.
+        """)
+
+
+def update_schema_task(cursor=None, num_updated=0, batch_size=100):
+    """Task that handles updating the models' schema.
+
+    This is started by
+    UpdateSchemaHandler. It scans every entity in the datastore for the
+    Picture model and re-saves it so that it has the new schema fields.
+    """
+
+    # Force ndb to use v2 of the model by re-loading it.
+    reload(models_v2)
+
+    # Get all of the entities for this Model.
+    query = models_v2.Picture.query()
+    pictures, cursor, more = query.fetch_page(batch_size, start_cursor=cursor)
+
+    to_put = []
+    for picture in pictures:
+        # Give the new fields default values.
+        # If you added new fields and were okay with the default values, you
+        # would not need to do this.
+        picture.num_votes = 1
+        picture.avg_rating = 5
+        to_put.append(picture)
+
+    # Save the updated entities.
+    if to_put:
+        ndb.put_multi(to_put)
+        num_updated += len(to_put)
+        logging.info(
+            'Put {} entities to Datastore for a total of {}'.format(
+                len(to_put), num_updated))
+
+    # If there are more entities, re-queue this task for the next page.
+    if more:
+        deferred.defer(
+            update_schema_task, cursor=query.cursor(), num_updated=num_updated)
+    else:
+        logging.debug(
+            'update_schema_task complete with {0} updates!'.format(
+                num_updated))
+
+
+app = webapp2.WSGIApplication([
+    ('/', DisplayEntitiesHandler),
+    ('/add_entities', AddEntitiesHandler),
+    ('/update_schema', UpdateSchemaHandler)])

appengine/standard/ndb/schema_update/main_test.py

@@ -0,0 +1,62 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.appengine.ext import deferred
+import pytest
+import webtest
+
+import main
+import models_v1
+import models_v2
+
+
+@pytest.fixture
+def app(testbed):
+    yield webtest.TestApp(main.app)
+
+
+def test_app(app):
+    response = app.get('/')
+    assert response.status_int == 200
+
+
+def test_add_entities(app):
+    response = app.post('/add_entities')
+    assert response.status_int == 200
+    response = app.get('/')
+    assert response.status_int == 200
+    assert 'Author: Bob' in response.body
+    assert 'Name: Sunrise' in response.body
+    assert 'Author: Alice' in response.body
+    assert 'Name: Sunset' in response.body
+
+
+def test_update_schema(app, testbed):
+    reload(models_v1)
+    test_model = models_v1.Picture(author='Test', name='Test')
+    test_model.put()
+
+    response = app.post('/update_schema')
+    assert response.status_int == 200
+
+    # Run the queued task.
+    tasks = testbed.taskqueue_stub.get_filtered_tasks()
+    assert len(tasks) == 1
+    deferred.run(tasks[0].payload)
+
+    # Check the updated items
+    reload(models_v2)
+    updated_model = test_model.key.get()
+    assert updated_model.num_votes == 1
+    assert updated_model.avg_rating == 5.0

appengine/standard/ndb/schema_update/models_v1.py

@@ -0,0 +1,20 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.appengine.ext import ndb
+
+
+class Picture(ndb.Model):
+    author = ndb.StringProperty()
+    name = ndb.StringProperty(default='')

appengine/standard/ndb/schema_update/models_v2.py

@@ -0,0 +1,23 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.appengine.ext import ndb
+
+
+class Picture(ndb.Model):
+    author = ndb.StringProperty()
+    name = ndb.StringProperty(default='')
+    # Two new fields
+    num_votes = ndb.IntegerProperty(default=0)
+    avg_rating = ndb.FloatProperty(default=0)

appengine/standard/ndb/schema_update/templates/index.html

@@ -0,0 +1,47 @@
+{#
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+<!DOCTYPE html>
+<html>
+  <head>
+  </head>
+  <body>
+    <p>If you've just added or updated entities, you may need to refresh
+      the page to see the changes due to
+      <a href="https://cloud.google.com/datastore/docs/articles/balancing-strong-and-eventual-consistency-with-google-cloud-datastore/">eventual consitency.</a></p>
+    {% for entity in entities %}
+      <p>
+        Author: {{entity.author}},
+        Name: {{entity.name}},
+        {% if 'num_votes' in entity._values %}
+          Votes: {{entity.num_votes}},
+        {% endif %}
+        {% if 'avg_rating' in entity._values %}
+          Average Rating: {{entity.avg_rating}}
+        {% endif %}
+      </p>
+    {% endfor %}
+    {% if entities|length == 0 %}
+      <form action="/add_entities" method="post" method="post">
+        <input type="submit" value="Add Entities">
+      </form>
+    {% endif %}
+    {% if entities|length > 0 %}
+      <form action="/update_schema" method="post" method="post">
+        <input type="submit" value="Update Schema">
+      </form>
+    {% endif %}
+  </body>
+</html>