ENH Append loads data if schema is subset of existing schema

Author: mr-mcox

pandas_gbq/gbq.py

@@ -588,6 +588,14 @@ def verify_schema(self, dataset_id, table_id, schema):
 
         return fields_remote == fields_local
 
+    def schema_is_subset(self, dataset_id, table_id, schema):
+        fields_remote = set([json.dumps(field)
+                             for field in self.schema(dataset_id, table_id)])
+        fields_local = set(json.dumps(field_local)
+                           for field_local in schema['fields'])
+
+        return fields_remote >= fields_local
+
     def delete_and_recreate_table(self, dataset_id, table_id, table_schema):
         delay = 0
 
@@ -850,7 +858,8 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000,
             connector.delete_and_recreate_table(
                 dataset_id, table_id, table_schema)
         elif if_exists == 'append':
-            if not connector.verify_schema(dataset_id, table_id, table_schema):
+            if not connector.schema_is_subset(dataset_id,
+                                              table_id, table_schema):
                 raise InvalidSchema("Please verify that the structure and "
                                     "data types in the DataFrame match the "
                                     "schema of the destination table.")

pandas_gbq/tests/test_gbq.py

@@ -19,12 +19,9 @@
 import pandas.util.testing as tm
 from pandas.compat.numpy import np_datetime64_compat
 
-PROJECT_ID = os.environ.get('PROJECT_ID')
-PRIVATE_KEY_JSON_PATH = 'client_secrets.json'
-PRIVATE_KEY_JSON_CONTENTS = os.environ.get('SERVICE_ACCOUNT_KEY')
-# PROJECT_ID = None
-# PRIVATE_KEY_JSON_PATH = None
-# PRIVATE_KEY_JSON_CONTENTS = None
+PROJECT_ID = os.getenv('PROJECT_ID')
+PRIVATE_KEY_JSON_PATH = os.getenv('PRIVATE_KEY_JSON_PATH')
+PRIVATE_KEY_JSON_CONTENTS = os.getenv('PRIVATE_KEY_JSON_CONTENTS')
 
 TABLE_ID = 'new_test'
 
@@ -1085,6 +1082,29 @@ def test_upload_data_if_table_exists_append(self):
                        _get_project_id(), if_exists='append',
                        private_key=_get_private_key_path())
 
+    def test_upload_subset_columns_if_table_exists_append(self):
+        test_id = "16"
+        test_size = 10
+        df = make_mixed_dataframe_v2(test_size)
+        df_subset_cols = df.iloc[:, :2]
+
+        # Initialize table with sample data
+        gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(),
+                   chunksize=10000, private_key=_get_private_key_path())
+
+        # Test the if_exists parameter with value 'append'
+        gbq.to_gbq(df_subset_cols,
+                   self.destination_table + test_id, _get_project_id(),
+                   if_exists='append', private_key=_get_private_key_path())
+
+        sleep(30)  # <- Curses Google!!!
+
+        result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}"
+                              .format(self.destination_table + test_id),
+                              project_id=_get_project_id(),
+                              private_key=_get_private_key_path())
+        self.assertEqual(result['num_rows'][0], test_size * 2)
+
     def test_upload_data_if_table_exists_replace(self):
         test_id = "4"
         test_size = 10