Skip to content

Move DatabaseController and Schema fully to adaptive mongo collection. #909

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 8, 2016
Merged

Move DatabaseController and Schema fully to adaptive mongo collection. #909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a163327
Remove usages of non-adaptive collection inside DatabaseController.
nlutsenko Mar 8, 2016
fb5b8fb
Migrate Schema.js to adaptive mongo collection.
nlutsenko Mar 8, 2016
49eb9df
Remove private Schema API usage from SchemasRouter.
nlutsenko Mar 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions src/Adapters/Storage/Mongo/MongoCollection.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,11 @@ export default class MongoCollection {
return this._mongoCollection.findAndModify(query, [], update, { new: true }).then(document => {
// Value is the object where mongo returns multiple fields.
return document.value;
})
});
}

insertOne(object) {
return this._mongoCollection.insertOne(object);
}

// Atomically updates data in the database for a single (first) object that matched the query
Expand All @@ -64,6 +68,10 @@ export default class MongoCollection {
return this._mongoCollection.update(query, update, { upsert: true });
}

updateOne(query, update) {
return this._mongoCollection.updateOne(query, update);
}

updateMany(query, update) {
return this._mongoCollection.updateMany(query, update);
}
Expand All @@ -83,8 +91,8 @@ export default class MongoCollection {
return this._mongoCollection.deleteOne(query);
}

remove(query) {
return this._mongoCollection.remove(query);
deleteMany(query) {
return this._mongoCollection.deleteMany(query);
}

drop() {
Expand Down
150 changes: 71 additions & 79 deletions src/Controllers/DatabaseController.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,6 @@ DatabaseController.prototype.connect = function() {
return this.adapter.connect();
};

// Returns a promise for a Mongo collection.
// Generally just for internal use.
DatabaseController.prototype.collection = function(className) {
if (!Schema.classNameIsValid(className)) {
throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME,
'invalid className: ' + className);
}
return this.adapter.collection(this.collectionPrefix + className);
};

DatabaseController.prototype.adaptiveCollection = function(className) {
return this.adapter.adaptiveCollection(this.collectionPrefix + className);
};
Expand All @@ -54,15 +44,23 @@ function returnsTrue() {
return true;
}

DatabaseController.prototype.validateClassName = function(className) {
if (!Schema.classNameIsValid(className)) {
const error = new Parse.Error(Parse.Error.INVALID_CLASS_NAME, 'invalid className: ' + className);
return Promise.reject(error);
}
return Promise.resolve();
};

// Returns a promise for a schema object.
// If we are provided a acceptor, then we run it on the schema.
// If the schema isn't accepted, we reload it at most once.
DatabaseController.prototype.loadSchema = function(acceptor = returnsTrue) {

if (!this.schemaPromise) {
this.schemaPromise = this.collection('_SCHEMA').then((coll) => {
this.schemaPromise = this.adaptiveCollection('_SCHEMA').then(collection => {
delete this.schemaPromise;
return Schema.load(coll);
return Schema.load(collection);
});
return this.schemaPromise;
}
Expand All @@ -71,9 +69,9 @@ DatabaseController.prototype.loadSchema = function(acceptor = returnsTrue) {
if (acceptor(schema)) {
return schema;
}
this.schemaPromise = this.collection('_SCHEMA').then((coll) => {
this.schemaPromise = this.adaptiveCollection('_SCHEMA').then(collection => {
delete this.schemaPromise;
return Schema.load(coll);
return Schema.load(collection);
});
return this.schemaPromise;
});
Expand Down Expand Up @@ -230,30 +228,28 @@ DatabaseController.prototype.handleRelationUpdates = function(className,

// Adds a relation.
// Returns a promise that resolves successfully iff the add was successful.
DatabaseController.prototype.addRelation = function(key, fromClassName,
fromId, toId) {
var doc = {
DatabaseController.prototype.addRelation = function(key, fromClassName, fromId, toId) {
let doc = {
relatedId: toId,
owningId: fromId
owningId : fromId
};
var className = '_Join:' + key + ':' + fromClassName;
return this.collection(className).then((coll) => {
return coll.update(doc, doc, {upsert: true});
let className = `_Join:${key}:${fromClassName}`;
return this.adaptiveCollection(className).then((coll) => {
return coll.upsertOne(doc, doc);
});
};

// Removes a relation.
// Returns a promise that resolves successfully iff the remove was
// successful.
DatabaseController.prototype.removeRelation = function(key, fromClassName,
fromId, toId) {
DatabaseController.prototype.removeRelation = function(key, fromClassName, fromId, toId) {
var doc = {
relatedId: toId,
owningId: fromId
};
var className = '_Join:' + key + ':' + fromClassName;
return this.collection(className).then((coll) => {
return coll.remove(doc);
let className = `_Join:${key}:${fromClassName}`;
return this.adaptiveCollection(className).then(coll => {
return coll.deleteOne(doc);
});
};

Expand All @@ -269,40 +265,36 @@ DatabaseController.prototype.destroy = function(className, query, options = {})
var aclGroup = options.acl || [];

var schema;
return this.loadSchema().then((s) => {
schema = s;
if (!isMaster) {
return schema.validatePermission(className, aclGroup, 'delete');
}
return Promise.resolve();
}).then(() => {

return this.collection(className);
}).then((coll) => {
var mongoWhere = transform.transformWhere(schema, className, query);

if (options.acl) {
var writePerms = [
{_wperm: {'$exists': false}}
];
for (var entry of options.acl) {
writePerms.push({_wperm: {'$in': [entry]}});
return this.loadSchema()
.then(s => {
schema = s;
if (!isMaster) {
return schema.validatePermission(className, aclGroup, 'delete');
}
mongoWhere = {'$and': [mongoWhere, {'$or': writePerms}]};
}

return coll.remove(mongoWhere);
}).then((resp) => {
//Check _Session to avoid changing password failed without any session.
if (resp.result.n === 0 && className !== "_Session") {
return Promise.reject(
new Parse.Error(Parse.Error.OBJECT_NOT_FOUND,
'Object not found.'));
return Promise.resolve();
})
.then(() => this.adaptiveCollection(className))
.then(collection => {
let mongoWhere = transform.transformWhere(schema, className, query);

}
}, (error) => {
throw error;
});
if (options.acl) {
var writePerms = [
{ _wperm: { '$exists': false } }
];
for (var entry of options.acl) {
writePerms.push({ _wperm: { '$in': [entry] } });
}
mongoWhere = { '$and': [mongoWhere, { '$or': writePerms }] };
}
return collection.deleteMany(mongoWhere);
})
.then(resp => {
//Check _Session to avoid changing password failed without any session.
// TODO: @nlutsenko Stop relying on `result.n`
if (resp.result.n === 0 && className !== "_Session") {
throw new Parse.Error(Parse.Error.OBJECT_NOT_FOUND, 'Object not found.');
}
});
};

// Inserts an object into the database.
Expand All @@ -312,21 +304,21 @@ DatabaseController.prototype.create = function(className, object, options) {
var isMaster = !('acl' in options);
var aclGroup = options.acl || [];

return this.loadSchema().then((s) => {
schema = s;
if (!isMaster) {
return schema.validatePermission(className, aclGroup, 'create');
}
return Promise.resolve();
}).then(() => {

return this.handleRelationUpdates(className, null, object);
}).then(() => {
return this.collection(className);
}).then((coll) => {
var mongoObject = transform.transformCreate(schema, className, object);
return coll.insert([mongoObject]);
});
return this.validateClassName(className)
.then(() => this.loadSchema())
.then(s => {
schema = s;
if (!isMaster) {
return schema.validatePermission(className, aclGroup, 'create');
}
return Promise.resolve();
})
.then(() => this.handleRelationUpdates(className, null, object))
.then(() => this.adaptiveCollection(className))
.then(coll => {
var mongoObject = transform.transformCreate(schema, className, object);
return coll.insertOne(mongoObject);
});
};

// Runs a mongo query on the database.
Expand Down Expand Up @@ -386,14 +378,14 @@ DatabaseController.prototype.owningIds = function(className, key, relatedIds) {
// equal-to-pointer constraints on relation fields.
// Returns a promise that resolves when query is mutated
DatabaseController.prototype.reduceInRelation = function(className, query, schema) {

// Search for an in-relation or equal-to-relation
// Make it sequential for now, not sure of paralleization side effects
if (query['$or']) {
let ors = query['$or'];
return Promise.all(ors.map((aQuery, index) => {
return this.reduceInRelation(className, aQuery, schema).then((aQuery) => {
query['$or'][index] = aQuery;
query['$or'][index] = aQuery;
})
}));
}
Expand All @@ -413,14 +405,14 @@ DatabaseController.prototype.reduceInRelation = function(className, query, schem
relatedIds = [query[key].objectId];
}
return this.owningIds(className, key, relatedIds).then((ids) => {
delete query[key];
delete query[key];
this.addInObjectIdsIds(ids, query);
return Promise.resolve(query);
});
}
return Promise.resolve(query);
})

return Promise.all(promises).then(() => {
return Promise.resolve(query);
})
Expand All @@ -429,13 +421,13 @@ DatabaseController.prototype.reduceInRelation = function(className, query, schem
// Modifies query so that it no longer has $relatedTo
// Returns a promise that resolves when query is mutated
DatabaseController.prototype.reduceRelationKeys = function(className, query) {

if (query['$or']) {
return Promise.all(query['$or'].map((aQuery) => {
return this.reduceRelationKeys(className, aQuery);
}));
}

var relatedTo = query['$relatedTo'];
if (relatedTo) {
return this.relatedIds(
Expand Down
2 changes: 1 addition & 1 deletion src/Controllers/HooksController.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ export class HooksController {

_removeHooks(query) {
return this.getCollection().then(collection => {
return collection.remove(query);
return collection.deleteMany(query);
}).then(() => {
return {};
});
Expand Down
41 changes: 22 additions & 19 deletions src/Routers/SchemasRouter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ function getAllSchemas(req) {
return req.config.database.adaptiveCollection('_SCHEMA')
.then(collection => collection.find({}))
.then(schemas => schemas.map(Schema.mongoSchemaToSchemaAPIResponse))
.then(schemas => ({ response: { results: schemas }}));
.then(schemas => ({ response: { results: schemas } }));
}

function getOneSchema(req) {
Expand Down Expand Up @@ -65,7 +65,7 @@ function modifySchema(req) {
throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${req.params.className} does not exist.`);
}

let existingFields = Object.assign(schema.data[className], {_id: className});
let existingFields = Object.assign(schema.data[className], { _id: className });
Object.keys(submittedFields).forEach(name => {
let field = submittedFields[name];
if (existingFields[name] && field.__op !== 'Delete') {
Expand All @@ -83,24 +83,27 @@ function modifySchema(req) {
}

// Finally we have checked to make sure the request is valid and we can start deleting fields.
// Do all deletions first, then a single save to _SCHEMA collection to handle all additions.
let deletionPromises = [];
Object.keys(submittedFields).forEach(submittedFieldName => {
if (submittedFields[submittedFieldName].__op === 'Delete') {
let promise = schema.deleteField(submittedFieldName, className, req.config.database);
deletionPromises.push(promise);
// Do all deletions first, then add fields to avoid duplicate geopoint error.
let deletePromises = [];
let insertedFields = [];
Object.keys(submittedFields).forEach(fieldName => {
if (submittedFields[fieldName].__op === 'Delete') {
const promise = schema.deleteField(fieldName, className, req.config.database);
deletePromises.push(promise);
} else {
insertedFields.push(fieldName);
}
});

return Promise.all(deletionPromises)
.then(() => new Promise((resolve, reject) => {
schema.collection.update({_id: className}, mongoObject.result, {w: 1}, (err, docs) => {
if (err) {
reject(err);
}
resolve({ response: Schema.mongoSchemaToSchemaAPIResponse(mongoObject.result)});
})
}));
return Promise.all(deletePromises) // Delete Everything
.then(() => schema.reloadData()) // Reload our Schema, so we have all the new values
.then(() => {
let promises = insertedFields.map(fieldName => {
const mongoType = mongoObject.result[fieldName];
return schema.validateField(className, fieldName, mongoType);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The reason I didn't use validateField originally is that it means you only need a single database operation to handle all the additions. Can you preserve that behaviour here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can try, but am not sure why this should be blocking, since you don't have atomicity guarantees on deletions anyhow, meaning that the entire operation is not atomic...

Can this survive without having a single write for inserts? I can add a todo... 😁

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yah there already a lot of failure modes but this does introduce a new failure mode where some fields are added and some are not. Although this eliminates the failure mode where one person adds a different schema in the middle and then it gets clobbered, so it's really just a trade from one failure mode to another, which is probably fine. In practise I doubt there would be many writers to the schema at the same time anyway.

});
return Promise.all(promises);
})
.then(() => ({ response: Schema.mongoSchemaToSchemaAPIResponse(mongoObject.result) }));
});
}

Expand Down Expand Up @@ -140,7 +143,7 @@ function deleteSchema(req) {
// We've dropped the collection now, so delete the item from _SCHEMA
// and clear the _Join collections
return req.config.database.adaptiveCollection('_SCHEMA')
.then(coll => coll.findOneAndDelete({_id: req.params.className}))
.then(coll => coll.findOneAndDelete({ _id: req.params.className }))
.then(document => {
if (document === null) {
//tried to delete non-existent class
Expand Down
Loading