Compound

README.md

@@ -597,7 +597,7 @@ NeDB supports indexing. It gives a very nice speed boost and can be used to enfo
 
 To create an index, use `datastore.ensureIndex(options, cb)`, where callback is optional and get passed an error if any (usually a unique constraint that was violated). `ensureIndex` can be called when you want, even after some data was inserted, though it's best to call it at application startup. The options are:  
 
-* **fieldName** (required): name of the field to index. Use the dot notation to index a field in a nested document.
+* **fieldName** (required): name of the field to index. Use the dot notation to index a field in a nested document. For a compound index, use an array of field names.
 * **unique** (optional, defaults to `false`): enforce field uniqueness. Note that a unique index will raise an error if you try to index two documents for which the field is not defined.
 * **sparse** (optional, defaults to `false`): don't index documents for which the field is not defined. Use this option along with "unique" if you want to accept multiple documents for which it is not defined.
 * **expireAfterSeconds** (number of seconds, optional): if set, the created index is a TTL (time to live) index, that will automatically remove documents when the system date becomes larger than the date on the indexed field plus `expireAfterSeconds`. Documents where the indexed field is not specified or not a `Date` object are ignored

lib/datastore.js

@@ -134,7 +134,6 @@ Datastore.prototype.ensureIndex = function (options, cb) {
 
   this.indexes[options.fieldName] = new Index(options);
   if (options.expireAfterSeconds !== undefined) { this.ttlIndexes[options.fieldName] = options.expireAfterSeconds; }   // With this implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here
-
   try {
     this.indexes[options.fieldName].insert(this.getAllData());
   } catch (e) {
@@ -255,27 +254,44 @@ Datastore.prototype.updateIndexes = function (oldDoc, newDoc) {
 Datastore.prototype.getCandidates = function (query, dontExpireStaleDocs, callback) {
   var indexNames = Object.keys(this.indexes)
     , self = this
-    , usableQueryKeys;
+    , usableQueryKeys
+    , basicQueryKeys
+    , compoundQueryKeys;
 
   if (typeof dontExpireStaleDocs === 'function') {
     callback = dontExpireStaleDocs;
     dontExpireStaleDocs = false;
   }
 
-
   async.waterfall([
   // STEP 1: get candidates list by checking indexes from most to least frequent usecase
   function (cb) {
-    // For a basic match
+
     usableQueryKeys = [];
     Object.keys(query).forEach(function (k) {
       if (typeof query[k] === 'string' || typeof query[k] === 'number' || typeof query[k] === 'boolean' || util.isDate(query[k]) || query[k] === null) {
         usableQueryKeys.push(k);
       }
     });
-    usableQueryKeys = _.intersection(usableQueryKeys, indexNames);
-    if (usableQueryKeys.length > 0) {
-      return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]]));
+
+    // For a basic match
+    basicQueryKeys = _.intersection(usableQueryKeys, indexNames);
+    if (basicQueryKeys.length > 0) {
+      return cb(null, self.indexes[basicQueryKeys[0]].getMatching(query[basicQueryKeys[0]]));
+    }
+
+    // For a compound match
+    compoundQueryKeys = [];
+    indexNames.forEach(function(indexName){
+      if (indexName.indexOf(',') === -1) return;
+      var subIndexNames = indexName.split(',');
+      if (_.intersection(subIndexNames, usableQueryKeys).length === subIndexNames.length) {
+        compoundQueryKeys.push(subIndexNames);
+      }
+    });
+
+    if (compoundQueryKeys.length > 0) {
+      return cb(null, self.indexes[compoundQueryKeys[0]].getMatching(_.pick(query,compoundQueryKeys[0])));
     }
 
     // For a $in match

lib/indexes.js

@@ -24,6 +24,24 @@ function projectForUnique (elt) {
   return elt;   // Arrays and objects, will check for pointer equality
 }
 
+/**
+ * Get dot values for either a bunch of fields or just one.
+ */
+function getDotValues (doc, fields) {
+  var field, key, i, len;
+
+  if (util.isArray(fields)) {
+    key = {};
+    for (i = 0, len = fields.length; i < len; i++) {
+      field = fields[i];
+      key[field] = model.getDotValue(doc, field);
+    }
+    return key;
+  } else {
+    return model.getDotValue(doc, fields);
+  }
+}
+
 
 /**
  * Create a new index
@@ -38,7 +56,8 @@ function Index (options) {
   this.unique = options.unique || false;
   this.sparse = options.sparse || false;
 
-  this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality };
+  var compareFunc = util.isArray(this.fieldName) ? model.compoundCompareThings(this.fieldName) : model.compareThings;
+  this.treeOptions = { unique: this.unique, compareKeys: compareFunc, checkValueEquality: checkValueEquality };
 
   this.reset();   // No data in the beginning
 }
@@ -68,10 +87,10 @@ Index.prototype.insert = function (doc) {
 
   if (util.isArray(doc)) { this.insertMultipleDocs(doc); return; }
 
-  key = model.getDotValue(doc, this.fieldName);
+  key = getDotValues(doc, this.fieldName);
 
   // We don't index documents that don't contain the field if the index is sparse
-  if (key === undefined && this.sparse) { return; }
+  if ((key === undefined || _.isEmpty(key)) && this.sparse) { return; }
 
   if (!util.isArray(key)) {
     this.tree.insert(key, doc);
@@ -140,9 +159,9 @@ Index.prototype.remove = function (doc) {
 
   if (util.isArray(doc)) { doc.forEach(function (d) { self.remove(d); }); return; }
 
-  key = model.getDotValue(doc, this.fieldName);
+  key = getDotValues(doc, this.fieldName);
 
-  if (key === undefined && this.sparse) { return; }
+  if ((key === undefined || _.isEmpty(key))&& this.sparse) { return; }
 
   if (!util.isArray(key)) {
     this.tree.delete(key, doc);

lib/model.js

@@ -231,6 +231,30 @@ function compareThings (a, b, _compareStrings) {
   return compareNSB(aKeys.length, bKeys.length);
 }
 
+/**
+ * Used primarily in compound indexes. Returns a comparison function usable as
+ * an Index's compareKeys function.
+ */
+function compoundCompareThings (fields) {
+  return function (a, b) {
+    var i, len, comparison;
+
+    // undefined
+    if (a === undefined) { return b === undefined ? 0 : -1; }
+    if (b === undefined) { return a === undefined ? 0 : 1; }
+
+    // null
+    if (a === null) { return b === null ? 0 : -1; }
+    if (b === null) { return a === null ? 0 : 1; }
+
+    for (i = 0, len = fields.length; i < len; i++) {
+      comparison = compareThings(a[fields[i]], b[fields[i]]);
+      if (comparison !== 0) { return comparison; }
+    }
+
+    return 0;
+  };
+}
 
 
 // ==============================================================
@@ -833,3 +857,4 @@ module.exports.getDotValue = getDotValue;
 module.exports.match = match;
 module.exports.areThingsEqual = areThingsEqual;
 module.exports.compareThings = compareThings;
+module.exports.compoundCompareThings = compoundCompareThings;

lib/persistence.js

@@ -129,8 +129,9 @@ Persistence.prototype.persistCachedDatabase = function (cb) {
     toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
   });
   Object.keys(this.db.indexes).forEach(function (fieldName) {
+
     if (fieldName != "_id") {   // The special _id index is managed by datastore.js, the others need to be persisted
-      toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n';
+      toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: self.db.indexes[fieldName].fieldName, unique: self.db.indexes[fieldName].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n';
     }
   });
 

test/db.test.js

@@ -450,6 +450,28 @@ describe('Database', function () {
       });
     });
 
+    it('Can use a compound index to get docs with a basic match', function (done) {
+      d.ensureIndex({ fieldName: ['tf', 'tg'] }, function (err) {
+        d.insert({ tf: 4, tg: 0, foo: 1 }, function () {
+          d.insert({ tf: 6, tg: 0, foo: 2 }, function () {
+            d.insert({ tf: 4, tg: 1, foo: 3 }, function (err, _doc1) {
+              d.insert({ tf: 6, tg: 1, foo: 4 }, function () {
+                d.getCandidates({ tf: 4, tg: 1 }, function (err, data) {
+                  var doc1 = _.find(data, function (d) { return d._id === _doc1._id; })
+                  ;
+
+                data.length.should.equal(1);
+                assert.deepEqual(doc1, { _id: doc1._id, tf: 4, tg: 1, foo: 3 });
+
+                done();
+                });
+              });
+            });
+          });
+        });
+      });
+    });
+
     it('Can use an index to get docs with a $in match', function (done) {
       d.ensureIndex({ fieldName: 'tf' }, function (err) {
         d.insert({ tf: 4 }, function (err) {

test/indexes.test.js

@@ -34,6 +34,31 @@ describe('Indexes', function () {
       doc3.a.should.equal(42);
     });
 
+    it('Can insert pointers to documents in the index correctly when they have compound fields', function () {
+      var idx = new Index({ fieldName: ['tf', 'tg'] })
+        , doc1 = { a: 5, tf: 'hello', tg: 'world' }
+        , doc2 = { a: 8, tf: 'hello', tg: 'bloup' }
+        , doc3 = { a: 2, tf: 'bloup', tg: 'bloup' }
+        ;
+
+      idx.insert(doc1);
+      idx.insert(doc2);
+      idx.insert(doc3);
+
+
+      // The underlying BST now has 3 nodes which contain the docs where it's expected
+      idx.tree.getNumberOfKeys().should.equal(3);
+      assert.deepEqual(idx.tree.search({tf: 'hello', tg: 'world'}), [{ a: 5, tf: 'hello', tg: 'world' }]);
+      assert.deepEqual(idx.tree.search({tf: 'hello', tg: 'bloup'}), [{ a: 8, tf: 'hello', tg: 'bloup' }]);
+      assert.deepEqual(idx.tree.search({tf: 'bloup', tg: 'bloup'}), [{ a: 2, tf: 'bloup', tg: 'bloup' }]);
+
+      // The nodes contain pointers to the actual documents
+      idx.tree.search({tf: 'hello', tg: 'bloup'})[0].should.equal(doc2);
+      idx.tree.search({tf: 'bloup', tg: 'bloup'})[0].a = 42;
+      doc3.a.should.equal(42);
+
+    });
+
     it('Inserting twice for the same fieldName in a unique index will result in an error thrown', function () {
       var idx = new Index({ fieldName: 'tf', unique: true })
         , doc1 = { a: 5, tf: 'hello' }
@@ -44,6 +69,16 @@ describe('Indexes', function () {
       (function () { idx.insert(doc1); }).should.throw();
     });
 
+    it('Inserting twice for the same compound fieldName in a unique index will result in an error thrown', function () {
+      var idx = new Index({ fieldName: ['tf', 'tg'], unique: true })
+        , doc1 = { a: 5, tf: 'hello', tg: 'world' }
+        ;
+
+      idx.insert(doc1);
+      idx.tree.getNumberOfKeys().should.equal(1);
+      (function () { idx.insert(doc1); }).should.throw();
+    });
+
     it('Inserting twice for a fieldName the docs dont have with a unique index results in an error thrown', function () {
       var idx = new Index({ fieldName: 'nope', unique: true })
         , doc1 = { a: 5, tf: 'hello' }
@@ -223,6 +258,33 @@ describe('Indexes', function () {
 
     });   // ==== End of 'Array fields' ==== //
 
+    describe('Compound Indexes', function () {
+
+      it('Supports arrays of fieldNames', function () {
+        var idx = new Index({ fieldName: ['tf', 'tf2'] })
+          , doc1 = { a: 5, tf: 'hello', tf2: 7}
+          , doc2 = { a: 8, tf: 'hello', tf2: 6}
+          , doc3 = { a: 2, tf: 'bloup', tf2: 3}
+          ;
+
+        idx.insert(doc1);
+        idx.insert(doc2);
+        idx.insert(doc3);
+
+        // The underlying BST now has 3 nodes which contain the docs where it's expected
+        idx.tree.getNumberOfKeys().should.equal(3);
+        assert.deepEqual(idx.tree.search({tf: 'hello', tf2: 7}), [{ a: 5, tf: 'hello', tf2: 7}]);
+        assert.deepEqual(idx.tree.search({tf: 'hello', tf2: 6}), [{ a: 8, tf: 'hello', tf2: 6}]);
+        assert.deepEqual(idx.tree.search({tf: 'bloup', tf2: 3}), [{ a: 2, tf: 'bloup', tf2: 3}]);
+
+        // The nodes contain pointers to the actual documents
+        idx.tree.search({tf: 'hello', tf2: 6})[0].should.equal(doc2);
+        idx.tree.search({tf: 'bloup', tf2: 3})[0].a = 42;
+        doc3.a.should.equal(42);
+      });
+
+    });
+
   });   // ==== End of 'Insertion' ==== //
 
 

test/model.test.js

@@ -949,7 +949,64 @@ describe('Model', function () {
     });
 
   });   // ==== End of 'Comparing things' ==== //
+
+  describe('Compound Comparing Things', function () {
 
+    it('undefined is the smallest', function () {
+      var otherStuff = [{a: 0, b:0}, {a: 0, b:1}, {a: 1, b:0}, {a: 1, b:1}, {b:0}, {a: 0}];
+      var fields = ['a', 'b'];
+
+      model.compoundCompareThings(fields)(undefined, undefined).should.equal(0);
+
+      otherStuff.forEach(function (stuff) {
+        model.compoundCompareThings(fields)(undefined, stuff).should.equal(-1);
+        model.compoundCompareThings(fields)(stuff, undefined).should.equal(1);
+      });
+    });
+
+    it('Then null', function () {
+      var otherStuff = [{a: 0, b:0}, {a: 0, b:1}, {a: 1, b:0}, {a: 1, b:1}, {b:0}, {a: 0}];
+      var fields = ['a', 'b'];
+
+      model.compoundCompareThings(fields)(null, null).should.equal(0);
+
+      otherStuff.forEach(function (stuff) {
+        model.compoundCompareThings(fields)(null, stuff).should.equal(-1);
+        model.compoundCompareThings(fields)(stuff, null).should.equal(1);
+      });
+    });
+
+    it('Then properties should be compared, returning the first non-zero result, or zero', function () {
+      var examples = [{a: 0, b:0}, {a: 0, b:1}, {a: 1, b:0}, {a: 1, b:1}, {b:0}, {a: 0}];
+      var fields = ['a', 'b'];
+
+      var less = function (a, b) {
+        assert.equal(model.compoundCompareThings(fields)(examples[a], examples[b]), -1);
+      };
+      var more = function (a, b) {
+        assert.equal(model.compoundCompareThings(fields)(examples[b], examples[a]), 1);
+      };
+
+      less(0, 1);
+      less(0, 2);
+      less(1, 2);
+      less(2, 3);
+      less(5, 0);
+      less(4, 0);
+
+      more(0, 1);
+      more(0, 2);
+      more(1, 2);
+      more(2, 3);
+      more(5, 0);
+      more(4, 0);
+
+      examples.forEach(function (example) {
+        assert.equal(model.compoundCompareThings(fields)(example, model.deepCopy(example)), 0);
+      });
+    });
+
+  });   // ==== End of 'Compound Comparing things' ==== //
 
   describe('Querying', function () {