Function to find all large docs in Mongo
@@ -213,6 +213,42 @@ var oak = (function(global){
return result;
};
+ /**
+ * Find and dumps _id of all documents where the document size exceeds
+ * 15MB size. It also dumps progress info after every 10k docs.
+ *
+ * The ids can be found by grepping for '^id|' pattern
+ *
+ * > oak.dumpLargeDocIds({db: "aem-author"})
+ *
+ * @param {object} options pass optional parameters for host, port, db, and filename
+ */
+ api.dumpLargeDocIds = function (options) {
+ var count = 0;
+ var ids = [];
+ db.nodes.find().forEach(function (doc) {
+ var size = Object.bsonsize(doc);
+ if (size > 15 * 1024 * 1024) {
+ print("id|" + doc._id);
+ ids.push(doc._id)
+ }
+ if (++count % 10000 === 0) {
+ print("Traversed #" + count)
+ }
+ });
+
+ print("Number of large documents : " + ids.length);
+
+ //Dump the export command to dump all such large docs
+ if (ids.length > 0) {
+ var query = JSON.stringify({_id: {$in: ids}});
+ print("Using following export command to tweak the output");
+ options = options || {};
+ options.db = db.getName();
+ print(createExportCommand(query, options));
+ }
+ };
+
/**
* Converts the given Revision String into a more human readable version,
* which also prints the date.
@@ -579,23 +615,7 @@ var oak = (function(global){
*/
api.printMongoExportCommand = function (path, options) {
- options = options || {};
- var host = options.host || "127.0.0.1";
- var port = options.port || "27017";
- var db = options.db || "oak";
- var filename = options.filename || "all-required-nodes.json"
-
- var query = JSON.stringify(getDocAndHierarchyQuery(path));
-
- var mongoExportCommand = "mongoexport"
- + " --host " + host
- + " --port " + port
- + " --db " + db
- + " --collection nodes"
- + " --out " + filename
- + " --query '" + query + "'";
-
- return mongoExportCommand;
+ return createExportCommand(JSON.stringify(getDocAndHierarchyQuery(path)), options);
};
/**
@@ -641,6 +661,22 @@ var oak = (function(global){
//~--------------------------------------------------< internal >
+ var createExportCommand = function (query, options) {
+ options = options || {};
+ var host = options.host || "127.0.0.1";
+ var port = options.port || "27017";
+ var db = options.db || "oak";
+ var filename = options.filename || "all-required-nodes.json"
+
+ return "mongoexport"
+ + " --host " + host
+ + " --port " + port
+ + " --db " + db
+ + " --collection nodes"
+ + " --out " + filename
+ + " --query '" + query + "'";
+ };
+
var checkOrFixDeepHistory = function(path, fix, prepare, verbose) {
if (prepare) {
// not issuing any header at all
