diff --git a/config.example.js b/config.example.js index aee49f7..cef4db9 100644 --- a/config.example.js +++ b/config.example.js @@ -8,9 +8,20 @@ module.exports = { mongoURL: "mongodb://:@mongourl.com:27017/database_name", serverURL: "https://api.customparseserver.com/parse", filesToTransfer: 'parseOnly', + renameFiles: false, renameInDatabase: false, transferTo: 'filesystem', + // If false, will migrate files array as well + // Note: The parameters renameFiles and renameInDatabase must be true + onlyFiles: false, + + // Extra collection and fields that use files array, use to migrate them as well + // Note: Use if onlyFiles equals false + extraFields: { + collectionName: ['fieldNameOne', 'fieldNameTwo'] + }, + // For filesystem configuration filesystemPath: './downloaded_files', diff --git a/lib/index.js b/lib/index.js index 0d1d5f1..45b9929 100644 --- a/lib/index.js +++ b/lib/index.js @@ -5,6 +5,8 @@ var schemas = require('./schemas'); var transfer = require('./transfer'); var questions = require('./questions.js'); +var stringFileRegex = new RegExp(/^(?=.*\bhttp\b)(?=.*\bparsetfss\b)(?=.*\btfss-\b).*$/); + module.exports = initialize; function initialize(config) { @@ -26,7 +28,7 @@ function initialize(config) { Parse.serverURL = config.serverURL; return transfer.init(config); }).then(function() { - return getAllFileObjects(); + return getAllFileObjects(config); }).then(function(objects) { return transfer.run(objects); }).then(function() { @@ -38,11 +40,15 @@ function initialize(config) { }); } -function getAllFileObjects() { +function getAllFileObjects(config) { console.log("Fetching schema..."); return schemas.get().then(function(res){ console.log("Fetching all objects with files..."); - var schemasWithFiles = onlyFiles(res); + if (config.onlyFiles){ + var schemasWithFiles = onlyFiles(res); + } else { + var schemasWithFiles = stringAndFilesArray(res, config.extraFields); + } return Promise.all(schemasWithFiles.map(getObjectsWithFilesFromSchema)); }).then(function(results) { var files = results.reduce(function(c, r) { @@ -57,16 +63,40 @@ function getAllFileObjects() { function onlyFiles(schemas) { return schemas.map(function(schema) { - var fileFields = Object.keys(schema.fields).filter(function(key){ - var value = schema.fields[key]; - return value.type == "File"; - }); - if (fileFields.length > 0) { - return { - className: schema.className, - fields: fileFields - } - } + var fileFields = Object.keys(schema.fields).filter(function(key){ + var value = schema.fields[key]; + return value.type == "File"; + }); + if (fileFields.length > 0) { + return { + className: schema.className, + fields: fileFields + } + } + }).filter(function(s){ return s != undefined }) +} + +function stringAndFilesArray(schemas, extra) { + return schemas.map(function(schema) { + var fileFields = Object.keys(schema.fields).filter(function(key){ + var value = schema.fields[key]; + + var is_valid = false; + + if (extra[schema.className] != undefined){ + if (extra[schema.className].indexOf(key) != -1){ + is_valid = true; + } + } + + return is_valid; + }); + if (fileFields.length > 0) { + return { + className: schema.className, + fields: fileFields + } + } }).filter(function(s){ return s != undefined }) } @@ -95,25 +125,61 @@ function getObjectsWithFilesFromSchema(schema) { query.limit(1000); var checks = schema.fields.map(function(field) { - return new Parse.Query(schema.className).exists(field); + return new Parse.Query(schema.className).exists(field); }); query._orQuery(checks); return getAllObjects(query).then(function(results) { return results.reduce(function(current, result){ - return current.concat( - schema.fields.map(function(field){ + var fileResults = []; + schema.fields.map(function(field){ + if (Array.isArray(result.get(field))) { + for (var i = 0; i < result.get(field).length; i++) { + if (result.get(field)[i].name !== undefined){ + var fName = result.get(field)[i] ? result.get(field)[i].name() : 'DELETE'; + var fUrl = result.get(field)[i] ? result.get(field)[i].url() : 'DELETE'; + + fileResults.push({ + className: schema.className, + objectId: result.id, + fieldName: field, + fileName: fName, + url: fUrl, + i: i + }); + } + } + } else if ((typeof result.get(field) === 'string') && (stringFileRegex.test(result.get(field)))){ + var fUrl = result.get(field) ? result.get(field) : 'DELETE'; + if (fUrl != 'DELETE') { + var splitString = fUrl.split("/"); + var fName = splitString[splitString.length-1]; + } else { + var fName = 'DELETE'; + } + fileResults.push({ + className: schema.className, + objectId: result.id, + fieldName: field, + fileName: fName, + url: fUrl, + i: 0, + type: 'string' + }) + } else if (typeof result.get(field) != 'string') { var fName = result.get(field) ? result.get(field).name() : 'DELETE'; var fUrl = result.get(field) ? result.get(field).url() : 'DELETE'; - return { + fileResults.push({ className: schema.className, objectId: result.id, fieldName: field, fileName: fName, - url: fUrl - } - }) - ); + url: fUrl, + i: 0 + }) + } + }); + return current.concat(fileResults); }, []); }); } diff --git a/lib/transfer.js b/lib/transfer.js index e21cb9b..2860749 100644 --- a/lib/transfer.js +++ b/lib/transfer.js @@ -131,7 +131,7 @@ function _processFiles(files, handler) { var asyncLimit = config.asyncLimit || 5; return new Promise(function(resolve, reject) { async.eachOfLimit(files, asyncLimit, function(file, index, callback) { - process.stdout.write('Processing '+(index+1)+'/'+files.length+'\r'); + process.stdout.write('Processing '+(index+1)+'/'+files.length+'\r'); file.newFileName = _createNewFileName(file.fileName); if (_shouldTransferFile(file)) { _transferFile(file).then(callback, callback); @@ -156,18 +156,72 @@ function _changeDBFileField(file) { if (file.fileName == file.newFileName || !config.renameInDatabase) { return resolve(); } + + if (config.transferTo === 'filesystem'){ + var _path = config.filesAdapter._filesDir + "/" + file.newFileName; + } else if (config.transferTo === 's3'){ + var _path = "https://" + config.filesAdapter._bucket + "." + config.filesAdapter._s3Client.config.endpoint + "/" + config.filesAdapter._bucketPrefix + file.newFileName; + } + var update = {$set:{}}; - update.$set[file.fieldName] = file.newFileName; - db.collection(file.className).update( - { _id : file.objectId }, - update, - function(error, result ) { - if (error) { - return reject(error); - } - resolve(); + + if ((config.extraFields[file.className] != undefined) && (config.extraFields[file.className].indexOf(file.fieldName) != -1)) { + + if (file.type === 'string') { + update.$set[file.fieldName] = _path; + db.collection(file.className).update( + { _id : file.objectId }, + update, + function(error, result) { + if (error) { + return reject(error); + } + resolve(); + } + ) + } else { + db.collection(file.className).findOne({_id: file.objectId}).then(function(result){ + var images = result[file.fieldName]; + + images[file.i] = { + __type: 'File', + name: file.newFileName, + url: _path + }; + + update.$set[file.fieldName] = images; + db.collection(file.className).update( + { _id : file.objectId }, + update, + function(error, result) { + if (error) { + return reject(error); + } + resolve(); + } + ); + }); } - ); + + } else { + + update.$set[file.fieldName] = { + __type: 'File', + name: file.newFileName, + url: _path + }; + + db.collection(file.className).update( + { _id : file.objectId }, + update, + function(error, result ) { + if (error) { + return reject(error); + } + resolve(); + } + ); + } }); }