Skip to content

Commit

Permalink
feat(edit-logs): cleanup diff on objectId's single byte
Browse files Browse the repository at this point in the history
  • Loading branch information
@naholyr
naholyr committed Sep 6, 2017
1 parent 1ff09dc commit 509b003
Show file tree
Hide file tree
Showing 2 changed files with 179 additions and 34 deletions.
201 changes: 168 additions & 33 deletions scripts/migration/migrateEditLogUnexpectedFields.js
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,43 @@
const { connect } = require('../../server/lib/model')
const { EditLog, flattenDiff, cleanupData } = require('../../server/lib/edit-logs')
const { set, get } = require('lodash')
const chalk = require('chalk')


const cleanupAll = logs => {
//console.log('COUNT LOGS', logs.length)
return logs.map(cleanupOne).filter(log => log !== null)
let logsToSave = {}
const fixLog = log =>
Promise.resolve(cleanupOne(log, logs))
.then(modified => modified && (logsToSave[log.id] = log))
return Promise.all(logs.map(fixLog)).then(() => Object.values(logsToSave))
}

const cleanupOne = log => {
const cleanupOne = (log, logs) => {
//console.log('CHECK LOG', log)
switch(log.action) {
case 'delete': return null
case 'create': {
const data = fixData(log.data)
return data ? Object.assign(log, { data }) : null
}
case 'update': {
const diff = fixDiff(log.diff)
return diff ? Object.assign(log, { diff }) : null
}
case 'create': return fixData(log, logs)
case 'update': return fixDiff(log, logs)
default: return null
}
}

const fixDiff = diff => {
const PATH_SEPARATOR = '.'
const pathString = d => d && d.path && d.path.join(PATH_SEPARATOR)
const pathDir = d => d && d.path && d.path.slice(0, d.path.length - 1).join(PATH_SEPARATOR)

const fixDiff = (log, logs) => {
let modified = false
let changes = log.diff

// First, flatten those weird diffs
const originalDiff = diff
diff = flattenDiff(diff)
// First, flatten those weird diffs (if necessary)
let shouldFlatten = changes.some(Array.isArray)
if (shouldFlatten) {
changes = flattenDiff(changes)
// console.log(chalk.blue.bold('MODIFY: flatten'))
modified = true
}

// MongoId instance: 2 changes FIELD._bsontype = 'ObjectID' + FIELD.id = buffer → replace with hex string
/*
Expand All @@ -43,30 +51,31 @@ const fixDiff = diff => {
return { field: 'rhs', value: d2.rhs.buffer.toString('hex') }
}
}
for (let i = 0; i < diff.length - 1; i++) {
const curr = diff[i]
const next = diff[i + 1]
for (let i = 0; i < changes.length - 1; i++) {
const curr = changes[i]
const next = changes[i + 1]
const c1 = mongoIdCouple(curr, next)
if (c1) {
next[c1.field] = c1.value
next.path.pop() // FIELD.id → FIELD
diff[i] = null
changes[i] = null
// console.log(chalk.blue.bold('MODIFY: ObjectID to string'))
modified = true
}
}

// Populated data: FIELD._id exists as ObjectID → replace with hex string, remove all other FIELD.*
const pathDir = d => d.path.slice(0, d.path.length - 1).join('/')
const idDiffs = diff.filter(d => d && d.path.length > 1 && d.path[d.path.length - 1] === '_id')
const idDiffs = changes.filter(d => d && d.path.length > 1 && d.path[d.path.length - 1] === '_id')
if (idDiffs.length > 0) {
// console.log(chalk.blue.bold('MODIFY: unpopulate'))
modified = true
}
idDiffs.forEach(d => {
if (!d) {
return
}
const dir = pathDir(d)
diff = diff.filter(dd => {
changes = changes.filter(dd => {
if (dd && pathDir(dd) === dir) {
// FIELD._id → mutate
if (dd.path[dd.path.length - 1] === '_id') {
Expand Down Expand Up @@ -108,19 +117,19 @@ const fixDiff = diff => {
{"kind":"N","path":["academicMemberships",0,"organization"],"rhs":"5894889a11d0d100381bcc97"}
*/
// Let's refactor deleted/new that cancel each other
diff.forEach((dDiff, dDiffIndex) => {
changes.forEach((dDiff, dDiffIndex) => {
if (dDiff && dDiff.kind === 'D' && dDiff.lhs) {
const dp = dDiff.path.join('/')
const nDiffIndex = diff.findIndex(d => d && d.kind === 'N' && dp === d.path.join('/') && d.rhs)
const dp = pathString(dDiff)
const nDiffIndex = changes.findIndex(d => d && d.kind === 'N' && dp === pathString(d) && d.rhs)
if (nDiffIndex !== -1) {
const nDiff = diff[nDiffIndex]
const nDiff = changes[nDiffIndex]
// Now compare the both elements: same value = delete both, different value = make it a 'edit' change
if (dDiff.lhs === nDiff.rhs) {
diff[dDiffIndex] = null
diff[nDiffIndex] = null
changes[dDiffIndex] = null
changes[nDiffIndex] = null
// console.log('FOUND CORRESPONDING D/N', nDiff, dDiff)
} else {
diff[nDiffIndex] = null
changes[nDiffIndex] = null
dDiff.kind = 'E'
dDiff.rhs = nDiff.rhs
// console.log('FOUND EDIT D/N', dDiff)
Expand All @@ -130,17 +139,143 @@ const fixDiff = diff => {
})

// Remove filtered changes
diff = diff.filter(d => d !== null)
changes = changes.filter(d => d !== null)

// if (modified) console.log({ originalDiff, diff })

return modified && diff
log.diff = changes

return fixObjectIDDiffs(log, logs, modified)
}

const isInnerChangeInObjectID = d =>
d.path[d.path.length - 2] === 'id' && Number(d.path[d.path.length - 1]) >= 0

const isFull = changes => {
// Collection of changes is full iff the 12 first changes are all edits on keys 0 to 11
const edits = changes.filter(d => d.kind === 'E' && Number(d.path[d.path.length - 1]) >= 0)
if (edits.length !== 12) {
//console.log(chalk.yellow('Ooops, wrong number of edits: ' + edits.length))
return false
}
const keys = edits.map(d => d.path[d.path.length - 1]).sort()
if (keys.join('.') !== '0.1.10.11.2.3.4.5.6.7.8.9') {
//console.log(chalk.yellow('Oooops, wrong keys met: ' + String(keys)))
return false
}
return true
}

const fixData = data => cleanupData(data, false, true)
function fakeObjectIdToString () {
return this.id.toString('hex')
}

const fixObjectIDDiffs = (log, logs, modified) => {
// When ObjectID is changed, that dumbass stored FIELD.id.INDEX changes one by one...
const innerChangesInObjectIDs = log.diff.filter(isInnerChangeInObjectID)

if (innerChangesInObjectIDs.length === 0) {
return modified
}

// Example: [ 'personalActivities.8.organizations.0.id', 'personalActivities.9.organizations.0.id', 'personalActivities.10.organizations.0.id' ]
const paths = [...new Set(innerChangesInObjectIDs.map(d => pathDir(d)))]

// Remove all those changes from initial diff, we'll add them back after
log.diff = log.diff.filter(d => !isInnerChangeInObjectID(d))

// There are diffs like 'PATH.0', 'PATH.1', etc… Replay the whole history of changes
// to convert them into simple string replacements when possible
//console.log('SHOULD FIX OBJECT ID DIFF IN LOG', log.id, innerChangeInObjectIDs.length)
//console.log('FIND ALL DIFFS FOR CORRESPONDING ITEM, AND REPLAY (so cool)')
const itemId = String(log.item)
paths.forEach(path => {
const parentPath = path.split(PATH_SEPARATOR).slice(0, -1).join(PATH_SEPARATOR)

console.log(chalk.blue.bold('ObjectID inner diff: Log#%s %s#%s %s'), log.id, log.model, log.item, path)
const logPathChanges = innerChangesInObjectIDs.filter(d => pathDir(d) === path)

// Case 1: the log contains a full change of id (keys 0-11)
if (isFull(logPathChanges)) {
let lBits = []
let rBits = []
logPathChanges.forEach(d => {
if (d.kind === 'E' && pathDir(d) === path) {
const index = Number(d.path[d.path.length - 1])
lBits[index] = d.lhs
rBits[index] = d.rhs
}
})
log.diff.push({
kind: 'E',
path: logPathChanges[0].path.slice(0, -1),
lhs: new Buffer(lBits).toString('hex'),
rhs: new Buffer(rBits).toString('hex'),
})
modified = true
return // next path
}

const history = logs
.filter(l => String(l.item) === itemId && l.date < log.date)
// Note: at this point, log.diff does not contain our inner changes, so we need to add it manually
.concat([Object.assign({}, log, { diff: innerChangesInObjectIDs })])
if (history[0].action === 'create') {
console.log(chalk.green('OK: we have an initial creation, replay history')) // eslint-disable-line no-console
let data = history[0].data
const getValue = () => get(data, parentPath)
let lhs = String(getValue())
//console.log(require('util').inspect(data,{colors:true,depth:10}))
for (let i = 1; i < history.length; i++) {
(history[i].diff || []).forEach(change => {
const currPath = pathString(change)
//console.log(currPath)
if (!path.startsWith(currPath) && !currPath.startsWith(path)) {
return // skip uninteresting change
}
//console.log(change)
// Note: shit can happen when previous diff had a stringified ObjectID and next change is about changing a bit of it
// Detect and handle this edge case now
if (currPath.startsWith(path) && typeof getValue() !== 'object') {
//console.log(chalk.red('HANDLE EDGE CASE'))
set(data, parentPath, { id: Buffer.from(getValue(), 'hex'), toString: fakeObjectIdToString })
}
data = EditLog.applyChange(data, change)
if (path.startsWith(currPath)) {
// Full change
//console.log('FULL CHANGE')
lhs = String(getValue())
}
//console.log({lhs})
})
}
//console.log({lhs, rhs: String(getValue())})
log.diff.push({ kind: 'E', path: parentPath.split(PATH_SEPARATOR), lhs, rhs: String(getValue()) })
} else {
if (history[history.length - 1].action === 'delete') {
console.log(chalk.red('FAIL: not enough data to create meaningful changes (item deleted in the end)')) // eslint-disable-line no-console
} else {
console.log(chalk.red.bold('FAIL: not enough data to create meaningful changes')) // eslint-disable-line no-console
}
log.diff.push({ kind: 'E', path: parentPath.split(PATH_SEPARATOR), lhs: 'N/A', rhs: 'N/A' })
}

modified = true
})

return modified
}

const fixData = log => {
const data = cleanupData(log.data, false, true)
if (data) {
log.data = data
}
return !!data
}

connect()
.then(() => EditLog.find())
.then(() => EditLog.find().sort({ date: 1 }))
.then(cleanupAll)
.then(logs => {
console.log('Modified logs', logs.length) // eslint-disable-line no-console
Expand All @@ -159,7 +294,7 @@ connect()
return allSaved.then(() => {
process.stdout.write('\n')
process.stderr.write('\n')
console.log('[OK] %s EditLog entries updated', ok)
console.log('[OK] %s EditLog entries updated', ok) // eslint-disable-line no-console
if (errors.length > 0) {
errors.forEach(([ log, e ]) => {
console.error('[ERR] Log #%s: %s', log.id, e.message) // eslint-disable-line no-console
Expand Down
12 changes: 11 additions & 1 deletion server/lib/edit-logs.js
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
'use strict'

const mongoose = require('mongoose')
const deepDiff = require('deep-diff').diff
const { diff: deepDiff, applyChange } = require('deep-diff')
const chalk = require('chalk')
const removeEmptyFields = require('./remove-empty-fields')
const { get, set } = require('lodash')


const EditLogSchema = new mongoose.Schema({
Expand Down Expand Up @@ -42,6 +43,15 @@ const EditLogSchema = new mongoose.Schema({
}
})

EditLogSchema.static('applyChange', (source, change) => {
const target = Object.assign({}, source)
if (change.kind === 'A' && !get(target, change.path)) {
set(target, change.path, [])
}
applyChange(target, source, change)
return target
})


const EditLog = mongoose.model('EditLog', EditLogSchema)

Expand Down

0 comments on commit 509b003

Please sign in to comment.