Feature leveldb persistence support and authorization

Dockerfile

@@ -1,9 +1,9 @@
 FROM node:12-alpine
 
-RUN mkdir -p /home/node/app/node_modules && chown -R node:node /home/node/app
-WORKDIR /home/node/app
-COPY package*.json ./
-USER node
-RUN npm install
-COPY --chown=node:node . .
-CMD [ "npm", "start" ]
+WORKDIR /home/node/
+COPY ./package*.json ./
+RUN yarn install && yarn cache clean --force
+ENV PATH /home/node/node_modules/.bin:$PATH
+WORKDIR /home/node/app/
+COPY . .
+CMD [ "npm", "start" ]

bin/collaborate-editing-server.js

@@ -0,0 +1,32 @@
+/**
+ * @type {any}
+ */
+const WebSocket = require('ws')
+const http = require('http')
+const wss = new WebSocket.Server({ noServer: true })
+const setupWSConnection = require('./utils.js').setupWSConnection
+
+const host = process.env.YWEBSOCKET_HOST || 'localhost'
+const port = process.env.YWEBSOCKET_PORT || 1234
+
+const server = http.createServer((request, response) => {
+  response.writeHead(200, { 'Content-Type': 'application/json' })
+  response.end(JSON.stringify({ status: 'ok' }))
+})
+
+wss.on('connection', setupWSConnection)
+
+server.on('upgrade', (request, socket, head) => {
+  // You may check auth of request here..
+  /**
+    * @param {any} ws
+    */
+  const handleAuth = ws => {
+    wss.emit('connection', ws, request)
+  }
+  wss.handleUpgrade(request, socket, head, handleAuth)
+})
+
+server.listen({ host, port })
+
+console.log(`collabortate editing server is running at '${host}' on port ${port}`)

bin/note-synchronizer-server.js

@@ -0,0 +1,29 @@
+const redisPersistenceBridge = require('./redis-persistence-bridge')
+
+const http = require('http')
+
+const host = process.env.YPERSISTENCE_SYNCHRONIZER_HOST || 'localhost'
+const port = process.env.YPERSISTENCE_SYNCHRONIZER_PORT || 1235
+
+const server = http.createServer((request, response) => {
+  if (request.method === 'POST') {
+    let body = ''
+
+    request.on('data', function (data) {
+      body += data
+    })
+
+    request.on('end', function () {
+      const post = JSON.parse(body)
+      for (const sharedObject of post.sharedObjects) {
+        redisPersistenceBridge.saveQuillDeltaToRedis(post.documentName, sharedObject.name, sharedObject.delta)
+      }
+    })
+  }
+  response.writeHead(200, { 'Content-Type': 'application/json' })
+  response.end(JSON.stringify({ status: 'ok' }))
+})
+
+server.listen({ host, port })
+
+console.log(`note synchronizer is running at '${host}' on port ${port}`)

bin/redis-persistence-bridge.js

@@ -0,0 +1,21 @@
+const yjs = require('yjs')
+const RedisPersistence = require('y-redis').RedisPersistence
+
+const REDIS_HOST = process.env.YPERSISTENCE_REDIS_HOST || 'localhost'
+const REDIS_PORT = process.env.YPERSISTENCE_REDIS_PORT || 6379
+
+const redisPersistence = new RedisPersistence({
+  redisOpts: { host: REDIS_HOST, port: REDIS_PORT }
+})
+
+// needs to be tested
+module.exports = {
+  saveQuillDeltaToRedis: async function (documentName, sharedObjectName, delta) {
+    await redisPersistence.clearDocument(documentName) // need to ivestigate how to clear single shared object, not entire doc
+    const doc = new yjs.Doc()
+    const persistedDoc = redisPersistence.bindState(documentName, doc)
+    await persistedDoc.synced
+    doc.getText(sharedObjectName).applyDelta(delta)
+    await persistedDoc.synced
+  }
+}

bin/server.js

@@ -1,34 +1,4 @@
 #!/usr/bin/env node
 
-/**
- * @type {any}
- */
-const WebSocket = require('ws')
-const http = require('http')
-const wss = new WebSocket.Server({ noServer: true })
-const setupWSConnection = require('./utils.js').setupWSConnection
-
-const host = process.env.HOST || 'localhost'
-const port = process.env.PORT || 1234
-
-const server = http.createServer((request, response) => {
-  response.writeHead(200, { 'Content-Type': 'text/plain' })
-  response.end('okay')
-})
-
-wss.on('connection', setupWSConnection)
-
-server.on('upgrade', (request, socket, head) => {
-  // You may check auth of request here..
-  /**
-   * @param {any} ws
-   */
-  const handleAuth = ws => {
-    wss.emit('connection', ws, request)
-  }
-  wss.handleUpgrade(request, socket, head, handleAuth)
-})
-
-server.listen({ host, port })
-
-console.log(`running at '${host}' on port ${port}`)
+require('./collaborate-editing-server')
+require('./note-synchronizer-server')

bin/transfer-data-from-mysql-to-redis.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+
+const redisPersistenceBridge = require('./redis-persistence-bridge')
+
+const mysql = require('mysql2')
+
+const MYSQL_HOST = process.env.MYSQL_HOST || 'localhost'
+const MYSQL_USER = process.env.MYSQL_USER || 'root'
+const MYSQL_PASSWORD = process.env.MYSQL_PASSWORD || ''
+const MYSQL_DATABASE = process.env.MYSQL_DATABASE || 'database'
+const MYSQL_TABLE = process.env.MYSQL_TABLE || 'table'
+const MYSQL_CONTENT_FIELDS = process.env.MYSQL_CONTENT_FIELDS || 'content'
+const MYSQL_KEY_FIELD = process.env.MYSQL_KEY_FIELD || 'id'
+const MYSQL_PORT = process.env.MYSQL_PORT || 3306
+
+const mysqlConnection = mysql.createConnection({
+  host: MYSQL_HOST,
+  user: MYSQL_USER,
+  database: MYSQL_DATABASE,
+  password: MYSQL_PASSWORD,
+  port: MYSQL_PORT
+})
+
+mysqlConnection.connect(function (error) {
+  if (error) throw error
+})
+
+mysqlConnection.query(
+  `SELECT ${MYSQL_KEY_FIELD}, ${MYSQL_CONTENT_FIELDS} FROM ${MYSQL_TABLE}`,
+  function (error, rows, fields) {
+    if (error) {
+      throw error
+    }
+    (async () => {
+      processMysqlDeltas(rows, fields)
+    })()
+  }
+)
+
+async function processMysqlDeltas (rows, fields) {
+  for (const row of rows) {
+    const key = row[MYSQL_KEY_FIELD]
+    for (const field of fields) {
+      if (field.name === MYSQL_KEY_FIELD) {
+        continue
+      }
+      const delta = JSON.parse(row[field.name])
+      await redisPersistenceBridge.saveQuillDeltaToRedis(key, field.name, delta)
+    }
+  }
+}

bin/utils.js

@@ -22,25 +22,36 @@ const wsReadyStateClosed = 3 // eslint-disable-line
 
 // disable gc when using snapshots!
 const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0'
-const persistenceDir = process.env.YPERSISTENCE
+const persistenceStorage = process.env.YPERSISTENCE_STORAGE || 'level-db'
+const persistenceHost = process.env.YPERSISTENCE_REDIS_HOST || 'localhost'
+const persistencePort = process.env.YPERSISTENCE_REDIS_PORT || 6379
+const persistencePath = process.env.YPERSISTENCE_LEVELDB_PATH || 'db'
+
 /**
  * @type {{bindState: function(string,WSSharedDoc):void, writeState:function(string,WSSharedDoc):Promise<any>, provider: any}|null}
  */
 let persistence = null
-if (typeof persistenceDir === 'string') {
-  console.info('Persisting documents to "' + persistenceDir + '"')
+if (typeof persistencePath === 'string') {
   // @ts-ignore
-  const LeveldbPersistence = require('y-leveldb').LeveldbPersistence
-  const ldb = new LeveldbPersistence(persistenceDir)
+  let persistenceDB
+  if (persistenceStorage === 'redis') {
+    console.info('Persisting documents to "' + persistenceHost + '"')
+    const RedisPersistence = require('y-redis').RedisPersistence
+    persistenceDB = new RedisPersistence({ redisOpts: { host: persistenceHost, port: persistencePort } })
+  } else {
+    console.info('Persisting documents to "' + persistencePath + '"')
+    const LeveldbPersistence = require('y-leveldb').LeveldbPersistence
+    persistenceDB = new LeveldbPersistence(persistencePath)
+  }
   persistence = {
-    provider: ldb,
+    provider: persistenceDB,
     bindState: async (docName, ydoc) => {
-      const persistedYdoc = await ldb.getYDoc(docName)
+      const persistedYdoc = await persistenceDB.getYDoc(docName)
       const newUpdates = Y.encodeStateAsUpdate(ydoc)
-      ldb.storeUpdate(docName, newUpdates)
+      persistenceDB.storeUpdate(docName, newUpdates)
       Y.applyUpdate(ydoc, Y.encodeStateAsUpdate(persistedYdoc))
       ydoc.on('update', update => {
-        ldb.storeUpdate(docName, update)
+        persistenceDB.storeUpdate(docName, update)
       })
     },
     writeState: async (docName, ydoc) => {}

package.json

@@ -47,7 +47,10 @@
   "dependencies": {
     "lib0": "^0.2.35",
     "lodash.debounce": "^4.0.8",
-    "y-protocols": "^1.0.4"
+    "y-protocols": "^1.0.4",
+    "ws": "^6.2.1",
+    "y-leveldb": "^0.1.0",
+    "y-redis": "^1.0.2"
   },
   "devDependencies": {
     "rollup": "^1.32.1",
@@ -58,9 +61,5 @@
   },
   "peerDependenies": {
     "yjs": "^13.5.0"
-  },
-  "optionalDependencies": {
-    "ws": "^6.2.1",
-    "y-leveldb": "^0.1.0"
   }
 }