diff --git a/Dockerfile b/Dockerfile index 42acd06a..0163065e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM node:12-alpine -RUN mkdir -p /home/node/app/node_modules && chown -R node:node /home/node/app -WORKDIR /home/node/app -COPY package*.json ./ -USER node -RUN npm install -COPY --chown=node:node . . -CMD [ "npm", "start" ] \ No newline at end of file +WORKDIR /home/node/ +COPY ./package*.json ./ +RUN yarn install && yarn cache clean --force +ENV PATH /home/node/node_modules/.bin:$PATH +WORKDIR /home/node/app/ +COPY . . +CMD [ "npm", "start" ] diff --git a/bin/collaborate-editing-server.js b/bin/collaborate-editing-server.js new file mode 100644 index 00000000..cbddd159 --- /dev/null +++ b/bin/collaborate-editing-server.js @@ -0,0 +1,32 @@ +/** + * @type {any} + */ +const WebSocket = require('ws') +const http = require('http') +const wss = new WebSocket.Server({ noServer: true }) +const setupWSConnection = require('./utils.js').setupWSConnection + +const host = process.env.YWEBSOCKET_HOST || 'localhost' +const port = process.env.YWEBSOCKET_PORT || 1234 + +const server = http.createServer((request, response) => { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) +}) + +wss.on('connection', setupWSConnection) + +server.on('upgrade', (request, socket, head) => { + // You may check auth of request here.. + /** + * @param {any} ws + */ + const handleAuth = ws => { + wss.emit('connection', ws, request) + } + wss.handleUpgrade(request, socket, head, handleAuth) +}) + +server.listen({ host, port }) + +console.log(`collabortate editing server is running at '${host}' on port ${port}`) diff --git a/bin/note-synchronizer-server.js b/bin/note-synchronizer-server.js new file mode 100644 index 00000000..2a9e1609 --- /dev/null +++ b/bin/note-synchronizer-server.js @@ -0,0 +1,29 @@ +const redisPersistenceBridge = require('./redis-persistence-bridge') + +const http = require('http') + +const host = process.env.YPERSISTENCE_SYNCHRONIZER_HOST || 'localhost' +const port = process.env.YPERSISTENCE_SYNCHRONIZER_PORT || 1235 + +const server = http.createServer((request, response) => { + if (request.method === 'POST') { + let body = '' + + request.on('data', function (data) { + body += data + }) + + request.on('end', function () { + const post = JSON.parse(body) + for (const sharedObject of post.sharedObjects) { + redisPersistenceBridge.saveQuillDeltaToRedis(post.documentName, sharedObject.name, sharedObject.delta) + } + }) + } + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) +}) + +server.listen({ host, port }) + +console.log(`note synchronizer is running at '${host}' on port ${port}`) diff --git a/bin/redis-persistence-bridge.js b/bin/redis-persistence-bridge.js new file mode 100644 index 00000000..771225c1 --- /dev/null +++ b/bin/redis-persistence-bridge.js @@ -0,0 +1,21 @@ +const yjs = require('yjs') +const RedisPersistence = require('y-redis').RedisPersistence + +const REDIS_HOST = process.env.YPERSISTENCE_REDIS_HOST || 'localhost' +const REDIS_PORT = process.env.YPERSISTENCE_REDIS_PORT || 6379 + +const redisPersistence = new RedisPersistence({ + redisOpts: { host: REDIS_HOST, port: REDIS_PORT } +}) + +// needs to be tested +module.exports = { + saveQuillDeltaToRedis: async function (documentName, sharedObjectName, delta) { + await redisPersistence.clearDocument(documentName) // need to ivestigate how to clear single shared object, not entire doc + const doc = new yjs.Doc() + const persistedDoc = redisPersistence.bindState(documentName, doc) + await persistedDoc.synced + doc.getText(sharedObjectName).applyDelta(delta) + await persistedDoc.synced + } +} diff --git a/bin/server.js b/bin/server.js index e17a4fa3..59b69332 100755 --- a/bin/server.js +++ b/bin/server.js @@ -1,34 +1,4 @@ #!/usr/bin/env node -/** - * @type {any} - */ -const WebSocket = require('ws') -const http = require('http') -const wss = new WebSocket.Server({ noServer: true }) -const setupWSConnection = require('./utils.js').setupWSConnection - -const host = process.env.HOST || 'localhost' -const port = process.env.PORT || 1234 - -const server = http.createServer((request, response) => { - response.writeHead(200, { 'Content-Type': 'text/plain' }) - response.end('okay') -}) - -wss.on('connection', setupWSConnection) - -server.on('upgrade', (request, socket, head) => { - // You may check auth of request here.. - /** - * @param {any} ws - */ - const handleAuth = ws => { - wss.emit('connection', ws, request) - } - wss.handleUpgrade(request, socket, head, handleAuth) -}) - -server.listen({ host, port }) - -console.log(`running at '${host}' on port ${port}`) +require('./collaborate-editing-server') +require('./note-synchronizer-server') diff --git a/bin/transfer-data-from-mysql-to-redis.js b/bin/transfer-data-from-mysql-to-redis.js new file mode 100644 index 00000000..283d3ddf --- /dev/null +++ b/bin/transfer-data-from-mysql-to-redis.js @@ -0,0 +1,51 @@ +#!/usr/bin/env node + +const redisPersistenceBridge = require('./redis-persistence-bridge') + +const mysql = require('mysql2') + +const MYSQL_HOST = process.env.MYSQL_HOST || 'localhost' +const MYSQL_USER = process.env.MYSQL_USER || 'root' +const MYSQL_PASSWORD = process.env.MYSQL_PASSWORD || '' +const MYSQL_DATABASE = process.env.MYSQL_DATABASE || 'database' +const MYSQL_TABLE = process.env.MYSQL_TABLE || 'table' +const MYSQL_CONTENT_FIELDS = process.env.MYSQL_CONTENT_FIELDS || 'content' +const MYSQL_KEY_FIELD = process.env.MYSQL_KEY_FIELD || 'id' +const MYSQL_PORT = process.env.MYSQL_PORT || 3306 + +const mysqlConnection = mysql.createConnection({ + host: MYSQL_HOST, + user: MYSQL_USER, + database: MYSQL_DATABASE, + password: MYSQL_PASSWORD, + port: MYSQL_PORT +}) + +mysqlConnection.connect(function (error) { + if (error) throw error +}) + +mysqlConnection.query( + `SELECT ${MYSQL_KEY_FIELD}, ${MYSQL_CONTENT_FIELDS} FROM ${MYSQL_TABLE}`, + function (error, rows, fields) { + if (error) { + throw error + } + (async () => { + processMysqlDeltas(rows, fields) + })() + } +) + +async function processMysqlDeltas (rows, fields) { + for (const row of rows) { + const key = row[MYSQL_KEY_FIELD] + for (const field of fields) { + if (field.name === MYSQL_KEY_FIELD) { + continue + } + const delta = JSON.parse(row[field.name]) + await redisPersistenceBridge.saveQuillDeltaToRedis(key, field.name, delta) + } + } +} diff --git a/bin/utils.js b/bin/utils.js index c49c7637..411b62f4 100644 --- a/bin/utils.js +++ b/bin/utils.js @@ -22,25 +22,36 @@ const wsReadyStateClosed = 3 // eslint-disable-line // disable gc when using snapshots! const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0' -const persistenceDir = process.env.YPERSISTENCE +const persistenceStorage = process.env.YPERSISTENCE_STORAGE || 'level-db' +const persistenceHost = process.env.YPERSISTENCE_REDIS_HOST || 'localhost' +const persistencePort = process.env.YPERSISTENCE_REDIS_PORT || 6379 +const persistencePath = process.env.YPERSISTENCE_LEVELDB_PATH || 'db' + /** * @type {{bindState: function(string,WSSharedDoc):void, writeState:function(string,WSSharedDoc):Promise, provider: any}|null} */ let persistence = null -if (typeof persistenceDir === 'string') { - console.info('Persisting documents to "' + persistenceDir + '"') +if (typeof persistencePath === 'string') { // @ts-ignore - const LeveldbPersistence = require('y-leveldb').LeveldbPersistence - const ldb = new LeveldbPersistence(persistenceDir) + let persistenceDB + if (persistenceStorage === 'redis') { + console.info('Persisting documents to "' + persistenceHost + '"') + const RedisPersistence = require('y-redis').RedisPersistence + persistenceDB = new RedisPersistence({ redisOpts: { host: persistenceHost, port: persistencePort } }) + } else { + console.info('Persisting documents to "' + persistencePath + '"') + const LeveldbPersistence = require('y-leveldb').LeveldbPersistence + persistenceDB = new LeveldbPersistence(persistencePath) + } persistence = { - provider: ldb, + provider: persistenceDB, bindState: async (docName, ydoc) => { - const persistedYdoc = await ldb.getYDoc(docName) + const persistedYdoc = await persistenceDB.getYDoc(docName) const newUpdates = Y.encodeStateAsUpdate(ydoc) - ldb.storeUpdate(docName, newUpdates) + persistenceDB.storeUpdate(docName, newUpdates) Y.applyUpdate(ydoc, Y.encodeStateAsUpdate(persistedYdoc)) ydoc.on('update', update => { - ldb.storeUpdate(docName, update) + persistenceDB.storeUpdate(docName, update) }) }, writeState: async (docName, ydoc) => {} diff --git a/package.json b/package.json index b2811d20..ef48bf89 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,10 @@ "dependencies": { "lib0": "^0.2.35", "lodash.debounce": "^4.0.8", - "y-protocols": "^1.0.4" + "y-protocols": "^1.0.4", + "ws": "^6.2.1", + "y-leveldb": "^0.1.0", + "y-redis": "^1.0.2" }, "devDependencies": { "rollup": "^1.32.1", @@ -58,9 +61,5 @@ }, "peerDependenies": { "yjs": "^13.5.0" - }, - "optionalDependencies": { - "ws": "^6.2.1", - "y-leveldb": "^0.1.0" } }