From 13a52e3cf2204f423adef9d4e0925c042cdb135a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Sun, 28 Mar 2021 07:48:23 +0300 Subject: [PATCH 01/14] Added Redis support --- bin/utils.js | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/bin/utils.js b/bin/utils.js index c49c7637..b59afa43 100644 --- a/bin/utils.js +++ b/bin/utils.js @@ -22,25 +22,32 @@ const wsReadyStateClosed = 3 // eslint-disable-line // disable gc when using snapshots! const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0' -const persistenceDir = process.env.YPERSISTENCE +const persistenceStorage = process.env.YPERSISTENCE_STORAGE || 'level-db' +const persistencePath = process.env.YPERSISTENCE /** * @type {{bindState: function(string,WSSharedDoc):void, writeState:function(string,WSSharedDoc):Promise, provider: any}|null} */ let persistence = null if (typeof persistenceDir === 'string') { - console.info('Persisting documents to "' + persistenceDir + '"') + console.info('Persisting documents to "' + persistencePath + '"') // @ts-ignore - const LeveldbPersistence = require('y-leveldb').LeveldbPersistence - const ldb = new LeveldbPersistence(persistenceDir) + let persistenceDB + if (persistenceStorage === 'redis') { + const RedisPersistence = require('y-redis').RedisPersistence + persistenceDB = new RedisPersistence(persistencePath) + } else { + const LeveldbPersistence = require('y-leveldb').LeveldbPersistence + persistenceDB = new LeveldbPersistence(persistencePath) + } persistence = { - provider: ldb, + provider: persistenceDB, bindState: async (docName, ydoc) => { - const persistedYdoc = await ldb.getYDoc(docName) + const persistedYdoc = await persistenceDB.getYDoc(docName) const newUpdates = Y.encodeStateAsUpdate(ydoc) - ldb.storeUpdate(docName, newUpdates) + persistenceDB.storeUpdate(docName, newUpdates) Y.applyUpdate(ydoc, Y.encodeStateAsUpdate(persistedYdoc)) ydoc.on('update', update => { - ldb.storeUpdate(docName, update) + persistenceDB.storeUpdate(docName, update) }) }, writeState: async (docName, ydoc) => {} From 30b68b74ad4fa7f9cb40833b42ca13298d0157fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Sun, 28 Mar 2021 10:15:20 +0300 Subject: [PATCH 02/14] Redis support fixes --- Dockerfile | 14 +++++++------- bin/utils.js | 11 ++++++++--- package.json | 9 ++++----- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 42acd06a..0163065e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM node:12-alpine -RUN mkdir -p /home/node/app/node_modules && chown -R node:node /home/node/app -WORKDIR /home/node/app -COPY package*.json ./ -USER node -RUN npm install -COPY --chown=node:node . . -CMD [ "npm", "start" ] \ No newline at end of file +WORKDIR /home/node/ +COPY ./package*.json ./ +RUN yarn install && yarn cache clean --force +ENV PATH /home/node/node_modules/.bin:$PATH +WORKDIR /home/node/app/ +COPY . . +CMD [ "npm", "start" ] diff --git a/bin/utils.js b/bin/utils.js index b59afa43..8512f34f 100644 --- a/bin/utils.js +++ b/bin/utils.js @@ -20,21 +20,26 @@ const wsReadyStateOpen = 1 const wsReadyStateClosing = 2 // eslint-disable-line const wsReadyStateClosed = 3 // eslint-disable-line +console.log('here') + // disable gc when using snapshots! const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0' const persistenceStorage = process.env.YPERSISTENCE_STORAGE || 'level-db' -const persistencePath = process.env.YPERSISTENCE +const persistencePath = process.env.YPERSISTENCE_PATH + +console.log(persistenceStorage) +console.log(persistencePath) /** * @type {{bindState: function(string,WSSharedDoc):void, writeState:function(string,WSSharedDoc):Promise, provider: any}|null} */ let persistence = null -if (typeof persistenceDir === 'string') { +if (typeof persistencePath === 'string') { console.info('Persisting documents to "' + persistencePath + '"') // @ts-ignore let persistenceDB if (persistenceStorage === 'redis') { const RedisPersistence = require('y-redis').RedisPersistence - persistenceDB = new RedisPersistence(persistencePath) + persistenceDB = new RedisPersistence({ redisOpts: { host: persistencePath } }) } else { const LeveldbPersistence = require('y-leveldb').LeveldbPersistence persistenceDB = new LeveldbPersistence(persistencePath) diff --git a/package.json b/package.json index b2811d20..ef48bf89 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,10 @@ "dependencies": { "lib0": "^0.2.35", "lodash.debounce": "^4.0.8", - "y-protocols": "^1.0.4" + "y-protocols": "^1.0.4", + "ws": "^6.2.1", + "y-leveldb": "^0.1.0", + "y-redis": "^1.0.2" }, "devDependencies": { "rollup": "^1.32.1", @@ -58,9 +61,5 @@ }, "peerDependenies": { "yjs": "^13.5.0" - }, - "optionalDependencies": { - "ws": "^6.2.1", - "y-leveldb": "^0.1.0" } } From 8610c71527a438985eef6710421cf5830c1e2f4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Sun, 28 Mar 2021 14:06:21 +0300 Subject: [PATCH 03/14] Added script to transfer mysql deltas to redis --- bin/transfer-data-from-mysql-to-redis.js | 70 ++++++++++++ package.json | 129 ++++++++++++----------- 2 files changed, 135 insertions(+), 64 deletions(-) create mode 100644 bin/transfer-data-from-mysql-to-redis.js diff --git a/bin/transfer-data-from-mysql-to-redis.js b/bin/transfer-data-from-mysql-to-redis.js new file mode 100644 index 00000000..e9c5edbd --- /dev/null +++ b/bin/transfer-data-from-mysql-to-redis.js @@ -0,0 +1,70 @@ +#!/usr/bin/env node + +const yjs = require('yjs') +const mysql = require('mysql2') +const RedisPersistence = require('y-redis').RedisPersistence + +const MYSQL_HOST = process.env.MYSQL_HOST || 'localhost' +const MYSQL_USER = process.env.MYSQL_USER || 'root' +const MYSQL_PASSWORD = process.env.MYSQL_PASSWORD || '' +const MYSQL_DATABASE = process.env.MYSQL_DATABASE || 'database' +const MYSQL_TABLE = process.env.MYSQL_TABLE || 'table' +const MYSQL_CONTENT_FIELD = process.env.MYSQL_CONTENT_FIELD || 'content' +const MYSQL_KEY_FIELD = process.env.MYSQL_KEY_FIELD || 'id' +const MYSQL_PORT = process.env.MYSQL_PORT || 3306 + +const REDIS_HOST = process.env.YPERSISTENCE_PATH || 'localhost' +const REDIS_PORT = process.env.YPERSISTENCE_POSRT || 6379 + +const redisPersistence = new RedisPersistence({ + redisOpts: { host: REDIS_HOST, port: REDIS_PORT } +}) + +const mysqlConnection = mysql.createConnection({ + host: MYSQL_HOST, + user: MYSQL_USER, + database: MYSQL_DATABASE, + password: MYSQL_PASSWORD, + port: MYSQL_PORT +}) + +mysqlConnection.connect(function (error) { + if (error) throw error +}) + +mysqlConnection.query( + 'SELECT ' + + MYSQL_KEY_FIELD + + ', ' + + MYSQL_CONTENT_FIELD + + ' FROM ' + + MYSQL_TABLE, + function (error, notes, fields) { + if (error) { + throw error + } + (async () => { + processMysqlDeltas(notes) + })() + } +) + +// needs to be tested +async function saveDeltaToRedis (docName, delta) { + await redisPersistence.clearDocument(docName) + const doc = new yjs.Doc() + const persistedDoc = redisPersistence.bindState(docName, doc) + await persistedDoc.synced + doc.getText('note-quill-delta').applyDelta(delta) + await persistedDoc.synced +} + +async function processMysqlDeltas (notes) { + for (const note of notes) { + const noteCode = note.code + const noteDelta = JSON.parse(note.ops_json) + await saveDeltaToRedis(noteCode, noteDelta) + } +} + +// process.exit(0) diff --git a/package.json b/package.json index b2811d20..bdd740e3 100644 --- a/package.json +++ b/package.json @@ -1,66 +1,67 @@ { - "name": "y-websocket", - "version": "1.3.11", - "description": "Websockets provider for Yjs", - "main": "./dist/y-websocket.cjs", - "module": "./src/y-websocket.js", - "types": "./dist/src/y-websocket.d.ts", - "sideEffects": false, - "funding": { - "type": "GitHub Sponsors ❤", - "url": "https://github.com/sponsors/dmonad" - }, - "scripts": { - "start": "node ./bin/server.js", - "dist": "rm -rf dist && rollup -c && tsc", - "lint": "standard && tsc", - "test": "npm run lint", - "preversion": "npm run lint && npm run dist && test -e dist/src/y-websocket.d.ts && test -e dist/y-websocket.cjs" - }, - "bin": { - "y-websocket-server": "./bin/server.js" - }, - "files": [ - "dist/*", - "bin/*", - "src/*" - ], - "repository": { - "type": "git", - "url": "git+https://github.com/yjs/y-websocket.git" - }, - "keywords": [ - "Yjs" - ], - "author": "Kevin Jahns ", - "license": "MIT", - "bugs": { - "url": "https://github.com/yjs/y-websocket/issues" - }, - "homepage": "https://github.com/yjs/y-websocket#readme", - "standard": { - "ignore": [ - "/dist", - "/node_modules" - ] - }, - "dependencies": { - "lib0": "^0.2.35", - "lodash.debounce": "^4.0.8", - "y-protocols": "^1.0.4" - }, - "devDependencies": { - "rollup": "^1.32.1", - "rollup-cli": "^1.0.9", - "standard": "^12.0.1", - "typescript": "^3.9.9", - "yjs": "^13.5.0" - }, - "peerDependenies": { - "yjs": "^13.5.0" - }, - "optionalDependencies": { - "ws": "^6.2.1", - "y-leveldb": "^0.1.0" + "name": "y-websocket", + "version": "1.3.11", + "description": "Websockets provider for Yjs", + "main": "./dist/y-websocket.cjs", + "module": "./src/y-websocket.js", + "types": "./dist/src/y-websocket.d.ts", + "sideEffects": false, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "scripts": { + "start": "node ./bin/server.js", + "dist": "rm -rf dist && rollup -c && tsc", + "lint": "standard && tsc", + "test": "npm run lint", + "preversion": "npm run lint && npm run dist && test -e dist/src/y-websocket.d.ts && test -e dist/y-websocket.cjs" + }, + "bin": { + "y-websocket-server": "./bin/server.js" + }, + "files": [ + "dist/*", + "bin/*", + "src/*" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/yjs/y-websocket.git" + }, + "keywords": [ + "Yjs" + ], + "author": "Kevin Jahns ", + "license": "MIT", + "bugs": { + "url": "https://github.com/yjs/y-websocket/issues" + }, + "homepage": "https://github.com/yjs/y-websocket#readme", + "standard": { + "ignore": [ + "/dist", + "/node_modules" + ] + }, + "dependencies": { + "lib0": "^0.2.35", + "lodash.debounce": "^4.0.8", + "y-protocols": "^1.0.4", + "ws": "^6.2.1", + "y-leveldb": "^0.1.0", + "y-redis": "^1.0.2", + "mysql2": "^2.2.5" + }, + "devDependencies": { + "rollup": "^1.32.1", + "rollup-cli": "^1.0.9", + "standard": "^12.0.1", + "typescript": "^3.9.9", + "yjs": "^13.5.0" + }, + "peerDependenies": { + "yjs": "^13.5.0" + } } -} + \ No newline at end of file From 236e0d0d2881c5bfad2279886ea1ceba56c36333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Fri, 2 Apr 2021 12:51:29 +0300 Subject: [PATCH 04/14] Removed console.logs --- bin/utils.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bin/utils.js b/bin/utils.js index 8512f34f..b2151f56 100644 --- a/bin/utils.js +++ b/bin/utils.js @@ -20,15 +20,11 @@ const wsReadyStateOpen = 1 const wsReadyStateClosing = 2 // eslint-disable-line const wsReadyStateClosed = 3 // eslint-disable-line -console.log('here') - // disable gc when using snapshots! const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0' const persistenceStorage = process.env.YPERSISTENCE_STORAGE || 'level-db' const persistencePath = process.env.YPERSISTENCE_PATH -console.log(persistenceStorage) -console.log(persistencePath) /** * @type {{bindState: function(string,WSSharedDoc):void, writeState:function(string,WSSharedDoc):Promise, provider: any}|null} */ From 97b6d03e2eb9cc708c1d09af25702a8e40563d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Sun, 11 Apr 2021 14:18:50 +0300 Subject: [PATCH 05/14] Moved Redis to separate file --- bin/note-synchronizer.js | 29 ++++++++++++++++++++++++ bin/redis-persistence-bridge.js | 19 ++++++++++++++++ bin/server.js | 8 ++++--- bin/transfer-data-from-mysql-to-redis.js | 23 ++----------------- 4 files changed, 55 insertions(+), 24 deletions(-) create mode 100644 bin/note-synchronizer.js create mode 100644 bin/redis-persistence-bridge.js diff --git a/bin/note-synchronizer.js b/bin/note-synchronizer.js new file mode 100644 index 00000000..c25838de --- /dev/null +++ b/bin/note-synchronizer.js @@ -0,0 +1,29 @@ +const redisPersistence = require('./redis-persistence-bridge') + +const http = require('http') + +const host = process.env.YPERSISTENCE_HOST || 'localhost' +const port = process.env.YPERSISTENCE_PORT || 1235 + +const server = http.createServer((request, response) => { + if (request.method === 'POST') { + let body = '' + + request.on('data', function (data) { + body += data + }) + + request.on('end', function () { + const post = JSON.parse(body) + console.log(body) + redisPersistence.saveDeltaToRedis(post.docName, post.delta) + }) + } + response.writeHead(200, { 'Content-Type': 'text/plain' }) + response.end('okay') +}) +console.log(host) +console.log(port) +server.listen(host, port) + +console.log(`note syncronizer is running at '${host}' on port ${port}`) diff --git a/bin/redis-persistence-bridge.js b/bin/redis-persistence-bridge.js new file mode 100644 index 00000000..eb5be24c --- /dev/null +++ b/bin/redis-persistence-bridge.js @@ -0,0 +1,19 @@ +const yjs = require('yjs') +const RedisPersistence = require('y-redis').RedisPersistence + +const REDIS_HOST = process.env.YPERSISTENCE_PATH || 'localhost' +const REDIS_PORT = process.env.YPERSISTENCE_POSRT || 6379 + +const redisPersistence = new RedisPersistence({ + redisOpts: { host: REDIS_HOST, port: REDIS_PORT } +}) + +// needs to be tested +async function saveDeltaToRedis (docName, delta) { + await redisPersistence.clearDocument(docName) + const doc = new yjs.Doc() + const persistedDoc = redisPersistence.bindState(docName, doc) + await persistedDoc.synced + doc.getText('note-quill-delta').applyDelta(delta) + await persistedDoc.synced +} diff --git a/bin/server.js b/bin/server.js index e17a4fa3..02ef5ba0 100755 --- a/bin/server.js +++ b/bin/server.js @@ -8,8 +8,8 @@ const http = require('http') const wss = new WebSocket.Server({ noServer: true }) const setupWSConnection = require('./utils.js').setupWSConnection -const host = process.env.HOST || 'localhost' -const port = process.env.PORT || 1234 +const host = process.env.YWEBSOCKET_HOST || 'localhost' +const port = process.env.YWEBSOCKET_PORT || 1234 const server = http.createServer((request, response) => { response.writeHead(200, { 'Content-Type': 'text/plain' }) @@ -29,6 +29,8 @@ server.on('upgrade', (request, socket, head) => { wss.handleUpgrade(request, socket, head, handleAuth) }) +console.log(host) +console.log(port) server.listen({ host, port }) -console.log(`running at '${host}' on port ${port}`) +console.log(`collabortate editing server is running at '${host}' on port ${port}`) diff --git a/bin/transfer-data-from-mysql-to-redis.js b/bin/transfer-data-from-mysql-to-redis.js index e9c5edbd..f6524b9e 100644 --- a/bin/transfer-data-from-mysql-to-redis.js +++ b/bin/transfer-data-from-mysql-to-redis.js @@ -1,8 +1,8 @@ #!/usr/bin/env node -const yjs = require('yjs') +import { saveDeltaToRedis } from './redis-persistence-bridge' + const mysql = require('mysql2') -const RedisPersistence = require('y-redis').RedisPersistence const MYSQL_HOST = process.env.MYSQL_HOST || 'localhost' const MYSQL_USER = process.env.MYSQL_USER || 'root' @@ -13,13 +13,6 @@ const MYSQL_CONTENT_FIELD = process.env.MYSQL_CONTENT_FIELD || 'content' const MYSQL_KEY_FIELD = process.env.MYSQL_KEY_FIELD || 'id' const MYSQL_PORT = process.env.MYSQL_PORT || 3306 -const REDIS_HOST = process.env.YPERSISTENCE_PATH || 'localhost' -const REDIS_PORT = process.env.YPERSISTENCE_POSRT || 6379 - -const redisPersistence = new RedisPersistence({ - redisOpts: { host: REDIS_HOST, port: REDIS_PORT } -}) - const mysqlConnection = mysql.createConnection({ host: MYSQL_HOST, user: MYSQL_USER, @@ -49,16 +42,6 @@ mysqlConnection.query( } ) -// needs to be tested -async function saveDeltaToRedis (docName, delta) { - await redisPersistence.clearDocument(docName) - const doc = new yjs.Doc() - const persistedDoc = redisPersistence.bindState(docName, doc) - await persistedDoc.synced - doc.getText('note-quill-delta').applyDelta(delta) - await persistedDoc.synced -} - async function processMysqlDeltas (notes) { for (const note of notes) { const noteCode = note.code @@ -66,5 +49,3 @@ async function processMysqlDeltas (notes) { await saveDeltaToRedis(noteCode, noteDelta) } } - -// process.exit(0) From b29446b442d11ad029e3d6e99d93ffea78352916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Sun, 11 Apr 2021 17:03:41 +0300 Subject: [PATCH 06/14] =?UTF-8?q?Created=20server=20to=20serve=20mysql=20?= =?UTF-8?q?=E2=86=92=20Redis=20syncronization=20requests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/collaborate-editing.server.js | 32 +++++++++++++++++++++ bin/note-synchronizer.js | 9 +++--- bin/redis-persistence-bridge.js | 16 ++++++----- bin/server.js | 36 ++---------------------- bin/transfer-data-from-mysql-to-redis.js | 4 +-- 5 files changed, 49 insertions(+), 48 deletions(-) create mode 100644 bin/collaborate-editing.server.js diff --git a/bin/collaborate-editing.server.js b/bin/collaborate-editing.server.js new file mode 100644 index 00000000..738f8f9b --- /dev/null +++ b/bin/collaborate-editing.server.js @@ -0,0 +1,32 @@ +/** + * @type {any} + */ +const WebSocket = require('ws') +const http = require('http') +const wss = new WebSocket.Server({ noServer: true }) +const setupWSConnection = require('./utils.js').setupWSConnection + +const host = process.env.YWEBSOCKET_HOST || 'localhost' +const port = process.env.YWEBSOCKET_PORT || 1234 + +const server = http.createServer((request, response) => { + response.writeHead(200, { 'Content-Type': 'text/plain' }) + response.end('okay') +}) + +wss.on('connection', setupWSConnection) + +server.on('upgrade', (request, socket, head) => { + // You may check auth of request here.. + /** + * @param {any} ws + */ + const handleAuth = ws => { + wss.emit('connection', ws, request) + } + wss.handleUpgrade(request, socket, head, handleAuth) +}) + +server.listen({ host, port }) + +console.log(`collabortate editing server is running at '${host}' on port ${port}`) diff --git a/bin/note-synchronizer.js b/bin/note-synchronizer.js index c25838de..cab66694 100644 --- a/bin/note-synchronizer.js +++ b/bin/note-synchronizer.js @@ -1,4 +1,4 @@ -const redisPersistence = require('./redis-persistence-bridge') +const redisPersistenceBridge = require('./redis-persistence-bridge') const http = require('http') @@ -16,14 +16,13 @@ const server = http.createServer((request, response) => { request.on('end', function () { const post = JSON.parse(body) console.log(body) - redisPersistence.saveDeltaToRedis(post.docName, post.delta) + redisPersistenceBridge.saveDeltaToRedis(post.docName, post.delta) }) } response.writeHead(200, { 'Content-Type': 'text/plain' }) response.end('okay') }) -console.log(host) -console.log(port) -server.listen(host, port) + +server.listen({ host, port }) console.log(`note syncronizer is running at '${host}' on port ${port}`) diff --git a/bin/redis-persistence-bridge.js b/bin/redis-persistence-bridge.js index eb5be24c..e116c54f 100644 --- a/bin/redis-persistence-bridge.js +++ b/bin/redis-persistence-bridge.js @@ -9,11 +9,13 @@ const redisPersistence = new RedisPersistence({ }) // needs to be tested -async function saveDeltaToRedis (docName, delta) { - await redisPersistence.clearDocument(docName) - const doc = new yjs.Doc() - const persistedDoc = redisPersistence.bindState(docName, doc) - await persistedDoc.synced - doc.getText('note-quill-delta').applyDelta(delta) - await persistedDoc.synced +module.exports = { + saveDeltaToRedis: async function (docName, delta) { + await redisPersistence.clearDocument(docName) + const doc = new yjs.Doc() + const persistedDoc = redisPersistence.bindState(docName, doc) + await persistedDoc.synced + doc.getText('note-quill-delta').applyDelta(delta) + await persistedDoc.synced + } } diff --git a/bin/server.js b/bin/server.js index 02ef5ba0..95400684 100755 --- a/bin/server.js +++ b/bin/server.js @@ -1,36 +1,4 @@ #!/usr/bin/env node -/** - * @type {any} - */ -const WebSocket = require('ws') -const http = require('http') -const wss = new WebSocket.Server({ noServer: true }) -const setupWSConnection = require('./utils.js').setupWSConnection - -const host = process.env.YWEBSOCKET_HOST || 'localhost' -const port = process.env.YWEBSOCKET_PORT || 1234 - -const server = http.createServer((request, response) => { - response.writeHead(200, { 'Content-Type': 'text/plain' }) - response.end('okay') -}) - -wss.on('connection', setupWSConnection) - -server.on('upgrade', (request, socket, head) => { - // You may check auth of request here.. - /** - * @param {any} ws - */ - const handleAuth = ws => { - wss.emit('connection', ws, request) - } - wss.handleUpgrade(request, socket, head, handleAuth) -}) - -console.log(host) -console.log(port) -server.listen({ host, port }) - -console.log(`collabortate editing server is running at '${host}' on port ${port}`) +require('./collaborate-editing.server') +require('./note-synchronizer') diff --git a/bin/transfer-data-from-mysql-to-redis.js b/bin/transfer-data-from-mysql-to-redis.js index f6524b9e..459b0317 100644 --- a/bin/transfer-data-from-mysql-to-redis.js +++ b/bin/transfer-data-from-mysql-to-redis.js @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { saveDeltaToRedis } from './redis-persistence-bridge' +const redisPersistenceBridge = require('./redis-persistence-bridge') const mysql = require('mysql2') @@ -46,6 +46,6 @@ async function processMysqlDeltas (notes) { for (const note of notes) { const noteCode = note.code const noteDelta = JSON.parse(note.ops_json) - await saveDeltaToRedis(noteCode, noteDelta) + await redisPersistenceBridge.saveDeltaToRedis(noteCode, noteDelta) } } From b89a162ad958f781c6157fcbf8ab9452744154c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Sun, 11 Apr 2021 17:05:04 +0300 Subject: [PATCH 07/14] Fixed newline issues in package.json --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index cc05f42a..ef48bf89 100644 --- a/package.json +++ b/package.json @@ -63,4 +63,3 @@ "yjs": "^13.5.0" } } - \ No newline at end of file From 79e8935d92415f5780f7e47c54f3d9b50698e7c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Sun, 11 Apr 2021 17:08:02 +0300 Subject: [PATCH 08/14] removed debug console.log --- bin/note-synchronizer.js | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/note-synchronizer.js b/bin/note-synchronizer.js index cab66694..73c126d2 100644 --- a/bin/note-synchronizer.js +++ b/bin/note-synchronizer.js @@ -15,7 +15,6 @@ const server = http.createServer((request, response) => { request.on('end', function () { const post = JSON.parse(body) - console.log(body) redisPersistenceBridge.saveDeltaToRedis(post.docName, post.delta) }) } From ec4833a5261db2ca8339231034883f08c4a86ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Mon, 12 Apr 2021 10:35:32 +0300 Subject: [PATCH 09/14] Cleaned up names --- bin/collaborate-editing.server.js | 4 ++-- bin/note-synchronizer.js | 10 +++++----- bin/redis-persistence-bridge.js | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bin/collaborate-editing.server.js b/bin/collaborate-editing.server.js index 738f8f9b..cbddd159 100644 --- a/bin/collaborate-editing.server.js +++ b/bin/collaborate-editing.server.js @@ -10,8 +10,8 @@ const host = process.env.YWEBSOCKET_HOST || 'localhost' const port = process.env.YWEBSOCKET_PORT || 1234 const server = http.createServer((request, response) => { - response.writeHead(200, { 'Content-Type': 'text/plain' }) - response.end('okay') + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) }) wss.on('connection', setupWSConnection) diff --git a/bin/note-synchronizer.js b/bin/note-synchronizer.js index 73c126d2..77d7785f 100644 --- a/bin/note-synchronizer.js +++ b/bin/note-synchronizer.js @@ -2,8 +2,8 @@ const redisPersistenceBridge = require('./redis-persistence-bridge') const http = require('http') -const host = process.env.YPERSISTENCE_HOST || 'localhost' -const port = process.env.YPERSISTENCE_PORT || 1235 +const host = process.env.YPERSISTENCE_SYNCHRONIZER_HOST || 'localhost' +const port = process.env.YPERSISTENCE_SYNCHRONIZER_PORT || 1235 const server = http.createServer((request, response) => { if (request.method === 'POST') { @@ -18,10 +18,10 @@ const server = http.createServer((request, response) => { redisPersistenceBridge.saveDeltaToRedis(post.docName, post.delta) }) } - response.writeHead(200, { 'Content-Type': 'text/plain' }) - response.end('okay') + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) }) server.listen({ host, port }) -console.log(`note syncronizer is running at '${host}' on port ${port}`) +console.log(`note synchronizer is running at '${host}' on port ${port}`) diff --git a/bin/redis-persistence-bridge.js b/bin/redis-persistence-bridge.js index e116c54f..e834e265 100644 --- a/bin/redis-persistence-bridge.js +++ b/bin/redis-persistence-bridge.js @@ -2,7 +2,7 @@ const yjs = require('yjs') const RedisPersistence = require('y-redis').RedisPersistence const REDIS_HOST = process.env.YPERSISTENCE_PATH || 'localhost' -const REDIS_PORT = process.env.YPERSISTENCE_POSRT || 6379 +const REDIS_PORT = process.env.YPERSISTENCE_PORT || 6379 const redisPersistence = new RedisPersistence({ redisOpts: { host: REDIS_HOST, port: REDIS_PORT } @@ -15,7 +15,7 @@ module.exports = { const doc = new yjs.Doc() const persistedDoc = redisPersistence.bindState(docName, doc) await persistedDoc.synced - doc.getText('note-quill-delta').applyDelta(delta) + doc.getText('delta').applyDelta(delta) await persistedDoc.synced } } From 42852d8e169229034d1ad767221135d2dac88885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Mon, 12 Apr 2021 10:36:41 +0300 Subject: [PATCH 10/14] Renamed server files --- ...borate-editing.server.js => collaborate-editing-server.js} | 0 bin/{note-synchronizer.js => note-synchronizer-server.js} | 0 bin/server.js | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename bin/{collaborate-editing.server.js => collaborate-editing-server.js} (100%) rename bin/{note-synchronizer.js => note-synchronizer-server.js} (100%) diff --git a/bin/collaborate-editing.server.js b/bin/collaborate-editing-server.js similarity index 100% rename from bin/collaborate-editing.server.js rename to bin/collaborate-editing-server.js diff --git a/bin/note-synchronizer.js b/bin/note-synchronizer-server.js similarity index 100% rename from bin/note-synchronizer.js rename to bin/note-synchronizer-server.js diff --git a/bin/server.js b/bin/server.js index 95400684..59b69332 100755 --- a/bin/server.js +++ b/bin/server.js @@ -1,4 +1,4 @@ #!/usr/bin/env node -require('./collaborate-editing.server') -require('./note-synchronizer') +require('./collaborate-editing-server') +require('./note-synchronizer-server') From 41ecdd3a0acfa2a0f1ef2ce35d1f051907c1d07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Mon, 12 Apr 2021 10:41:28 +0300 Subject: [PATCH 11/14] Made code less bound to conspectus --- bin/transfer-data-from-mysql-to-redis.js | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/bin/transfer-data-from-mysql-to-redis.js b/bin/transfer-data-from-mysql-to-redis.js index 459b0317..1e106f8e 100644 --- a/bin/transfer-data-from-mysql-to-redis.js +++ b/bin/transfer-data-from-mysql-to-redis.js @@ -26,26 +26,21 @@ mysqlConnection.connect(function (error) { }) mysqlConnection.query( - 'SELECT ' + - MYSQL_KEY_FIELD + - ', ' + - MYSQL_CONTENT_FIELD + - ' FROM ' + - MYSQL_TABLE, - function (error, notes, fields) { + `SELECT ${MYSQL_KEY_FIELD}, ${MYSQL_CONTENT_FIELD} FROM ${MYSQL_TABLE}`, + function (error, rows, fields) { if (error) { throw error } (async () => { - processMysqlDeltas(notes) + processMysqlDeltas(rows) })() } ) -async function processMysqlDeltas (notes) { - for (const note of notes) { - const noteCode = note.code - const noteDelta = JSON.parse(note.ops_json) - await redisPersistenceBridge.saveDeltaToRedis(noteCode, noteDelta) +async function processMysqlDeltas (rows) { + for (const row of rows) { + const key = row[MYSQL_KEY_FIELD] + const delta = JSON.parse(row[MYSQL_CONTENT_FIELD]) + await redisPersistenceBridge.saveDeltaToRedis(key, delta) } } From 4390f9f6bd8846c46d1a5469190439f073220d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Mon, 12 Apr 2021 14:08:46 +0300 Subject: [PATCH 12/14] Added support for multiple shared objects --- bin/note-synchronizer-server.js | 4 +++- bin/redis-persistence-bridge.js | 8 ++++---- bin/transfer-data-from-mysql-to-redis.js | 17 +++++++++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/bin/note-synchronizer-server.js b/bin/note-synchronizer-server.js index 77d7785f..2a9e1609 100644 --- a/bin/note-synchronizer-server.js +++ b/bin/note-synchronizer-server.js @@ -15,7 +15,9 @@ const server = http.createServer((request, response) => { request.on('end', function () { const post = JSON.parse(body) - redisPersistenceBridge.saveDeltaToRedis(post.docName, post.delta) + for (const sharedObject of post.sharedObjects) { + redisPersistenceBridge.saveQuillDeltaToRedis(post.documentName, sharedObject.name, sharedObject.delta) + } }) } response.writeHead(200, { 'Content-Type': 'application/json' }) diff --git a/bin/redis-persistence-bridge.js b/bin/redis-persistence-bridge.js index e834e265..cf1b531f 100644 --- a/bin/redis-persistence-bridge.js +++ b/bin/redis-persistence-bridge.js @@ -10,12 +10,12 @@ const redisPersistence = new RedisPersistence({ // needs to be tested module.exports = { - saveDeltaToRedis: async function (docName, delta) { - await redisPersistence.clearDocument(docName) + saveQuillDeltaToRedis: async function (documentName, sharedObjectName, delta) { + await redisPersistence.clearDocument(documentName) // need to ivestigate how to clear single shared object, not entire doc const doc = new yjs.Doc() - const persistedDoc = redisPersistence.bindState(docName, doc) + const persistedDoc = redisPersistence.bindState(documentName, doc) await persistedDoc.synced - doc.getText('delta').applyDelta(delta) + doc.getText(sharedObjectName).applyDelta(delta) await persistedDoc.synced } } diff --git a/bin/transfer-data-from-mysql-to-redis.js b/bin/transfer-data-from-mysql-to-redis.js index 1e106f8e..283d3ddf 100644 --- a/bin/transfer-data-from-mysql-to-redis.js +++ b/bin/transfer-data-from-mysql-to-redis.js @@ -9,7 +9,7 @@ const MYSQL_USER = process.env.MYSQL_USER || 'root' const MYSQL_PASSWORD = process.env.MYSQL_PASSWORD || '' const MYSQL_DATABASE = process.env.MYSQL_DATABASE || 'database' const MYSQL_TABLE = process.env.MYSQL_TABLE || 'table' -const MYSQL_CONTENT_FIELD = process.env.MYSQL_CONTENT_FIELD || 'content' +const MYSQL_CONTENT_FIELDS = process.env.MYSQL_CONTENT_FIELDS || 'content' const MYSQL_KEY_FIELD = process.env.MYSQL_KEY_FIELD || 'id' const MYSQL_PORT = process.env.MYSQL_PORT || 3306 @@ -26,21 +26,26 @@ mysqlConnection.connect(function (error) { }) mysqlConnection.query( - `SELECT ${MYSQL_KEY_FIELD}, ${MYSQL_CONTENT_FIELD} FROM ${MYSQL_TABLE}`, + `SELECT ${MYSQL_KEY_FIELD}, ${MYSQL_CONTENT_FIELDS} FROM ${MYSQL_TABLE}`, function (error, rows, fields) { if (error) { throw error } (async () => { - processMysqlDeltas(rows) + processMysqlDeltas(rows, fields) })() } ) -async function processMysqlDeltas (rows) { +async function processMysqlDeltas (rows, fields) { for (const row of rows) { const key = row[MYSQL_KEY_FIELD] - const delta = JSON.parse(row[MYSQL_CONTENT_FIELD]) - await redisPersistenceBridge.saveDeltaToRedis(key, delta) + for (const field of fields) { + if (field.name === MYSQL_KEY_FIELD) { + continue + } + const delta = JSON.parse(row[field.name]) + await redisPersistenceBridge.saveQuillDeltaToRedis(key, field.name, delta) + } } } From 1a49720c04a8dc9fc10500906a5ecb560d2cb384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Mon, 12 Apr 2021 15:41:09 +0300 Subject: [PATCH 13/14] Changed env names for better code clearance --- bin/utils.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/utils.js b/bin/utils.js index b2151f56..411b62f4 100644 --- a/bin/utils.js +++ b/bin/utils.js @@ -23,20 +23,23 @@ const wsReadyStateClosed = 3 // eslint-disable-line // disable gc when using snapshots! const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0' const persistenceStorage = process.env.YPERSISTENCE_STORAGE || 'level-db' -const persistencePath = process.env.YPERSISTENCE_PATH +const persistenceHost = process.env.YPERSISTENCE_REDIS_HOST || 'localhost' +const persistencePort = process.env.YPERSISTENCE_REDIS_PORT || 6379 +const persistencePath = process.env.YPERSISTENCE_LEVELDB_PATH || 'db' /** * @type {{bindState: function(string,WSSharedDoc):void, writeState:function(string,WSSharedDoc):Promise, provider: any}|null} */ let persistence = null if (typeof persistencePath === 'string') { - console.info('Persisting documents to "' + persistencePath + '"') // @ts-ignore let persistenceDB if (persistenceStorage === 'redis') { + console.info('Persisting documents to "' + persistenceHost + '"') const RedisPersistence = require('y-redis').RedisPersistence - persistenceDB = new RedisPersistence({ redisOpts: { host: persistencePath } }) + persistenceDB = new RedisPersistence({ redisOpts: { host: persistenceHost, port: persistencePort } }) } else { + console.info('Persisting documents to "' + persistencePath + '"') const LeveldbPersistence = require('y-leveldb').LeveldbPersistence persistenceDB = new LeveldbPersistence(persistencePath) } From bc0a6e82c9159390f16ef66758a354d24b1a383f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=82=D0=B8=D0=BD?= Date: Mon, 12 Apr 2021 15:42:54 +0300 Subject: [PATCH 14/14] Changed env names for better code clearance --- bin/redis-persistence-bridge.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/redis-persistence-bridge.js b/bin/redis-persistence-bridge.js index cf1b531f..771225c1 100644 --- a/bin/redis-persistence-bridge.js +++ b/bin/redis-persistence-bridge.js @@ -1,8 +1,8 @@ const yjs = require('yjs') const RedisPersistence = require('y-redis').RedisPersistence -const REDIS_HOST = process.env.YPERSISTENCE_PATH || 'localhost' -const REDIS_PORT = process.env.YPERSISTENCE_PORT || 6379 +const REDIS_HOST = process.env.YPERSISTENCE_REDIS_HOST || 'localhost' +const REDIS_PORT = process.env.YPERSISTENCE_REDIS_PORT || 6379 const redisPersistence = new RedisPersistence({ redisOpts: { host: REDIS_HOST, port: REDIS_PORT }