diff --git a/package.json b/package.json index 10ede3a377..fa8a9fb4d2 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "filesize": "6.1.0", "fs-extra": "9.0.1", "getos": "3.2.1", + "@google-cloud/storage": "7.21.0", "graphql": "15.3.0", "graphql-list-fields": "2.0.2", "graphql-rate-limit-directive": "1.2.1", diff --git a/server/modules/storage/gcs/definition.yml b/server/modules/storage/gcs/definition.yml new file mode 100755 index 0000000000..8ea0f28e74 --- /dev/null +++ b/server/modules/storage/gcs/definition.yml @@ -0,0 +1,42 @@ +key: gcs +title: Google Cloud Storage +description: Google Cloud Storage is a fully-managed, scalable object storage service offered by Google Cloud Platform. +author: Horus Gonzalez (built with Claude) +logo: https://upload.wikimedia.org/wikipedia/commons/d/df/Google_Cloud_storage.svg +website: https://cloud.google.com/storage +isAvailable: true +supportedModes: + - push +defaultMode: push +schedule: false +props: + bucket: + type: String + title: Bucket Name + default: '' + hint: The unique name of an existing Google Cloud Storage bucket (e.g. my-wiki-bucket). The bucket must already exist; it will not be created automatically. + order: 1 + projectId: + type: String + title: Project ID + default: '' + hint: The Google Cloud project ID that owns the bucket. + order: 2 + serviceAccountKey: + type: String + title: Service Account Key (JSON) + default: '' + hint: Paste the full contents of a Google Cloud service account JSON key. The account needs 2 roles on the bucket -- Storage Object Admin AND Storage Legacy Bucket Reader (the second one is required for the bucket existence check on startup). Create the key at https://console.cloud.google.com/iam-admin/serviceaccounts (select your project → open the service account → Keys tab → Add Key → JSON). + sensitive: true + multiline: true + order: 3 + publicRead: + type: Boolean + title: Make Uploaded Files Public + default: false + hint: When enabled, files uploaded to the bucket will be set as publicly readable. Requires the service account to have permission to set object ACLs and Fine-grained access enabled on the bucket. + order: 4 +actions: + - handler: exportAll + label: Export All + hint: Output all content from the DB to Google Cloud Storage, overwriting any existing data. If you enabled Google Cloud Storage after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content. diff --git a/server/modules/storage/gcs/storage.js b/server/modules/storage/gcs/storage.js new file mode 100755 index 0000000000..61059fdb70 --- /dev/null +++ b/server/modules/storage/gcs/storage.js @@ -0,0 +1,175 @@ +/** + * Google Cloud Storage module for Wiki.js + * + * Author: Horus Gonzalez + * Built with help from Claude (Anthropic) + * + * Fills a gap in Wiki.js core, which does not ship a GCS storage target + * out of the box (only S3, Azure, Dropbox, Box, GDrive, OneDrive, + * DigitalOcean Spaces, SFTP, Git, and local disk). + */ +const { Storage } = require('@google-cloud/storage') +const { pipeline } = require('node:stream/promises') +const { Transform } = require('node:stream') +const pageHelper = require('../../../helpers/page.js') +const _ = require('lodash') + +/* global WIKI */ + +const getFilePath = (page, pathKey) => { + const fileName = `${page[pathKey]}.${pageHelper.getFileExtension(page.contentType)}` + const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode + return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName +} + +module.exports = { + async activated() { + + }, + async deactivated() { + + }, + async init() { + WIKI.logger.info(`(STORAGE/GCS) Initializing...`) + const { bucket, projectId, serviceAccountKey } = this.config + + let credentials + try { + credentials = JSON.parse(serviceAccountKey) + } catch (err) { + throw new Error('Invalid Service Account Key: must be valid JSON.') + } + + this.client = new Storage({ + projectId: projectId || credentials.project_id, + credentials + }) + this.bucket = this.client.bucket(bucket) + + const [exists] = await this.bucket.exists() + if (!exists) { + throw new Error(`Bucket ${bucket} does not exist or is not accessible with the provided credentials.`) + } + + WIKI.logger.info(`(STORAGE/GCS) Initialization completed.`) + }, + async created (page) { + WIKI.logger.info(`(STORAGE/GCS) Creating file ${page.path}...`) + const filePath = getFilePath(page, 'path') + const pageContent = page.injectMetadata() + await this.saveContent(filePath, pageContent) + }, + async updated (page) { + WIKI.logger.info(`(STORAGE/GCS) Updating file ${page.path}...`) + const filePath = getFilePath(page, 'path') + const pageContent = page.injectMetadata() + await this.saveContent(filePath, pageContent) + }, + async deleted (page) { + WIKI.logger.info(`(STORAGE/GCS) Deleting file ${page.path}...`) + const filePath = getFilePath(page, 'path') + await this.bucket.file(filePath).delete({ ignoreNotFound: true }) + }, + async renamed (page) { + WIKI.logger.info(`(STORAGE/GCS) Renaming file ${page.path} to ${page.destinationPath}...`) + let sourceFilePath = getFilePath(page, 'path') + let destinationFilePath = getFilePath(page, 'destinationPath') + if (WIKI.config.lang.namespacing) { + if (WIKI.config.lang.code !== page.localeCode) { + sourceFilePath = `${page.localeCode}/${sourceFilePath}` + } + if (WIKI.config.lang.code !== page.destinationLocaleCode) { + destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}` + } + } + await this.bucket.file(sourceFilePath).move(destinationFilePath) + }, + /** + * ASSET UPLOAD + * + * @param {Object} asset Asset to upload + */ + async assetUploaded (asset) { + WIKI.logger.info(`(STORAGE/GCS) Creating new file ${asset.path}...`) + await this.saveContent(asset.path, asset.data) + }, + /** + * ASSET DELETE + * + * @param {Object} asset Asset to delete + */ + async assetDeleted (asset) { + WIKI.logger.info(`(STORAGE/GCS) Deleting file ${asset.path}...`) + await this.bucket.file(asset.path).delete({ ignoreNotFound: true }) + }, + /** + * ASSET RENAME + * + * @param {Object} asset Asset to rename + */ + async assetRenamed (asset) { + WIKI.logger.info(`(STORAGE/GCS) Renaming file from ${asset.path} to ${asset.destinationPath}...`) + await this.bucket.file(asset.path).move(asset.destinationPath) + }, + async getLocalLocation () { + + }, + /** + * Save content to the bucket and apply public ACL if configured. + * + * @param {String} filePath Destination path within the bucket + * @param {String|Buffer} content File content + */ + async saveContent (filePath, content) { + const file = this.bucket.file(filePath) + await file.save(content, { resumable: false }) + if (this.config.publicRead) { + try { + await file.makePublic() + } catch (err) { + WIKI.logger.warn(`(STORAGE/GCS) Could not set public ACL on ${filePath}: ${err.message}`) + } + } + }, + /** + * HANDLERS + */ + async exportAll () { + WIKI.logger.info(`(STORAGE/GCS) Exporting all content to Google Cloud Storage...`) + + // -> Pages + await pipeline( + WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt').select().from('pages').where({ + isPrivate: false + }).stream(), + new Transform({ + objectMode: true, + transform: async (page, enc, cb) => { + const filePath = getFilePath(page, 'path') + WIKI.logger.info(`(STORAGE/GCS) Adding page ${filePath}...`) + const pageContent = pageHelper.injectPageMetadata(page) + await this.saveContent(filePath, pageContent) + cb() + } + }) + ) + + // -> Assets + const assetFolders = await WIKI.models.assetFolders.getAllPaths() + + await pipeline( + WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(), + new Transform({ + objectMode: true, + transform: async (asset, enc, cb) => { + const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename + WIKI.logger.info(`(STORAGE/GCS) Adding asset ${filename}...`) + await this.saveContent(filename, asset.data) + cb() + } + }) + ) + + WIKI.logger.info('(STORAGE/GCS) All content has been pushed to Google Cloud Storage.') + } +} diff --git a/server/test/modules/storage/gcs/storage.test.js b/server/test/modules/storage/gcs/storage.test.js new file mode 100644 index 0000000000..03699d227f --- /dev/null +++ b/server/test/modules/storage/gcs/storage.test.js @@ -0,0 +1,273 @@ +const mockFile = { + save: jest.fn().mockResolvedValue(), + delete: jest.fn().mockResolvedValue(), + move: jest.fn().mockResolvedValue(), + makePublic: jest.fn().mockResolvedValue() +} + +const mockBucketExists = jest.fn().mockResolvedValue([true]) + +const mockBucket = { + exists: mockBucketExists, + file: jest.fn(() => mockFile) +} + +const mockStorageInstance = { + bucket: jest.fn(() => mockBucket) +} + +const MockStorage = jest.fn(() => mockStorageInstance) + +jest.mock('@google-cloud/storage', () => ({ + Storage: MockStorage +})) + +jest.mock('../../../../helpers/page.js', () => ({ + getFileExtension: jest.fn(() => 'md') +})) + +const gcsStorage = require('../../../../modules/storage/gcs/storage') + +const validServiceAccountKey = JSON.stringify({ + project_id: 'from-key-project', + client_email: 'test@from-key-project.iam.gserviceaccount.com' +}) + +const makeContext = (config) => Object.assign(Object.create(gcsStorage), { config }) + +describe('modules/storage/gcs', () => { + beforeEach(() => { + jest.clearAllMocks() + mockBucketExists.mockResolvedValue([true]) + global.WIKI = { + logger: { + info: jest.fn(), + warn: jest.fn() + }, + config: { + lang: { + namespacing: false, + code: 'en' + } + } + } + }) + + describe('init', () => { + it('throws when the service account key is not valid JSON', async () => { + const ctx = makeContext({ + bucket: 'my-bucket', + projectId: 'my-project', + serviceAccountKey: 'not-json' + }) + + await expect(ctx.init()).rejects.toThrow('Invalid Service Account Key: must be valid JSON.') + }) + + it('creates the Storage client using the explicit projectId when provided', async () => { + const ctx = makeContext({ + bucket: 'my-bucket', + projectId: 'explicit-project', + serviceAccountKey: validServiceAccountKey + }) + + await ctx.init() + + expect(MockStorage).toHaveBeenCalledWith({ + projectId: 'explicit-project', + credentials: JSON.parse(validServiceAccountKey) + }) + }) + + it('falls back to the project_id from the key when projectId is not provided', async () => { + const ctx = makeContext({ + bucket: 'my-bucket', + projectId: '', + serviceAccountKey: validServiceAccountKey + }) + + await ctx.init() + + expect(MockStorage).toHaveBeenCalledWith({ + projectId: 'from-key-project', + credentials: JSON.parse(validServiceAccountKey) + }) + }) + + it('throws when the bucket does not exist or is not accessible', async () => { + mockBucketExists.mockResolvedValue([false]) + const ctx = makeContext({ + bucket: 'missing-bucket', + projectId: 'my-project', + serviceAccountKey: validServiceAccountKey + }) + + await expect(ctx.init()).rejects.toThrow( + 'Bucket missing-bucket does not exist or is not accessible with the provided credentials.' + ) + }) + + it('completes successfully when the bucket exists', async () => { + const ctx = makeContext({ + bucket: 'my-bucket', + projectId: 'my-project', + serviceAccountKey: validServiceAccountKey + }) + + await expect(ctx.init()).resolves.toBeUndefined() + expect(mockStorageInstance.bucket).toHaveBeenCalledWith('my-bucket') + }) + }) + + describe('created / updated', () => { + it('saves the page content to the path derived from page.path', async () => { + const ctx = makeContext({ publicRead: false }) + ctx.bucket = mockBucket + const page = { + path: 'en/home', + localeCode: 'en', + contentType: 'markdown', + injectMetadata: jest.fn(() => 'PAGE CONTENT') + } + + await ctx.created(page) + + expect(mockBucket.file).toHaveBeenCalledWith('en/home.md') + expect(mockFile.save).toHaveBeenCalledWith('PAGE CONTENT', { resumable: false }) + }) + + it('prefixes the file path with the locale code when namespacing is enabled and the locale differs', async () => { + global.WIKI.config.lang.namespacing = true + global.WIKI.config.lang.code = 'en' + + const ctx = makeContext({ publicRead: false }) + ctx.bucket = mockBucket + const page = { + path: 'bienvenida', + localeCode: 'es', + contentType: 'markdown', + injectMetadata: jest.fn(() => 'CONTENIDO') + } + + await ctx.updated(page) + + expect(mockBucket.file).toHaveBeenCalledWith('es/bienvenida.md') + }) + + it('makes the file public when publicRead is enabled', async () => { + const ctx = makeContext({ publicRead: true }) + ctx.bucket = mockBucket + const page = { + path: 'en/home', + localeCode: 'en', + contentType: 'markdown', + injectMetadata: jest.fn(() => 'PAGE CONTENT') + } + + await ctx.created(page) + + expect(mockFile.makePublic).toHaveBeenCalled() + }) + + it('does not throw when makePublic fails, and logs a warning instead', async () => { + mockFile.makePublic.mockRejectedValueOnce(new Error('permission denied')) + const ctx = makeContext({ publicRead: true }) + ctx.bucket = mockBucket + const page = { + path: 'en/home', + localeCode: 'en', + contentType: 'markdown', + injectMetadata: jest.fn(() => 'PAGE CONTENT') + } + + await expect(ctx.created(page)).resolves.toBeUndefined() + expect(global.WIKI.logger.warn).toHaveBeenCalled() + }) + }) + + describe('deleted', () => { + it('deletes the file at the page path, ignoring not-found errors', async () => { + const ctx = makeContext({}) + ctx.bucket = mockBucket + const page = { path: 'en/old-page', localeCode: 'en', contentType: 'markdown' } + + await ctx.deleted(page) + + expect(mockBucket.file).toHaveBeenCalledWith('en/old-page.md') + expect(mockFile.delete).toHaveBeenCalledWith({ ignoreNotFound: true }) + }) + }) + + describe('renamed', () => { + it('moves the file from the source path to the destination path', async () => { + const ctx = makeContext({}) + ctx.bucket = mockBucket + const page = { + path: 'en/old-path', + destinationPath: 'en/new-path', + localeCode: 'en', + destinationLocaleCode: 'en', + contentType: 'markdown' + } + + await ctx.renamed(page) + + expect(mockBucket.file).toHaveBeenCalledWith('en/old-path.md') + expect(mockFile.move).toHaveBeenCalledWith('en/new-path.md') + }) + + it('namespaces source and destination independently when locales differ', async () => { + global.WIKI.config.lang.namespacing = true + global.WIKI.config.lang.code = 'en' + + const ctx = makeContext({}) + ctx.bucket = mockBucket + const page = { + path: 'bienvenida', + destinationPath: 'welcome', + localeCode: 'es', + destinationLocaleCode: 'fr', + contentType: 'markdown' + } + + await ctx.renamed(page) + + expect(mockBucket.file).toHaveBeenCalledWith('es/bienvenida.md') + expect(mockFile.move).toHaveBeenCalledWith('fr/welcome.md') + }) + }) + + describe('assets', () => { + it('assetUploaded saves the asset data to its path', async () => { + const ctx = makeContext({ publicRead: false }) + ctx.bucket = mockBucket + const asset = { path: 'uploads/logo.png', data: Buffer.from('binary') } + + await ctx.assetUploaded(asset) + + expect(mockBucket.file).toHaveBeenCalledWith('uploads/logo.png') + expect(mockFile.save).toHaveBeenCalledWith(asset.data, { resumable: false }) + }) + + it('assetDeleted deletes the file, ignoring not-found errors', async () => { + const ctx = makeContext({}) + ctx.bucket = mockBucket + const asset = { path: 'uploads/logo.png' } + + await ctx.assetDeleted(asset) + + expect(mockFile.delete).toHaveBeenCalledWith({ ignoreNotFound: true }) + }) + + it('assetRenamed moves the file to the destination path', async () => { + const ctx = makeContext({}) + ctx.bucket = mockBucket + const asset = { path: 'uploads/old.png', destinationPath: 'uploads/new.png' } + + await ctx.assetRenamed(asset) + + expect(mockBucket.file).toHaveBeenCalledWith('uploads/old.png') + expect(mockFile.move).toHaveBeenCalledWith('uploads/new.png') + }) + }) +})