From 775a8f3e4579ebf1c335113cf6ec6f74cd258694 Mon Sep 17 00:00:00 2001 From: Zazama Date: Wed, 5 Apr 2023 21:40:01 +0200 Subject: [PATCH 1/6] Stream files on read instead of fully consuming them --- src/api/read.ts | 5 +-- src/id3-tag.ts | 7 +++- src/util-file.ts | 90 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 src/util-file.ts diff --git a/src/api/read.ts b/src/api/read.ts index 7d9baa6..0b0cf87 100644 --- a/src/api/read.ts +++ b/src/api/read.ts @@ -3,6 +3,7 @@ import { getTagsFromId3Tag } from '../id3-tag' import { isFunction, isString } from '../util' import { Tags, TagIdentifiers } from '../types/Tags' import { Options } from '../types/Options' +import { getId3TagDataFromFileAsync, getId3TagDataFromFileSync } from '../util-file' /** * Callback signature for successful asynchronous read operation. @@ -71,7 +72,7 @@ export function read( function readSync(filebuffer: string | Buffer, options: Options) { if (isString(filebuffer)) { - filebuffer = fs.readFileSync(filebuffer) + filebuffer = getId3TagDataFromFileSync(filebuffer) } return getTagsFromId3Tag(filebuffer, options) } @@ -82,7 +83,7 @@ function readAsync( callback: ReadCallback ) { if (isString(filebuffer)) { - fs.readFile(filebuffer, (error, data) => { + getId3TagDataFromFileAsync(filebuffer, (error, data) => { if(error) { callback(error, null) } else { diff --git a/src/id3-tag.ts b/src/id3-tag.ts index 98f0b99..d71bfd1 100644 --- a/src/id3-tag.ts +++ b/src/id3-tag.ts @@ -112,6 +112,11 @@ export function getTagsFromId3Tag(buffer: Buffer, options: Options) { return getTags(tagBody, options) } +export function getId3TagSize(buffer: Buffer): number { + const encodedSize = subarray(buffer, Header.offset.size, 4) + return Header.size + decodeSize(encodedSize); +} + function getId3TagBody(buffer: Buffer) { const tagPosition = findId3TagPosition(buffer) if (tagPosition === -1) { @@ -202,7 +207,7 @@ function parseTagHeaderFlags(header: Buffer): TagHeaderFlags { /** * Returns the position of the first valid tag found or -1 if no tag was found. */ -function findId3TagPosition(buffer: Buffer) { +export function findId3TagPosition(buffer: Buffer) { // Search Buffer for valid ID3 frame let position = -1 do { diff --git a/src/util-file.ts b/src/util-file.ts new file mode 100644 index 0000000..5758b23 --- /dev/null +++ b/src/util-file.ts @@ -0,0 +1,90 @@ +import * as fs from 'fs' +import { promisify } from 'util' +import { findId3TagPosition, getId3TagSize } from './id3-tag' + +const FileBufferSize = 20 * 1024 * 1024 +const Id3TagHeaderSize = 10 + +const fsOpenPromise = promisify(fs.open) +const fsReadPromise = promisify(fs.read) +const fsClosePromise = promisify(fs.close) + +type SuccessCallback = (err: null, buffer: Buffer) => void +type ErrorCallback = (err: Error, buffer: Buffer) => void +type Callback = SuccessCallback & ErrorCallback + +export function getId3TagDataFromFileSync(filepath: string): Buffer { + const fileDescriptor = fs.openSync(filepath, 'r') + const buffer = Buffer.alloc(FileBufferSize) + + while(fs.readSync(fileDescriptor, buffer, {offset: Id3TagHeaderSize})) { + const id3TagPosition = findId3TagPosition(buffer) + if(id3TagPosition === -1) { + buffer.copyWithin(0, buffer.length - 10) + continue + } + fs.closeSync(fileDescriptor) + return completePartialId3TagData( + fileDescriptor, + buffer.subarray(id3TagPosition) + ) + } + + fs.closeSync(fileDescriptor) + return Buffer.alloc(0) +} + +export function getId3TagDataFromFileAsync(filepath: string, callback: Callback) { + fsOpenPromise(filepath, 'r').then(async (fileDescriptor) => { + const buffer = Buffer.alloc(FileBufferSize) + while((await fsReadPromise(fileDescriptor, {buffer, offset: Id3TagHeaderSize})).bytesRead) { + const id3TagPosition = findId3TagPosition(buffer) + if(id3TagPosition === -1) { + buffer.copyWithin(0, buffer.length - 10) + continue + } + await fsClosePromise(fileDescriptor) + callback(null, await completePartialId3TagDataAsync( + fileDescriptor, + buffer.subarray(id3TagPosition) + )) + } + + await fsClosePromise(fileDescriptor) + callback(null, Buffer.alloc(0)) + }).catch((error: Error) => { + callback(error, Buffer.alloc(0)) + }) +} + +function calculateMissingBytes(id3TagSize: number, id3TagBuffer: Buffer) { + return Math.max(0, id3TagSize - id3TagBuffer.length) +} + +function completePartialId3TagData(fileDescriptor: number, partialId3TagData: Buffer): Buffer { + const id3TagSize = getId3TagSize(partialId3TagData); + const missingBytesCount = calculateMissingBytes(id3TagSize, partialId3TagData) + if(missingBytesCount) { + const id3TagRemainingBuffer = Buffer.alloc(missingBytesCount, 0x00) + fs.readSync(fileDescriptor, id3TagRemainingBuffer) + return Buffer.concat([ + partialId3TagData, + id3TagRemainingBuffer + ]) + } + return partialId3TagData.subarray(0, id3TagSize) +} + +async function completePartialId3TagDataAsync(fileDescriptor: number, partialId3TagData: Buffer): Promise { + const id3TagSize = getId3TagSize(partialId3TagData); + const missingBytesCount = calculateMissingBytes(id3TagSize, partialId3TagData) + if(missingBytesCount) { + const id3TagRemainingBuffer = Buffer.alloc(missingBytesCount, 0x00) + await fsReadPromise(fileDescriptor, {buffer: id3TagRemainingBuffer}) + return Buffer.concat([ + partialId3TagData, + id3TagRemainingBuffer + ]) + } + return partialId3TagData.subarray(0, id3TagSize) +} \ No newline at end of file From 69f0c10254a144111f5858ce952824fa31f8db93 Mon Sep 17 00:00:00 2001 From: Zazama Date: Thu, 6 Apr 2023 23:30:19 +0200 Subject: [PATCH 2/6] Fix fs close bugs and refactor --- src/api/read.ts | 6 +- src/file-read.ts | 139 +++++++++++++++++++++++++++++++++++++++++++++++ src/id3-tag.ts | 4 +- src/util-file.ts | 90 ------------------------------ 4 files changed, 144 insertions(+), 95 deletions(-) create mode 100644 src/file-read.ts delete mode 100644 src/util-file.ts diff --git a/src/api/read.ts b/src/api/read.ts index 0b0cf87..1c7f541 100644 --- a/src/api/read.ts +++ b/src/api/read.ts @@ -3,7 +3,7 @@ import { getTagsFromId3Tag } from '../id3-tag' import { isFunction, isString } from '../util' import { Tags, TagIdentifiers } from '../types/Tags' import { Options } from '../types/Options' -import { getId3TagDataFromFileAsync, getId3TagDataFromFileSync } from '../util-file' +import { getId3TagDataFromFileAsync, getId3TagDataFromFileSync } from '../file-read' /** * Callback signature for successful asynchronous read operation. @@ -72,7 +72,7 @@ export function read( function readSync(filebuffer: string | Buffer, options: Options) { if (isString(filebuffer)) { - filebuffer = getId3TagDataFromFileSync(filebuffer) + filebuffer = getId3TagDataFromFileSync(filebuffer) ?? Buffer.alloc(0) } return getTagsFromId3Tag(filebuffer, options) } @@ -87,7 +87,7 @@ function readAsync( if(error) { callback(error, null) } else { - callback(null, getTagsFromId3Tag(data, options)) + callback(null, getTagsFromId3Tag(data ?? Buffer.alloc(0), options)) } }) } else { diff --git a/src/file-read.ts b/src/file-read.ts new file mode 100644 index 0000000..042ab90 --- /dev/null +++ b/src/file-read.ts @@ -0,0 +1,139 @@ +import * as fs from 'fs' +import { promisify } from 'util' +import { findId3TagPosition, getId3TagSize, Header } from './id3-tag' + +const FileBufferSize = 20 * 1024 * 1024 + +const fsOpenPromise = promisify(fs.open) +const fsReadPromise = promisify(fs.read) +const fsClosePromise = promisify(fs.close) + +type SuccessCallback = (err: null, buffer: Buffer|null) => void +type ErrorCallback = (err: Error, buffer: null) => void +type Callback = SuccessCallback & ErrorCallback + +export function getId3TagDataFromFileSync(filepath: string): Buffer|null { + return processFile(filepath, 'r', (fileDescriptor) => { + const partialId3TagData = findPartialId3TagSync(fileDescriptor) + return partialId3TagData ? completePartialId3TagData( + fileDescriptor, + partialId3TagData + ) : null + }) +} + +export function getId3TagDataFromFileAsync(filepath: string, callback: Callback) { + processFileAsync(filepath, 'r', async (fileDescriptor) => { + const partialId3TagData = await findPartialId3TagAsync(fileDescriptor) + return partialId3TagData ? completePartialId3TagDataAsync( + fileDescriptor, + partialId3TagData + ) : null + }).then((data) => { + callback(null, data) + }).catch((error) => { + callback(error, null) + }) +} + +function findPartialId3TagSync(fileDescriptor: number): Buffer|null { + const buffer = Buffer.alloc(FileBufferSize) + let data + while((data = getNextBufferSubarraySync(fileDescriptor, buffer)).length > Header.size) { + const id3TagPosition = findId3TagPosition(data) + if(id3TagPosition !== -1) { + return data.subarray(id3TagPosition) + } + buffer.copyWithin(0, buffer.length - Header.size) + } + return null +} + +async function findPartialId3TagAsync(fileDescriptor: number): Promise { + const buffer = Buffer.alloc(FileBufferSize) + let data + while((data = await getNextBufferSubarrayAsync(fileDescriptor, buffer)).length > Header.size) { + const id3TagPosition = findId3TagPosition(data) + if(id3TagPosition !== -1) { + return data.subarray(id3TagPosition) + } + buffer.copyWithin(0, buffer.length - Header.size) + } + return null +} + +function getNextBufferSubarraySync(fileDescriptor: number, buffer: Buffer): Buffer { + const bytesRead = fs.readSync(fileDescriptor, buffer, {offset: Header.size}) + return buffer.subarray(0, bytesRead + Header.size) +} + +async function getNextBufferSubarrayAsync(fileDescriptor: number, buffer: Buffer): Promise { + const bytesRead = (await fsReadPromise(fileDescriptor, {buffer, offset: Header.size})).bytesRead + return buffer.subarray(0, bytesRead + Header.size) +} + +function processFile( + filepath: string, + flags: string, + process: (fileDescriptor: number) => T +) { + const fileDescriptor = fs.openSync(filepath, flags) + try { + return process(fileDescriptor) + } + catch (error) { + throw error + } + finally { + fs.closeSync(fileDescriptor) + } +} + +async function processFileAsync( + filepath: string, + flags: string, + process: (fileDescriptor: number) => Promise +): Promise { + const fileDescriptor = await fsOpenPromise(filepath, flags) + try { + return await process(fileDescriptor) + } + catch (error) { + throw error + } + finally { + await fsClosePromise(fileDescriptor) + } +} + +function calculateMissingBytes(id3TagSize: number, id3TagBuffer: Buffer): number { + return Math.max(0, id3TagSize - id3TagBuffer.length) +} + +function completePartialId3TagData(fileDescriptor: number, partialId3TagData: Buffer): Buffer { + const id3TagSize = getId3TagSize(partialId3TagData) + const missingBytesCount = calculateMissingBytes(id3TagSize, partialId3TagData) + if(missingBytesCount) { + const id3TagRemainingBuffer = Buffer.alloc(missingBytesCount, 0x00) + fs.readSync(fileDescriptor, id3TagRemainingBuffer) + return Buffer.concat([ + partialId3TagData, + id3TagRemainingBuffer + ]) + } + return partialId3TagData.subarray(0, id3TagSize) +} + +async function completePartialId3TagDataAsync(fileDescriptor: number, partialId3TagData: Buffer): Promise { + const id3TagSize = getId3TagSize(partialId3TagData) + const missingBytesCount = calculateMissingBytes(id3TagSize, partialId3TagData) + if(missingBytesCount) { + const id3TagRemainingBuffer = Buffer.alloc(missingBytesCount, 0x00) + await fsReadPromise(fileDescriptor, {buffer: id3TagRemainingBuffer}) + return Buffer.concat([ + partialId3TagData, + id3TagRemainingBuffer + ]) + } + return partialId3TagData.subarray(0, id3TagSize) +} \ No newline at end of file diff --git a/src/id3-tag.ts b/src/id3-tag.ts index d71bfd1..2a69869 100644 --- a/src/id3-tag.ts +++ b/src/id3-tag.ts @@ -4,7 +4,7 @@ import { Options } from "./types/Options" import { WriteTags } from "./types/Tags" import { decodeSize, encodeSize } from "./util-size" -const Header = { +export const Header = { identifier: "ID3", size: 10, offset: { @@ -114,7 +114,7 @@ export function getTagsFromId3Tag(buffer: Buffer, options: Options) { export function getId3TagSize(buffer: Buffer): number { const encodedSize = subarray(buffer, Header.offset.size, 4) - return Header.size + decodeSize(encodedSize); + return Header.size + decodeSize(encodedSize) } function getId3TagBody(buffer: Buffer) { diff --git a/src/util-file.ts b/src/util-file.ts deleted file mode 100644 index 5758b23..0000000 --- a/src/util-file.ts +++ /dev/null @@ -1,90 +0,0 @@ -import * as fs from 'fs' -import { promisify } from 'util' -import { findId3TagPosition, getId3TagSize } from './id3-tag' - -const FileBufferSize = 20 * 1024 * 1024 -const Id3TagHeaderSize = 10 - -const fsOpenPromise = promisify(fs.open) -const fsReadPromise = promisify(fs.read) -const fsClosePromise = promisify(fs.close) - -type SuccessCallback = (err: null, buffer: Buffer) => void -type ErrorCallback = (err: Error, buffer: Buffer) => void -type Callback = SuccessCallback & ErrorCallback - -export function getId3TagDataFromFileSync(filepath: string): Buffer { - const fileDescriptor = fs.openSync(filepath, 'r') - const buffer = Buffer.alloc(FileBufferSize) - - while(fs.readSync(fileDescriptor, buffer, {offset: Id3TagHeaderSize})) { - const id3TagPosition = findId3TagPosition(buffer) - if(id3TagPosition === -1) { - buffer.copyWithin(0, buffer.length - 10) - continue - } - fs.closeSync(fileDescriptor) - return completePartialId3TagData( - fileDescriptor, - buffer.subarray(id3TagPosition) - ) - } - - fs.closeSync(fileDescriptor) - return Buffer.alloc(0) -} - -export function getId3TagDataFromFileAsync(filepath: string, callback: Callback) { - fsOpenPromise(filepath, 'r').then(async (fileDescriptor) => { - const buffer = Buffer.alloc(FileBufferSize) - while((await fsReadPromise(fileDescriptor, {buffer, offset: Id3TagHeaderSize})).bytesRead) { - const id3TagPosition = findId3TagPosition(buffer) - if(id3TagPosition === -1) { - buffer.copyWithin(0, buffer.length - 10) - continue - } - await fsClosePromise(fileDescriptor) - callback(null, await completePartialId3TagDataAsync( - fileDescriptor, - buffer.subarray(id3TagPosition) - )) - } - - await fsClosePromise(fileDescriptor) - callback(null, Buffer.alloc(0)) - }).catch((error: Error) => { - callback(error, Buffer.alloc(0)) - }) -} - -function calculateMissingBytes(id3TagSize: number, id3TagBuffer: Buffer) { - return Math.max(0, id3TagSize - id3TagBuffer.length) -} - -function completePartialId3TagData(fileDescriptor: number, partialId3TagData: Buffer): Buffer { - const id3TagSize = getId3TagSize(partialId3TagData); - const missingBytesCount = calculateMissingBytes(id3TagSize, partialId3TagData) - if(missingBytesCount) { - const id3TagRemainingBuffer = Buffer.alloc(missingBytesCount, 0x00) - fs.readSync(fileDescriptor, id3TagRemainingBuffer) - return Buffer.concat([ - partialId3TagData, - id3TagRemainingBuffer - ]) - } - return partialId3TagData.subarray(0, id3TagSize) -} - -async function completePartialId3TagDataAsync(fileDescriptor: number, partialId3TagData: Buffer): Promise { - const id3TagSize = getId3TagSize(partialId3TagData); - const missingBytesCount = calculateMissingBytes(id3TagSize, partialId3TagData) - if(missingBytesCount) { - const id3TagRemainingBuffer = Buffer.alloc(missingBytesCount, 0x00) - await fsReadPromise(fileDescriptor, {buffer: id3TagRemainingBuffer}) - return Buffer.concat([ - partialId3TagData, - id3TagRemainingBuffer - ]) - } - return partialId3TagData.subarray(0, id3TagSize) -} \ No newline at end of file From 18486580f2edc65025801076aa23c73b546803d7 Mon Sep 17 00:00:00 2001 From: Zazama Date: Thu, 6 Apr 2023 23:44:54 +0200 Subject: [PATCH 3/6] Add NodeJS 10 support for file streaming --- src/file-read.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/file-read.ts b/src/file-read.ts index 042ab90..b877351 100644 --- a/src/file-read.ts +++ b/src/file-read.ts @@ -63,12 +63,24 @@ async function findPartialId3TagAsync(fileDescriptor: number): Promise { - const bytesRead = (await fsReadPromise(fileDescriptor, {buffer, offset: Header.size})).bytesRead + const bytesRead = (await fsReadPromise( + fileDescriptor, + buffer, + Header.size, + buffer.length - Header.size, + null + )).bytesRead return buffer.subarray(0, bytesRead + Header.size) } @@ -81,9 +93,6 @@ function processFile( try { return process(fileDescriptor) } - catch (error) { - throw error - } finally { fs.closeSync(fileDescriptor) } @@ -98,9 +107,6 @@ async function processFileAsync( try { return await process(fileDescriptor) } - catch (error) { - throw error - } finally { await fsClosePromise(fileDescriptor) } From 0c9321051e46f77ef374e6cd16de47e2164d89e1 Mon Sep 17 00:00:00 2001 From: Zazama Date: Fri, 7 Apr 2023 18:16:53 +0200 Subject: [PATCH 4/6] Add file streaming to write operation --- package.json | 4 +- src/api/promises.ts | 6 +- src/api/write.ts | 51 ++++++++++------- src/file-read.ts | 60 +------------------- src/file-write.ts | 134 ++++++++++++++++++++++++++++++++++++++++++++ src/util-file.ts | 59 +++++++++++++++++++ 6 files changed, 233 insertions(+), 81 deletions(-) create mode 100644 src/file-write.ts create mode 100644 src/util-file.ts diff --git a/package.json b/package.json index f02f8a3..625b631 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,8 @@ }, "license": "MIT", "dependencies": { - "iconv-lite": "0.6.2" + "iconv-lite": "0.6.2", + "tmp": "^0.2.1" }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.2", @@ -57,6 +58,7 @@ "@types/mocha": "^10.0.1", "@types/node": "^18.11.10", "@types/source-map-support": "^0.5.6", + "@types/tmp": "^0.2.3", "@typescript-eslint/eslint-plugin": "^5.45.0", "@typescript-eslint/parser": "^5.45.0", "chai": "^4.3.7", diff --git a/src/api/promises.ts b/src/api/promises.ts index a5eb434..7d93f92 100644 --- a/src/api/promises.ts +++ b/src/api/promises.ts @@ -4,7 +4,7 @@ import { create } from "./create" import { read, ReadCallback } from "./read" import { removeTags } from "./remove" import { update } from "./update" -import { write, WriteCallback } from "./write" +import { write, WriteCallback, WriteFileCallback } from "./write" type Settle = { (error: NodeJS.ErrnoException | Error, result: null): void @@ -38,11 +38,11 @@ export const Promises = { create(tags, result => settle(null, result)), ), write: (tags: WriteTags, filebuffer: string | Buffer) => - makePromise((callback: WriteCallback) => + makePromise((callback: WriteCallback | WriteFileCallback) => write(tags, filebuffer, callback) ), update: (tags: WriteTags, filebuffer: string | Buffer, options?: Options) => - makePromise((callback: WriteCallback) => + makePromise((callback: WriteCallback | WriteFileCallback) => update(tags, filebuffer, options ?? {}, callback) ), read: (file: string | Buffer, options?: Options) => diff --git a/src/api/write.ts b/src/api/write.ts index 50a3c93..6fdc9cd 100644 --- a/src/api/write.ts +++ b/src/api/write.ts @@ -3,6 +3,7 @@ import { WriteTags } from "../types/Tags" import { create } from "./create" import { removeTagsFromBuffer } from "./remove" import { isFunction, isString } from "../util" +import { writeId3TagToFileAsync, writeId3TagToFileSync } from "../file-write" /** * Callback signature for successful asynchronous update and write operations. @@ -28,6 +29,9 @@ export type WriteErrorCallback = export type WriteCallback = WriteSuccessCallback & WriteErrorCallback +export type WriteFileCallback = + (error: NodeJS.ErrnoException | Error | null) => void + /** * Replaces any existing tags with the given tags in the given buffer. * @@ -43,6 +47,26 @@ export function write(tags: WriteTags, buffer: Buffer): Buffer */ export function write(tags: WriteTags, filepath: string): true | Error +/** + * Replaces asynchronously any existing tags with the given tags in the + * given buffer. + * + * @public + */ +export function write( + tags: WriteTags, filebuffer: Buffer, callback: WriteCallback +): void + +/** + * Replaces asynchronously any existing tags with the given tags in the + * given file. + * + * @public + */ +export function write( + tags: WriteTags, filebuffer: string, callback: WriteFileCallback +): void + /** * Replaces asynchronously any existing tags with the given tags in the * given buffer or specified file. @@ -50,19 +74,19 @@ export function write(tags: WriteTags, filepath: string): true | Error * @public */ export function write( - tags: WriteTags, filebuffer: string | Buffer, callback: WriteCallback + tags: WriteTags, filebuffer: string | Buffer, callback: WriteFileCallback | WriteCallback ): void export function write( tags: WriteTags, filebuffer: string | Buffer, - callback?: WriteCallback + callback?: WriteCallback | WriteFileCallback ): Buffer | true | Error | void { const tagsBuffer = create(tags) if (isFunction(callback)) { if (isString(filebuffer)) { - return writeAsync(tagsBuffer, filebuffer, callback) + return writeAsync(tagsBuffer, filebuffer, callback as WriteFileCallback) } return callback(null, writeInBuffer(tagsBuffer, filebuffer)) } @@ -77,28 +101,15 @@ function writeInBuffer(tags: Buffer, buffer: Buffer) { return Buffer.concat([tags, buffer]) } -function writeAsync(tags: Buffer, filepath: string, callback: WriteCallback) { - fs.readFile(filepath, (error, data) => { - if (error) { - callback(error, null) - return - } - const newData = writeInBuffer(tags, data) - fs.writeFile(filepath, newData, 'binary', (error) => { - if (error) { - callback(error, null) - } else { - callback(null, newData) - } - }) +function writeAsync(tags: Buffer, filepath: string, callback: WriteFileCallback) { + writeId3TagToFileAsync(filepath, tags, (err) => { + callback(err) }) } function writeSync(tags: Buffer, filepath: string) { try { - const data = fs.readFileSync(filepath) - const newData = writeInBuffer(tags, data) - fs.writeFileSync(filepath, newData, 'binary') + writeId3TagToFileSync(filepath, tags) return true } catch(error) { return error as Error diff --git a/src/file-read.ts b/src/file-read.ts index b877351..a41f5de 100644 --- a/src/file-read.ts +++ b/src/file-read.ts @@ -1,13 +1,9 @@ import * as fs from 'fs' -import { promisify } from 'util' import { findId3TagPosition, getId3TagSize, Header } from './id3-tag' +import { fsReadPromise, getNextBufferSubarrayAsync, getNextBufferSubarraySync, processFile, processFileAsync } from './util-file' const FileBufferSize = 20 * 1024 * 1024 -const fsOpenPromise = promisify(fs.open) -const fsReadPromise = promisify(fs.read) -const fsClosePromise = promisify(fs.close) - type SuccessCallback = (err: null, buffer: Buffer|null) => void type ErrorCallback = (err: Error, buffer: null) => void type Callback = SuccessCallback & ErrorCallback @@ -39,7 +35,7 @@ export function getId3TagDataFromFileAsync(filepath: string, callback: Callback) function findPartialId3TagSync(fileDescriptor: number): Buffer|null { const buffer = Buffer.alloc(FileBufferSize) let data - while((data = getNextBufferSubarraySync(fileDescriptor, buffer)).length > Header.size) { + while((data = getNextBufferSubarraySync(fileDescriptor, buffer, Header.size)).length > Header.size) { const id3TagPosition = findId3TagPosition(data) if(id3TagPosition !== -1) { return data.subarray(id3TagPosition) @@ -52,7 +48,7 @@ function findPartialId3TagSync(fileDescriptor: number): Buffer|null { async function findPartialId3TagAsync(fileDescriptor: number): Promise { const buffer = Buffer.alloc(FileBufferSize) let data - while((data = await getNextBufferSubarrayAsync(fileDescriptor, buffer)).length > Header.size) { + while((data = await getNextBufferSubarrayAsync(fileDescriptor, buffer, Header.size)).length > Header.size) { const id3TagPosition = findId3TagPosition(data) if(id3TagPosition !== -1) { return data.subarray(id3TagPosition) @@ -62,56 +58,6 @@ async function findPartialId3TagAsync(fileDescriptor: number): Promise { - const bytesRead = (await fsReadPromise( - fileDescriptor, - buffer, - Header.size, - buffer.length - Header.size, - null - )).bytesRead - return buffer.subarray(0, bytesRead + Header.size) -} - -function processFile( - filepath: string, - flags: string, - process: (fileDescriptor: number) => T -) { - const fileDescriptor = fs.openSync(filepath, flags) - try { - return process(fileDescriptor) - } - finally { - fs.closeSync(fileDescriptor) - } -} - -async function processFileAsync( - filepath: string, - flags: string, - process: (fileDescriptor: number) => Promise -): Promise { - const fileDescriptor = await fsOpenPromise(filepath, flags) - try { - return await process(fileDescriptor) - } - finally { - await fsClosePromise(fileDescriptor) - } -} - function calculateMissingBytes(id3TagSize: number, id3TagBuffer: Buffer): number { return Math.max(0, id3TagSize - id3TagBuffer.length) } diff --git a/src/file-write.ts b/src/file-write.ts new file mode 100644 index 0000000..d43a10e --- /dev/null +++ b/src/file-write.ts @@ -0,0 +1,134 @@ +import { + fsReadPromise, + fsRenamePromise, + fsUnlinkPromise, + fsWritePromise, + getNextBufferSubarrayAsync, + getNextBufferSubarraySync, + processFile, + processFileAsync +} from "./util-file" +import * as tmp from 'tmp' +import * as path from 'path' +import * as fs from 'fs' +import { findId3TagPosition, getId3TagSize } from "./id3-tag" + +const FileBufferSize = 20 * 1024 * 1024 + +export function writeId3TagToFileSync(filepath: string, id3Tag: Buffer) { + const tmpFile = getTmpFilePathSync(filepath) + processFile(filepath, 'r', (readFileDescriptor) => { + processFile(tmpFile, 'w', (writeFileDescriptor) => { + fs.writeSync(writeFileDescriptor, id3Tag) + streamOriginalIntoNewFileSync(readFileDescriptor, writeFileDescriptor) + }) + }) + fs.unlinkSync(filepath) + fs.renameSync(tmpFile, filepath) +} + +export function writeId3TagToFileAsync(filepath: string, id3Tag: Buffer, callback: (err: Error|null) => void) { + getTmpFileAsync(filepath, (err, tmpFile) => { + if(err || !tmpFile) { + return callback(err) + } + + processFileAsync(filepath, 'r', async (readFileDescriptor) => { + return processFileAsync(tmpFile, 'w', async (writeFileDescriptor) => { + await fsWritePromise(writeFileDescriptor, id3Tag) + await streamOriginalIntoNewFileAsync(readFileDescriptor, writeFileDescriptor) + }) + }).then(async () => { + await fsUnlinkPromise(filepath) + await fsRenamePromise(tmpFile, filepath) + }).catch((error) => { + callback(error) + }) + }) +} + +function getTmpFilePathSync(filepath: string): string { + const parsedPath = path.parse(filepath) + return tmp.tmpNameSync({ + tmpdir: parsedPath.dir, + template: `${parsedPath.base}.tmp-XXXXXX`, + }) +} + +function getTmpFileAsync(filepath: string, callback: tmp.TmpNameCallback) { + const parsedPath = path.parse(filepath) + tmp.tmpName({ + tmpdir: parsedPath.dir, + template: `${parsedPath.base}.tmp-XXXXXX`, + }, (err, filename) => { + callback(err, filename) + }) +} + +function streamOriginalIntoNewFileSync(readFileDescriptor: number, writeFileDescriptor: number) { + const buffer = Buffer.alloc(FileBufferSize) + let data + while((data = getNextBufferSubarraySync(readFileDescriptor, buffer)).length) { + const id3TagPosition = findId3TagPosition(data) + if(id3TagPosition !== -1) { + data = getBufferWithoutId3TagAndSkipSync(readFileDescriptor, data, id3TagPosition) + } + fs.writeSync(writeFileDescriptor, data, 0, data.length, null) + } +} + +async function streamOriginalIntoNewFileAsync(readFileDescriptor: number, writeFileDescriptor: number) { + const buffer = Buffer.alloc(FileBufferSize) + let data + while((data = await getNextBufferSubarrayAsync(readFileDescriptor, buffer)).length) { + const id3TagPosition = findId3TagPosition(data) + if(id3TagPosition !== -1) { + data = await getBufferWithoutId3TagAndSkipAsync(readFileDescriptor, data, id3TagPosition) + } + await fsWritePromise(writeFileDescriptor, data, 0, data.length, null) + } +} + +function getBufferWithoutId3TagAndSkipSync(fileDescriptor: number, data: Buffer, id3TagPosition: number): Buffer { + const dataFromId3Start = data.subarray(id3TagPosition) + const id3TagSize = getId3TagSize(dataFromId3Start) + if(id3TagSize > dataFromId3Start.length) { + const missingBytesCount = id3TagSize - dataFromId3Start.length + fs.readSync( + fileDescriptor, + Buffer.alloc(missingBytesCount), + 0, + missingBytesCount, + null + ) + return data.subarray(0, id3TagPosition) + } + + const id3TagEndPosition = id3TagPosition + id3TagSize + return Buffer.concat([ + data.subarray(0, id3TagPosition), + data.subarray(id3TagEndPosition) + ]) +} + +async function getBufferWithoutId3TagAndSkipAsync(fileDescriptor: number, data: Buffer, id3TagPosition: number): Promise { + const dataFromId3Start = data.subarray(id3TagPosition) + const id3TagSize = getId3TagSize(dataFromId3Start) + if(id3TagSize > dataFromId3Start.length) { + const missingBytesCount = id3TagSize - dataFromId3Start.length + await fsReadPromise( + fileDescriptor, + Buffer.alloc(missingBytesCount), + 0, + missingBytesCount, + null + ) + return data.subarray(0, id3TagPosition) + } + + const id3TagEndPosition = id3TagPosition + id3TagSize + return Buffer.concat([ + data.subarray(0, id3TagPosition), + data.subarray(id3TagEndPosition) + ]) +} \ No newline at end of file diff --git a/src/util-file.ts b/src/util-file.ts new file mode 100644 index 0000000..d36d713 --- /dev/null +++ b/src/util-file.ts @@ -0,0 +1,59 @@ +import * as fs from 'fs' +import { promisify } from 'util' + +export const fsOpenPromise = promisify(fs.open) +export const fsReadPromise = promisify(fs.read) +export const fsClosePromise = promisify(fs.close) +export const fsWritePromise = promisify(fs.write) +export const fsUnlinkPromise = promisify(fs.unlink) +export const fsRenamePromise = promisify(fs.rename) + +export function processFile( + filepath: string, + flags: string, + process: (fileDescriptor: number) => T +) { + const fileDescriptor = fs.openSync(filepath, flags) + try { + return process(fileDescriptor) + } + finally { + fs.closeSync(fileDescriptor) + } +} + +export async function processFileAsync( + filepath: string, + flags: string, + process: (fileDescriptor: number) => Promise +): Promise { + const fileDescriptor = await fsOpenPromise(filepath, flags) + try { + return await process(fileDescriptor) + } + finally { + await fsClosePromise(fileDescriptor) + } +} + +export function getNextBufferSubarraySync(fileDescriptor: number, buffer: Buffer, offset: number = 0): Buffer { + const bytesRead = fs.readSync( + fileDescriptor, + buffer, + offset, + buffer.length - offset, + null + ) + return buffer.subarray(0, bytesRead + offset) +} + +export async function getNextBufferSubarrayAsync(fileDescriptor: number, buffer: Buffer, offset: number = 0): Promise { + const bytesRead = (await fsReadPromise( + fileDescriptor, + buffer, + offset, + buffer.length - offset, + null + )).bytesRead + return buffer.subarray(0, bytesRead + offset) +} \ No newline at end of file From 6160f45310f65703ea5a049d8096afe6c1335921 Mon Sep 17 00:00:00 2001 From: Zazama Date: Fri, 7 Apr 2023 18:29:58 +0200 Subject: [PATCH 5/6] eslint fixes --- src/util-file.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util-file.ts b/src/util-file.ts index d36d713..fbfbe23 100644 --- a/src/util-file.ts +++ b/src/util-file.ts @@ -36,7 +36,7 @@ export async function processFileAsync( } } -export function getNextBufferSubarraySync(fileDescriptor: number, buffer: Buffer, offset: number = 0): Buffer { +export function getNextBufferSubarraySync(fileDescriptor: number, buffer: Buffer, offset = 0): Buffer { const bytesRead = fs.readSync( fileDescriptor, buffer, @@ -47,7 +47,7 @@ export function getNextBufferSubarraySync(fileDescriptor: number, buffer: Buffer return buffer.subarray(0, bytesRead + offset) } -export async function getNextBufferSubarrayAsync(fileDescriptor: number, buffer: Buffer, offset: number = 0): Promise { +export async function getNextBufferSubarrayAsync(fileDescriptor: number, buffer: Buffer, offset = 0): Promise { const bytesRead = (await fsReadPromise( fileDescriptor, buffer, From 7a8672dcd0f3e662d3e28254487655822cefc184 Mon Sep 17 00:00:00 2001 From: Zazama Date: Fri, 7 Apr 2023 18:30:14 +0200 Subject: [PATCH 6/6] Call callback on success --- src/file-write.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/file-write.ts b/src/file-write.ts index d43a10e..ddba36f 100644 --- a/src/file-write.ts +++ b/src/file-write.ts @@ -41,6 +41,7 @@ export function writeId3TagToFileAsync(filepath: string, id3Tag: Buffer, callbac }).then(async () => { await fsUnlinkPromise(filepath) await fsRenamePromise(tmpFile, filepath) + callback(null) }).catch((error) => { callback(error) })