diff --git a/core.py b/core.py index 072d2d311..c287ec87d 100644 --- a/core.py +++ b/core.py @@ -417,6 +417,7 @@ def run_preprocess_script( chunk_len: float, overlap_len: float, normalization_mode: str = "none", + audio_format: str = "wav", ): preprocess_script_path = os.path.join("rvc", "train", "preprocess", "preprocess.py") command = [ @@ -436,6 +437,7 @@ def run_preprocess_script( chunk_len, overlap_len, normalization_mode, + audio_format, ], ), ] @@ -456,6 +458,8 @@ def run_extract_script( embedder_model: str, embedder_model_custom: str = None, include_mutes: int = 2, + extract_precision: str = "fp32", + remove_sliced_16k: bool = False, ): model_path = os.path.join(logs_path, model_name) extract = os.path.join("rvc", "train", "extract", "extract.py") @@ -474,6 +478,8 @@ def run_extract_script( embedder_model, embedder_model_custom, include_mutes, + extract_precision, + remove_sliced_16k, ], ), ] @@ -1055,6 +1061,15 @@ def tts(**kwargs): default="none", help="Normalization mode.", ) +@click.option( + "--audio-format", + type=click.Choice(["wav", "flac"]), + default="wav", + help=( + "Format of the sliced audio files. 'flac' saves ~30% disk, but quantizes to 24-bit " + "and clips peaks above 0 dBFS - avoid it with --normalization-mode none." + ), +) def preprocess(**kwargs): """Preprocess a dataset for training.""" kwargs["sample_rate"] = int(kwargs["sample_rate"]) @@ -1074,6 +1089,7 @@ def preprocess(**kwargs): chunk_len=kwargs["chunk_len"], overlap_len=kwargs["overlap_len"], normalization_mode=kwargs["normalization_mode"], + audio_format=kwargs["audio_format"], ) click.echo(result) @@ -1124,6 +1140,24 @@ def preprocess(**kwargs): default=2, help="Number of silent files to include.", ) +@click.option( + "--extract-precision", + type=click.Choice(["fp32", "fp16"]), + default="fp32", + help=( + "Format of the extracted files. 'fp16' halves disk usage: coarse f0 as uint8 is " + "lossless, features as float16 lose precision - negligible, but irreversible." + ), +) +@click.option( + "--remove-sliced-16k", + is_flag=True, + default=False, + help=( + "Delete the sliced_audios_16k folder after extraction. Training does not read those " + "files, but re-extracting later will require preprocessing again." + ), +) def extract(**kwargs): """Extract features from a preprocessed dataset.""" kwargs["sample_rate"] = int(kwargs["sample_rate"]) @@ -1137,6 +1171,8 @@ def extract(**kwargs): embedder_model=kwargs["embedder_model"], embedder_model_custom=kwargs["embedder_model_custom"], include_mutes=kwargs["include_mutes"], + extract_precision=kwargs["extract_precision"], + remove_sliced_16k=kwargs["remove_sliced_16k"], ) click.echo(result) diff --git a/rvc/train/data_utils.py b/rvc/train/data_utils.py index a363abd0a..33fbd4211 100644 --- a/rvc/train/data_utils.py +++ b/rvc/train/data_utils.py @@ -95,17 +95,18 @@ def get_labels(self, phone, pitch, pitchf): pitch (str): Path to pitch label file. pitchf (str): Path to pitchf label file. """ - phone = np.load(phone) + # features may be float16 and coarse pitch uint8 on disk + phone = np.load(phone).astype(np.float32) phone = np.repeat(phone, 2, axis=0) - pitch = np.load(pitch) - pitchf = np.load(pitchf) + pitch = np.load(pitch).astype(np.int64) + pitchf = np.load(pitchf).astype(np.float32) n_num = min(phone.shape[0], 900) phone = phone[:n_num, :] pitch = pitch[:n_num] pitchf = pitchf[:n_num] - phone = torch.FloatTensor(phone) - pitch = torch.LongTensor(pitch) - pitchf = torch.FloatTensor(pitchf) + phone = torch.from_numpy(phone) + pitch = torch.from_numpy(pitch) + pitchf = torch.from_numpy(pitchf) return phone, pitch, pitchf def get_audio(self, filename): @@ -122,7 +123,7 @@ def get_audio(self, filename): ) audio_norm = audio audio_norm = audio_norm.unsqueeze(0) - spec_filename = filename.replace(".wav", ".spec.pt") + spec_filename = os.path.splitext(filename)[0] + ".spec.pt" if os.path.exists(spec_filename): try: spec = torch.load(spec_filename, weights_only=True) diff --git a/rvc/train/extract/extract.py b/rvc/train/extract/extract.py index 63ff15bb0..c6fc82517 100644 --- a/rvc/train/extract/extract.py +++ b/rvc/train/extract/extract.py @@ -3,6 +3,7 @@ import json import multiprocessing as mp import os +import shutil import sys import time @@ -26,7 +27,7 @@ class FeatureInput: - def __init__(self, f0_method="rmvpe", device="cpu"): + def __init__(self, f0_method="rmvpe", device="cpu", quantize=False): self.hop_size = 160 # default self.sample_rate = 16000 # default self.f0_bin = 256 @@ -35,6 +36,7 @@ def __init__(self, f0_method="rmvpe", device="cpu"): self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700) self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700) self.device = device + self.quantize = quantize if f0_method in ("crepe", "crepe-tiny"): self.model = CREPE( device=self.device, sample_rate=self.sample_rate, hop_size=self.hop_size @@ -91,6 +93,9 @@ def process_file(self, file_info): feature_pit = self.compute_f0(np_arr) np.save(opt_path_full, feature_pit, allow_pickle=False) coarse_pit = self.coarse_f0(feature_pit) + if self.quantize: + # coarse pitch fits in [1, 255], so uint8 is lossless + coarse_pit = coarse_pit.astype(np.uint8) np.save(opt_path_coarse, coarse_pit, allow_pickle=False) except Exception as error: print( @@ -98,15 +103,15 @@ def process_file(self, file_info): ) -def process_files(files, f0_method, device, threads): - fe = FeatureInput(f0_method=f0_method, device=device) +def process_files(files, f0_method, device, threads, quantize=False): + fe = FeatureInput(f0_method=f0_method, device=device, quantize=quantize) with tqdm.tqdm(total=len(files), leave=True) as pbar: for file_info in files: fe.process_file(file_info) pbar.update(1) -def run_pitch_extraction(files, devices, f0_method, threads): +def run_pitch_extraction(files, devices, f0_method, threads, quantize=False): devices_str = ", ".join(devices) print(f"Starting pitch extraction on {devices_str} using {f0_method}...") start_time = time.time() @@ -119,6 +124,7 @@ def run_pitch_extraction(files, devices, f0_method, threads): f0_method, devices[i], threads // len(devices), + quantize, ) for i in range(len(devices)) ] @@ -128,7 +134,13 @@ def run_pitch_extraction(files, devices, f0_method, threads): def process_file_embedding( - files, embedder_model, embedder_model_custom, device_num, device, n_threads + files, + embedder_model, + embedder_model_custom, + device_num, + device, + n_threads, + half_precision=False, ): model = load_embedding(embedder_model, embedder_model_custom).to(device).float() model.eval() @@ -143,6 +155,8 @@ def worker(file_info): with torch.no_grad(): result = model(feats)["last_hidden_state"] feats_out = result.squeeze(0).float().cpu().numpy() + if half_precision: + feats_out = feats_out.astype(np.float16) if not np.isnan(feats_out).any(): np.save(out_file_path, feats_out, allow_pickle=False) else: @@ -156,7 +170,7 @@ def worker(file_info): def run_embedding_extraction( - files, devices, embedder_model, embedder_model_custom, threads + files, devices, embedder_model, embedder_model_custom, threads, half_precision=False ): devices_str = ", ".join(devices) print( @@ -173,6 +187,7 @@ def run_embedding_extraction( i, devices[i], threads // len(devices), + half_precision, ) for i in range(len(devices)) ] @@ -190,6 +205,14 @@ def run_embedding_extraction( embedder_model = sys.argv[6] embedder_model_custom = sys.argv[7] if len(sys.argv) > 7 else None include_mutes = int(sys.argv[8]) if len(sys.argv) > 8 else 2 + # fp16: features as float16, coarse f0 as uint8 + precision = sys.argv[9].lower() if len(sys.argv) > 9 else "fp32" + remove_sliced_16k = ( + sys.argv[10].lower() in ("yes", "true", "t", "y", "1") + if len(sys.argv) > 10 + else False + ) + half_precision = precision == "fp16" wav_path = os.path.join(exp_dir, "sliced_audios_16k") @@ -217,13 +240,17 @@ def run_embedding_extraction( json.dump(data, f, indent=4) files = [] - for file in glob.glob(os.path.join(wav_path, "*.wav")): + audio_files = sorted( + glob.glob(os.path.join(wav_path, "*.wav")) + + glob.glob(os.path.join(wav_path, "*.flac")) + ) + for file in audio_files: file_name = os.path.basename(file) file_info = [ file, os.path.join(exp_dir, "f0", file_name + ".npy"), os.path.join(exp_dir, "f0_voiced", file_name + ".npy"), - os.path.join(exp_dir, "extracted", file_name.replace("wav", "npy")), + os.path.join(exp_dir, "extracted", os.path.splitext(file_name)[0] + ".npy"), ] files.append(file_info) @@ -235,11 +262,23 @@ def run_embedding_extraction( devices = ["cpu"] if gpus == "-" else [f"cuda:{idx}" for idx in gpus.split("-")] - run_pitch_extraction(files, devices, f0_method, num_processes) + run_pitch_extraction(files, devices, f0_method, num_processes, half_precision) run_embedding_extraction( - files, devices, embedder_model, embedder_model_custom, num_processes + files, + devices, + embedder_model, + embedder_model_custom, + num_processes, + half_precision, ) generate_config(sample_rate, exp_dir) generate_filelist(exp_dir, sample_rate, include_mutes) + + if remove_sliced_16k: + try: + shutil.rmtree(wav_path) + print(f"Removed {wav_path} to save disk space.") + except OSError as error: + print(f"Could not remove {wav_path}: {error}") diff --git a/rvc/train/extract/preparing_files.py b/rvc/train/extract/preparing_files.py index 70f61eae2..31ca42d13 100644 --- a/rvc/train/extract/preparing_files.py +++ b/rvc/train/extract/preparing_files.py @@ -23,12 +23,33 @@ def generate_filelist(model_path: str, sample_rate: int, include_mutes: int = 2) f0_dir = os.path.join(model_path, "f0") f0nsf_dir = os.path.join(model_path, "f0_voiced") - gt_wavs_files = set(name.split(".")[0] for name in os.listdir(gt_wavs_dir)) - feature_files = set(name.split(".")[0] for name in os.listdir(feature_dir)) - - f0_files = set(name.split(".")[0] for name in os.listdir(f0_dir)) - f0nsf_files = set(name.split(".")[0] for name in os.listdir(f0nsf_dir)) - names = gt_wavs_files & feature_files & f0_files & f0nsf_files + # slices may be .wav or .flac, and the f0 files inherit that extension + gt_wavs_files = { + name.split(".")[0]: name + for name in os.listdir(gt_wavs_dir) + if name.lower().endswith((".wav", ".flac")) + } + feature_files = { + name.split(".")[0]: name + for name in os.listdir(feature_dir) + if name.lower().endswith(".npy") + } + f0_files = { + name.split(".")[0]: name + for name in os.listdir(f0_dir) + if name.lower().endswith(".npy") + } + f0nsf_files = { + name.split(".")[0]: name + for name in os.listdir(f0nsf_dir) + if name.lower().endswith(".npy") + } + names = ( + gt_wavs_files.keys() + & feature_files.keys() + & f0_files.keys() + & f0nsf_files.keys() + ) try: model_info_path = os.path.join(model_path, "model_info.json") @@ -53,10 +74,10 @@ def generate_filelist(model_path: str, sample_rate: int, include_mutes: int = 2) sids.append(sid) # Calculate relative pathing - rel_wav = os.path.relpath(f"{os.path.join(gt_wavs_dir, name)}.wav") - rel_feat = os.path.relpath(f"{os.path.join(feature_dir, name)}.npy") - rel_f0 = os.path.relpath(f"{os.path.join(f0_dir, name)}.wav.npy") - rel_f0nsf = os.path.relpath(f"{os.path.join(f0nsf_dir, name)}.wav.npy") + rel_wav = os.path.relpath(os.path.join(gt_wavs_dir, gt_wavs_files[name])) + rel_feat = os.path.relpath(os.path.join(feature_dir, feature_files[name])) + rel_f0 = os.path.relpath(os.path.join(f0_dir, f0_files[name])) + rel_f0nsf = os.path.relpath(os.path.join(f0nsf_dir, f0nsf_files[name])) options.append( f"{rel_wav}|{rel_feat}|{rel_f0}|{rel_f0nsf}|{sid}".replace("\\", "/") diff --git a/rvc/train/preprocess/preprocess.py b/rvc/train/preprocess/preprocess.py index 123f4fc6d..45c3ae343 100644 --- a/rvc/train/preprocess/preprocess.py +++ b/rvc/train/preprocess/preprocess.py @@ -14,6 +14,7 @@ def strtobool(val): import librosa import noisereduce as nr import numpy as np +import soundfile as sf import soxr from scipy import signal from scipy.io import wavfile @@ -41,7 +42,7 @@ def strtobool(val): class PreProcess: - def __init__(self, sr: int, exp_dir: str): + def __init__(self, sr: int, exp_dir: str, audio_format: str = "wav"): self.slicer = Slicer( sr=sr, threshold=-42, @@ -56,11 +57,20 @@ def __init__(self, sr: int, exp_dir: str): ) self.exp_dir = exp_dir self.device = "cpu" + self.audio_format = audio_format.lower() self.gt_wavs_dir = os.path.join(exp_dir, "sliced_audios") self.wavs16k_dir = os.path.join(exp_dir, "sliced_audios_16k") os.makedirs(self.gt_wavs_dir, exist_ok=True) os.makedirs(self.wavs16k_dir, exist_ok=True) + def _write_audio(self, directory: str, name: str, sr: int, audio: np.ndarray): + path = os.path.join(directory, f"{name}.{self.audio_format}") + if self.audio_format == "flac": + # FLAC is integer-only, so clip before the 24 bit quantization + sf.write(path, np.clip(audio, -1.0, 1.0), sr, format="FLAC", subtype="PCM_24") + else: + wavfile.write(path, sr, audio.astype(np.float32)) + def _normalize_audio(self, audio: np.ndarray): tmp_max = np.abs(audio).max() if tmp_max > 2.5: @@ -80,10 +90,8 @@ def process_audio_segment( return if normalization_mode == "post": normalized_audio = self._normalize_audio(normalized_audio) - wavfile.write( - os.path.join(self.gt_wavs_dir, f"{sid}_{idx0}_{idx1}.wav"), - self.sr, - normalized_audio.astype(np.float32), + self._write_audio( + self.gt_wavs_dir, f"{sid}_{idx0}_{idx1}", self.sr, normalized_audio ) audio_16k = librosa.resample( normalized_audio, @@ -91,10 +99,8 @@ def process_audio_segment( target_sr=SAMPLE_RATE_16K, res_type=RES_TYPE, ) - wavfile.write( - os.path.join(self.wavs16k_dir, f"{sid}_{idx0}_{idx1}.wav"), - SAMPLE_RATE_16K, - audio_16k.astype(np.float32), + self._write_audio( + self.wavs16k_dir, f"{sid}_{idx0}_{idx1}", SAMPLE_RATE_16K, audio_16k ) def simple_cut( @@ -114,26 +120,15 @@ def simple_cut( if normalization_mode == "post": chunk = self._normalize_audio(chunk) if len(chunk) == chunk_length: + chunk_name = f"{sid}_{idx0}_{i // (chunk_length - overlap_length)}" # full SR for training - wavfile.write( - os.path.join( - self.gt_wavs_dir, - f"{sid}_{idx0}_{i // (chunk_length - overlap_length)}.wav", - ), - self.sr, - chunk.astype(np.float32), - ) + self._write_audio(self.gt_wavs_dir, chunk_name, self.sr, chunk) # 16KHz for feature extraction chunk_16k = librosa.resample( chunk, orig_sr=self.sr, target_sr=SAMPLE_RATE_16K, res_type=RES_TYPE ) - wavfile.write( - os.path.join( - self.wavs16k_dir, - f"{sid}_{idx0}_{i // (chunk_length - overlap_length)}.wav", - ), - SAMPLE_RATE_16K, - chunk_16k.astype(np.float32), + self._write_audio( + self.wavs16k_dir, chunk_name, SAMPLE_RATE_16K, chunk_16k ) i += chunk_length - overlap_length @@ -286,6 +281,7 @@ def preprocess_training_set( chunk_len: float, overlap_len: float, normalization_mode: str, + audio_format: str = "wav", ): if not os.path.exists(input_root): print(f"The dataset path does not exist: '{input_root}'.") @@ -295,7 +291,7 @@ def preprocess_training_set( print(f"The dataset path is not a directory: '{input_root}'.") sys.exit(1) start_time = time.time() - pp = PreProcess(sr, exp_dir) + pp = PreProcess(sr, exp_dir, audio_format) print(f"Starting preprocess with {num_processes} processes...") files = [] @@ -371,6 +367,7 @@ def preprocess_training_set( chunk_len = float(sys.argv[9]) overlap_len = float(sys.argv[10]) normalization_mode = str(sys.argv[11]) + audio_format = str(sys.argv[12]) if len(sys.argv) > 12 else "wav" preprocess_training_set( input_root, sample_rate, @@ -383,4 +380,5 @@ def preprocess_training_set( chunk_len, overlap_len, normalization_mode, + audio_format, ) diff --git a/rvc/train/process/extract_index.py b/rvc/train/process/extract_index.py index 7617173d2..c14e24fe5 100644 --- a/rvc/train/process/extract_index.py +++ b/rvc/train/process/extract_index.py @@ -40,7 +40,8 @@ ) sys.exit(1) - big_npy = np.concatenate(npys, axis=0) + # features may be float16 on disk, but faiss requires float32 + big_npy = np.concatenate(npys, axis=0).astype(np.float32) big_npy_idx = np.arange(big_npy.shape[0]) np.random.shuffle(big_npy_idx) diff --git a/rvc/train/train.py b/rvc/train/train.py index 5b274c926..8bf7efbab 100644 --- a/rvc/train/train.py +++ b/rvc/train/train.py @@ -178,8 +178,9 @@ def main(): os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = str(randint(20000, 55555)) # Check sample rate - wavs = glob.glob( - os.path.join(os.path.join(experiment_dir, "sliced_audios"), "*.wav") + sliced_dir = os.path.join(experiment_dir, "sliced_audios") + wavs = glob.glob(os.path.join(sliced_dir, "*.wav")) + glob.glob( + os.path.join(sliced_dir, "*.flac") ) if wavs: _, sr = load_wav_to_torch(wavs[0]) @@ -189,7 +190,7 @@ def main(): ) os._exit(1) else: - print("No wav file found.") + print("No sliced audio file found.") if torch.cuda.is_available(): device = torch.device("cuda") diff --git a/tabs/train/train.py b/tabs/train/train.py index 2344bf413..de059ca17 100644 --- a/tabs/train/train.py +++ b/tabs/train/train.py @@ -326,6 +326,8 @@ def _preprocess_with_toast(*args): def _extract_with_toast(*args): gr.Info(i18n("Extracting features...")) + args = list(args) + args[8] = "fp16" if args[8] == "fp16/uint8" else "fp32" result = run_extract_script(*args) if isinstance(result, str): if "error" in result.lower() or "failed" in result.lower(): @@ -471,6 +473,15 @@ def _extract_with_toast(*args): interactive=True, ) + audio_format = gr.Radio( + label=i18n("Sliced audio format"), + info=i18n( + "Format of the sliced audio files. 'wav' keeps the exact 32-bit float samples. 'flac' saves around 30% disk space, but quantizes to 24-bit (inaudible noise) and clips peaks above 0 dBFS, so avoid it with the 'none' normalization mode." + ), + choices=["wav", "flac"], + value="wav", + interactive=True, + ) with gr.Row(): process_effects = gr.Checkbox( label=i18n("Noise filter"), @@ -537,6 +548,7 @@ def _extract_with_toast(*args): chunk_len, overlap_len, normalization_mode, + audio_format, ], outputs=[preprocess_output_info], ) @@ -586,6 +598,24 @@ def _extract_with_toast(*args): value=True, interactive=True, ) + with gr.Accordion(i18n("Advanced Settings"), open=False): + extract_precision = gr.Radio( + label=i18n("Feature storage precision"), + info=i18n( + "Format of the extracted files. 'fp16/uint8' roughly halves disk usage: the coarse f0 in uint8 is lossless, while the features rounded to float16 lose precision. The quality impact is negligible, but irreversible without extracting again." + ), + choices=["fp32/int", "fp16/uint8"], + value="fp32/int", + interactive=True, + ) + remove_sliced_16k = gr.Checkbox( + label=i18n("Remove 16kHz sliced audios after extraction"), + info=i18n( + "Deletes the sliced_audios_16k folder after extraction. Training does not read those files, so quality is unaffected, but changing the pitch algorithm or embedder later will require preprocessing again." + ), + value=False, + interactive=True, + ) with gr.Row(visible=False) as embedder_custom: with gr.Accordion(i18n("Custom Embedder"), open=True): with gr.Row(): @@ -629,6 +659,8 @@ def _extract_with_toast(*args): embedder_model, embedder_model_custom, include_mutes, + extract_precision, + remove_sliced_16k, ], outputs=[extract_output_info], )